diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index aa5a50918a..50dbde2aee 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -20,4 +20,4 @@ - [x] I understand that this PR may be closed in case there was no previous discussion or issues. (This doesn't apply to typos!) - [x] I've added a test for each change that was introduced, and I tried as much as possible to make a single atomic change. - [x] I've updated the documentation accordingly. -- [x] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods +- [x] I ran `make lint` and `make type-check` (backend) and `cd web && npx lint-staged` (frontend) to appease the lint gods diff --git a/Makefile b/Makefile index 07afd8187e..60c32948b9 100644 --- a/Makefile +++ b/Makefile @@ -60,9 +60,10 @@ check: @echo "✅ Code check complete" lint: - @echo "🔧 Running ruff format, check with fixes, and import linter..." + @echo "🔧 Running ruff format, check with fixes, import linter, and dotenv-linter..." @uv run --project api --dev sh -c 'ruff format ./api && ruff check --fix ./api' @uv run --directory api --dev lint-imports + @uv run --project api --dev dotenv-linter ./api/.env.example ./web/.env.example @echo "✅ Linting complete" type-check: @@ -122,7 +123,7 @@ help: @echo "Backend Code Quality:" @echo " make format - Format code with ruff" @echo " make check - Check code with ruff" - @echo " make lint - Format and fix code with ruff" + @echo " make lint - Format, fix, and lint code (ruff, imports, dotenv)" @echo " make type-check - Run type checking with basedpyright" @echo " make test - Run backend unit tests" @echo "" diff --git a/api/.env.example b/api/.env.example index 88611e016e..44d770ed70 100644 --- a/api/.env.example +++ b/api/.env.example @@ -575,6 +575,10 @@ LOGSTORE_DUAL_WRITE_ENABLED=false # Enable dual-read fallback to SQL database when LogStore returns no results (default: true) # Useful for migration scenarios where historical data exists only in SQL database LOGSTORE_DUAL_READ_ENABLED=true +# Control flag for whether to write the `graph` field to LogStore. +# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; +# otherwise write an empty {} instead. Defaults to writing the `graph` field. +LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true # Celery beat configuration CELERY_BEAT_SCHEDULER_TIME=1 diff --git a/api/.importlinter b/api/.importlinter index 24ece72b30..2dec958788 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -3,9 +3,11 @@ root_packages = core configs controllers + extensions models tasks services +include_external_packages = True [importlinter:contract:workflow] name = Workflow @@ -33,6 +35,28 @@ ignore_imports = core.workflow.nodes.loop.loop_node -> core.workflow.graph core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine.command_channels +[importlinter:contract:workflow-infrastructure-dependencies] +name = Workflow Infrastructure Dependencies +type = forbidden +source_modules = + core.workflow +forbidden_modules = + extensions.ext_database + extensions.ext_redis +allow_indirect_imports = True +ignore_imports = + core.workflow.nodes.agent.agent_node -> extensions.ext_database + core.workflow.nodes.datasource.datasource_node -> extensions.ext_database + core.workflow.nodes.knowledge_index.knowledge_index_node -> extensions.ext_database + core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node -> extensions.ext_database + core.workflow.nodes.llm.file_saver -> extensions.ext_database + core.workflow.nodes.llm.llm_utils -> extensions.ext_database + core.workflow.nodes.llm.node -> extensions.ext_database + core.workflow.nodes.tool.tool_node -> extensions.ext_database + core.workflow.graph_engine.command_channels.redis_channel -> extensions.ext_redis + core.workflow.graph_engine.manager -> extensions.ext_redis + core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node -> extensions.ext_redis + [importlinter:contract:rsc] name = RSC type = layers diff --git a/api/Dockerfile b/api/Dockerfile index 02df91bfc1..a08d4e3aab 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -50,16 +50,33 @@ WORKDIR /app/api # Create non-root user ARG dify_uid=1001 +ARG NODE_MAJOR=22 +ARG NODE_PACKAGE_VERSION=22.21.0-1nodesource1 +ARG NODESOURCE_KEY_FPR=6F71F525282841EEDAF851B42F59B5F99B1BE0B4 RUN groupadd -r -g ${dify_uid} dify && \ useradd -r -u ${dify_uid} -g ${dify_uid} -s /bin/bash dify && \ chown -R dify:dify /app RUN \ apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource.gpg \ + && gpg --show-keys --with-colons /tmp/nodesource.gpg \ + | awk -F: '/^fpr:/ {print $10}' \ + | grep -Fx "${NODESOURCE_KEY_FPR}" \ + && gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource.gpg \ + && rm -f /tmp/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" \ + > /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ # Install dependencies && apt-get install -y --no-install-recommends \ # basic environment - curl nodejs \ + nodejs=${NODE_PACKAGE_VERSION} \ # for gmpy2 \ libgmp-dev libmpfr-dev libmpc-dev \ # For Security @@ -79,7 +96,8 @@ COPY --from=packages --chown=dify:dify ${VIRTUAL_ENV} ${VIRTUAL_ENV} ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" # Download nltk data -RUN mkdir -p /usr/local/share/nltk_data && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger'); nltk.download('stopwords')" \ +RUN mkdir -p /usr/local/share/nltk_data \ + && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; from unstructured.nlp.tokenize import download_nltk_packages; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger'); nltk.download('stopwords'); download_nltk_packages()" \ && chmod -R 755 /usr/local/share/nltk_data ENV TIKTOKEN_CACHE_DIR=/app/api/.tiktoken_cache diff --git a/api/commands.py b/api/commands.py index a8d89ac200..7ebf5b4874 100644 --- a/api/commands.py +++ b/api/commands.py @@ -235,7 +235,7 @@ def migrate_annotation_vector_database(): if annotations: for annotation in annotations: document = Document( - page_content=annotation.question, + page_content=annotation.question_text, metadata={"annotation_id": annotation.id, "app_id": app.id, "doc_id": annotation.id}, ) documents.append(document) @@ -1184,6 +1184,217 @@ def remove_orphaned_files_on_storage(force: bool): click.echo(click.style(f"Removed {removed_files} orphaned files, with {error_files} errors.", fg="yellow")) +@click.command("file-usage", help="Query file usages and show where files are referenced.") +@click.option("--file-id", type=str, default=None, help="Filter by file UUID.") +@click.option("--key", type=str, default=None, help="Filter by storage key.") +@click.option("--src", type=str, default=None, help="Filter by table.column pattern (e.g., 'documents.%' or '%.icon').") +@click.option("--limit", type=int, default=100, help="Limit number of results (default: 100).") +@click.option("--offset", type=int, default=0, help="Offset for pagination (default: 0).") +@click.option("--json", "output_json", is_flag=True, help="Output results in JSON format.") +def file_usage( + file_id: str | None, + key: str | None, + src: str | None, + limit: int, + offset: int, + output_json: bool, +): + """ + Query file usages and show where files are referenced in the database. + + This command reuses the same reference checking logic as clear-orphaned-file-records + and displays detailed information about where each file is referenced. + """ + # define tables and columns to process + files_tables = [ + {"table": "upload_files", "id_column": "id", "key_column": "key"}, + {"table": "tool_files", "id_column": "id", "key_column": "file_key"}, + ] + ids_tables = [ + {"type": "uuid", "table": "message_files", "column": "upload_file_id", "pk_column": "id"}, + {"type": "text", "table": "documents", "column": "data_source_info", "pk_column": "id"}, + {"type": "text", "table": "document_segments", "column": "content", "pk_column": "id"}, + {"type": "text", "table": "messages", "column": "answer", "pk_column": "id"}, + {"type": "text", "table": "workflow_node_executions", "column": "inputs", "pk_column": "id"}, + {"type": "text", "table": "workflow_node_executions", "column": "process_data", "pk_column": "id"}, + {"type": "text", "table": "workflow_node_executions", "column": "outputs", "pk_column": "id"}, + {"type": "text", "table": "conversations", "column": "introduction", "pk_column": "id"}, + {"type": "text", "table": "conversations", "column": "system_instruction", "pk_column": "id"}, + {"type": "text", "table": "accounts", "column": "avatar", "pk_column": "id"}, + {"type": "text", "table": "apps", "column": "icon", "pk_column": "id"}, + {"type": "text", "table": "sites", "column": "icon", "pk_column": "id"}, + {"type": "json", "table": "messages", "column": "inputs", "pk_column": "id"}, + {"type": "json", "table": "messages", "column": "message", "pk_column": "id"}, + ] + + # Stream file usages with pagination to avoid holding all results in memory + paginated_usages = [] + total_count = 0 + + # First, build a mapping of file_id -> storage_key from the base tables + file_key_map = {} + for files_table in files_tables: + query = f"SELECT {files_table['id_column']}, {files_table['key_column']} FROM {files_table['table']}" + with db.engine.begin() as conn: + rs = conn.execute(sa.text(query)) + for row in rs: + file_key_map[str(row[0])] = f"{files_table['table']}:{row[1]}" + + # If filtering by key or file_id, verify it exists + if file_id and file_id not in file_key_map: + if output_json: + click.echo(json.dumps({"error": f"File ID {file_id} not found in base tables"})) + else: + click.echo(click.style(f"File ID {file_id} not found in base tables.", fg="red")) + return + + if key: + valid_prefixes = {f"upload_files:{key}", f"tool_files:{key}"} + matching_file_ids = [fid for fid, fkey in file_key_map.items() if fkey in valid_prefixes] + if not matching_file_ids: + if output_json: + click.echo(json.dumps({"error": f"Key {key} not found in base tables"})) + else: + click.echo(click.style(f"Key {key} not found in base tables.", fg="red")) + return + + guid_regexp = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" + + # For each reference table/column, find matching file IDs and record the references + for ids_table in ids_tables: + src_filter = f"{ids_table['table']}.{ids_table['column']}" + + # Skip if src filter doesn't match (use fnmatch for wildcard patterns) + if src: + if "%" in src or "_" in src: + import fnmatch + + # Convert SQL LIKE wildcards to fnmatch wildcards (% -> *, _ -> ?) + pattern = src.replace("%", "*").replace("_", "?") + if not fnmatch.fnmatch(src_filter, pattern): + continue + else: + if src_filter != src: + continue + + if ids_table["type"] == "uuid": + # Direct UUID match + query = ( + f"SELECT {ids_table['pk_column']}, {ids_table['column']} " + f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL" + ) + with db.engine.begin() as conn: + rs = conn.execute(sa.text(query)) + for row in rs: + record_id = str(row[0]) + ref_file_id = str(row[1]) + if ref_file_id not in file_key_map: + continue + storage_key = file_key_map[ref_file_id] + + # Apply filters + if file_id and ref_file_id != file_id: + continue + if key and not storage_key.endswith(key): + continue + + # Only collect items within the requested page range + if offset <= total_count < offset + limit: + paginated_usages.append( + { + "src": f"{ids_table['table']}.{ids_table['column']}", + "record_id": record_id, + "file_id": ref_file_id, + "key": storage_key, + } + ) + total_count += 1 + + elif ids_table["type"] in ("text", "json"): + # Extract UUIDs from text/json content + column_cast = f"{ids_table['column']}::text" if ids_table["type"] == "json" else ids_table["column"] + query = ( + f"SELECT {ids_table['pk_column']}, {column_cast} " + f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL" + ) + with db.engine.begin() as conn: + rs = conn.execute(sa.text(query)) + for row in rs: + record_id = str(row[0]) + content = str(row[1]) + + # Find all UUIDs in the content + import re + + uuid_pattern = re.compile(guid_regexp, re.IGNORECASE) + matches = uuid_pattern.findall(content) + + for ref_file_id in matches: + if ref_file_id not in file_key_map: + continue + storage_key = file_key_map[ref_file_id] + + # Apply filters + if file_id and ref_file_id != file_id: + continue + if key and not storage_key.endswith(key): + continue + + # Only collect items within the requested page range + if offset <= total_count < offset + limit: + paginated_usages.append( + { + "src": f"{ids_table['table']}.{ids_table['column']}", + "record_id": record_id, + "file_id": ref_file_id, + "key": storage_key, + } + ) + total_count += 1 + + # Output results + if output_json: + result = { + "total": total_count, + "offset": offset, + "limit": limit, + "usages": paginated_usages, + } + click.echo(json.dumps(result, indent=2)) + else: + click.echo( + click.style(f"Found {total_count} file usages (showing {len(paginated_usages)} results)", fg="white") + ) + click.echo("") + + if not paginated_usages: + click.echo(click.style("No file usages found matching the specified criteria.", fg="yellow")) + return + + # Print table header + click.echo( + click.style( + f"{'Src (Table.Column)':<50} {'Record ID':<40} {'File ID':<40} {'Storage Key':<60}", + fg="cyan", + ) + ) + click.echo(click.style("-" * 190, fg="white")) + + # Print each usage + for usage in paginated_usages: + click.echo(f"{usage['src']:<50} {usage['record_id']:<40} {usage['file_id']:<40} {usage['key']:<60}") + + # Show pagination info + if offset + limit < total_count: + click.echo("") + click.echo( + click.style( + f"Showing {offset + 1}-{offset + len(paginated_usages)} of {total_count} results", fg="white" + ) + ) + click.echo(click.style(f"Use --offset {offset + limit} to see next page", fg="white")) + + @click.command("setup-system-tool-oauth-client", help="Setup system tool oauth client.") @click.option("--provider", prompt=True, help="Provider name") @click.option("--client-params", prompt=True, help="Client Params") diff --git a/api/configs/middleware/vdb/milvus_config.py b/api/configs/middleware/vdb/milvus_config.py index 05cee51cc9..eb9b0ac2ab 100644 --- a/api/configs/middleware/vdb/milvus_config.py +++ b/api/configs/middleware/vdb/milvus_config.py @@ -16,7 +16,6 @@ class MilvusConfig(BaseSettings): description="Authentication token for Milvus, if token-based authentication is enabled", default=None, ) - MILVUS_USER: str | None = Field( description="Username for authenticating with Milvus, if username/password authentication is enabled", default=None, diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 44cf89d6a9..d66bb7063f 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -1,14 +1,16 @@ import re import uuid -from typing import Literal +from datetime import datetime +from typing import Any, Literal, TypeAlias from flask import request -from flask_restx import Resource, fields, marshal, marshal_with -from pydantic import BaseModel, Field, field_validator +from flask_restx import Resource +from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator from sqlalchemy import select from sqlalchemy.orm import Session from werkzeug.exceptions import BadRequest +from controllers.common.schema import register_schema_models from controllers.console import console_ns from controllers.console.app.wraps import get_app_model from controllers.console.wraps import ( @@ -19,27 +21,19 @@ from controllers.console.wraps import ( is_admin_or_owner_required, setup_required, ) +from core.file import helpers as file_helpers from core.ops.ops_trace_manager import OpsTraceManager from core.workflow.enums import NodeType from extensions.ext_database import db -from fields.app_fields import ( - deleted_tool_fields, - model_config_fields, - model_config_partial_fields, - site_fields, - tag_fields, -) -from fields.workflow_fields import workflow_partial_fields as _workflow_partial_fields_dict -from libs.helper import AppIconUrlField, TimestampField from libs.login import current_account_with_tenant, login_required from models import App, Workflow +from models.model import IconType from services.app_dsl_service import AppDslService, ImportMode from services.app_service import AppService from services.enterprise.enterprise_service import EnterpriseService from services.feature_service import FeatureService ALLOW_CREATE_APP_MODES = ["chat", "agent-chat", "advanced-chat", "workflow", "completion"] -DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" class AppListQuery(BaseModel): @@ -192,124 +186,292 @@ class AppTracePayload(BaseModel): return value -def reg(cls: type[BaseModel]): - console_ns.schema_model(cls.__name__, cls.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)) +JSONValue: TypeAlias = Any -reg(AppListQuery) -reg(CreateAppPayload) -reg(UpdateAppPayload) -reg(CopyAppPayload) -reg(AppExportQuery) -reg(AppNamePayload) -reg(AppIconPayload) -reg(AppSiteStatusPayload) -reg(AppApiStatusPayload) -reg(AppTracePayload) +class ResponseModel(BaseModel): + model_config = ConfigDict( + from_attributes=True, + extra="ignore", + populate_by_name=True, + serialize_by_alias=True, + protected_namespaces=(), + ) -# Register models for flask_restx to avoid dict type issues in Swagger -# Register base models first -tag_model = console_ns.model("Tag", tag_fields) -workflow_partial_model = console_ns.model("WorkflowPartial", _workflow_partial_fields_dict) +def _to_timestamp(value: datetime | int | None) -> int | None: + if isinstance(value, datetime): + return int(value.timestamp()) + return value -model_config_model = console_ns.model("ModelConfig", model_config_fields) -model_config_partial_model = console_ns.model("ModelConfigPartial", model_config_partial_fields) +def _build_icon_url(icon_type: str | IconType | None, icon: str | None) -> str | None: + if icon is None or icon_type is None: + return None + icon_type_value = icon_type.value if isinstance(icon_type, IconType) else str(icon_type) + if icon_type_value.lower() != IconType.IMAGE.value: + return None + return file_helpers.get_signed_file_url(icon) -deleted_tool_model = console_ns.model("DeletedTool", deleted_tool_fields) -site_model = console_ns.model("Site", site_fields) +class Tag(ResponseModel): + id: str + name: str + type: str -app_partial_model = console_ns.model( - "AppPartial", - { - "id": fields.String, - "name": fields.String, - "max_active_requests": fields.Raw(), - "description": fields.String(attribute="desc_or_prompt"), - "mode": fields.String(attribute="mode_compatible_with_agent"), - "icon_type": fields.String, - "icon": fields.String, - "icon_background": fields.String, - "icon_url": AppIconUrlField, - "model_config": fields.Nested(model_config_partial_model, attribute="app_model_config", allow_null=True), - "workflow": fields.Nested(workflow_partial_model, allow_null=True), - "use_icon_as_answer_icon": fields.Boolean, - "created_by": fields.String, - "created_at": TimestampField, - "updated_by": fields.String, - "updated_at": TimestampField, - "tags": fields.List(fields.Nested(tag_model)), - "access_mode": fields.String, - "create_user_name": fields.String, - "author_name": fields.String, - "has_draft_trigger": fields.Boolean, - }, -) -app_detail_model = console_ns.model( - "AppDetail", - { - "id": fields.String, - "name": fields.String, - "description": fields.String, - "mode": fields.String(attribute="mode_compatible_with_agent"), - "icon": fields.String, - "icon_background": fields.String, - "enable_site": fields.Boolean, - "enable_api": fields.Boolean, - "model_config": fields.Nested(model_config_model, attribute="app_model_config", allow_null=True), - "workflow": fields.Nested(workflow_partial_model, allow_null=True), - "tracing": fields.Raw, - "use_icon_as_answer_icon": fields.Boolean, - "created_by": fields.String, - "created_at": TimestampField, - "updated_by": fields.String, - "updated_at": TimestampField, - "access_mode": fields.String, - "tags": fields.List(fields.Nested(tag_model)), - }, -) +class WorkflowPartial(ResponseModel): + id: str + created_by: str | None = None + created_at: int | None = None + updated_by: str | None = None + updated_at: int | None = None -app_detail_with_site_model = console_ns.model( - "AppDetailWithSite", - { - "id": fields.String, - "name": fields.String, - "description": fields.String, - "mode": fields.String(attribute="mode_compatible_with_agent"), - "icon_type": fields.String, - "icon": fields.String, - "icon_background": fields.String, - "icon_url": AppIconUrlField, - "enable_site": fields.Boolean, - "enable_api": fields.Boolean, - "model_config": fields.Nested(model_config_model, attribute="app_model_config", allow_null=True), - "workflow": fields.Nested(workflow_partial_model, allow_null=True), - "api_base_url": fields.String, - "use_icon_as_answer_icon": fields.Boolean, - "max_active_requests": fields.Integer, - "created_by": fields.String, - "created_at": TimestampField, - "updated_by": fields.String, - "updated_at": TimestampField, - "deleted_tools": fields.List(fields.Nested(deleted_tool_model)), - "access_mode": fields.String, - "tags": fields.List(fields.Nested(tag_model)), - "site": fields.Nested(site_model), - }, -) + @field_validator("created_at", "updated_at", mode="before") + @classmethod + def _normalize_timestamp(cls, value: datetime | int | None) -> int | None: + return _to_timestamp(value) -app_pagination_model = console_ns.model( - "AppPagination", - { - "page": fields.Integer, - "limit": fields.Integer(attribute="per_page"), - "total": fields.Integer, - "has_more": fields.Boolean(attribute="has_next"), - "data": fields.List(fields.Nested(app_partial_model), attribute="items"), - }, + +class ModelConfigPartial(ResponseModel): + model: JSONValue | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model")) + pre_prompt: str | None = None + created_by: str | None = None + created_at: int | None = None + updated_by: str | None = None + updated_at: int | None = None + + @field_validator("created_at", "updated_at", mode="before") + @classmethod + def _normalize_timestamp(cls, value: datetime | int | None) -> int | None: + return _to_timestamp(value) + + +class ModelConfig(ResponseModel): + opening_statement: str | None = None + suggested_questions: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("suggested_questions_list", "suggested_questions") + ) + suggested_questions_after_answer: JSONValue | None = Field( + default=None, + validation_alias=AliasChoices("suggested_questions_after_answer_dict", "suggested_questions_after_answer"), + ) + speech_to_text: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("speech_to_text_dict", "speech_to_text") + ) + text_to_speech: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("text_to_speech_dict", "text_to_speech") + ) + retriever_resource: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("retriever_resource_dict", "retriever_resource") + ) + annotation_reply: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("annotation_reply_dict", "annotation_reply") + ) + more_like_this: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("more_like_this_dict", "more_like_this") + ) + sensitive_word_avoidance: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("sensitive_word_avoidance_dict", "sensitive_word_avoidance") + ) + external_data_tools: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("external_data_tools_list", "external_data_tools") + ) + model: JSONValue | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model")) + user_input_form: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("user_input_form_list", "user_input_form") + ) + dataset_query_variable: str | None = None + pre_prompt: str | None = None + agent_mode: JSONValue | None = Field(default=None, validation_alias=AliasChoices("agent_mode_dict", "agent_mode")) + prompt_type: str | None = None + chat_prompt_config: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("chat_prompt_config_dict", "chat_prompt_config") + ) + completion_prompt_config: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("completion_prompt_config_dict", "completion_prompt_config") + ) + dataset_configs: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("dataset_configs_dict", "dataset_configs") + ) + file_upload: JSONValue | None = Field( + default=None, validation_alias=AliasChoices("file_upload_dict", "file_upload") + ) + created_by: str | None = None + created_at: int | None = None + updated_by: str | None = None + updated_at: int | None = None + + @field_validator("created_at", "updated_at", mode="before") + @classmethod + def _normalize_timestamp(cls, value: datetime | int | None) -> int | None: + return _to_timestamp(value) + + +class Site(ResponseModel): + access_token: str | None = Field(default=None, validation_alias="code") + code: str | None = None + title: str | None = None + icon_type: str | IconType | None = None + icon: str | None = None + icon_background: str | None = None + description: str | None = None + default_language: str | None = None + chat_color_theme: str | None = None + chat_color_theme_inverted: bool | None = None + customize_domain: str | None = None + copyright: str | None = None + privacy_policy: str | None = None + custom_disclaimer: str | None = None + customize_token_strategy: str | None = None + prompt_public: bool | None = None + app_base_url: str | None = None + show_workflow_steps: bool | None = None + use_icon_as_answer_icon: bool | None = None + created_by: str | None = None + created_at: int | None = None + updated_by: str | None = None + updated_at: int | None = None + + @computed_field(return_type=str | None) # type: ignore + @property + def icon_url(self) -> str | None: + return _build_icon_url(self.icon_type, self.icon) + + @field_validator("icon_type", mode="before") + @classmethod + def _normalize_icon_type(cls, value: str | IconType | None) -> str | None: + if isinstance(value, IconType): + return value.value + return value + + @field_validator("created_at", "updated_at", mode="before") + @classmethod + def _normalize_timestamp(cls, value: datetime | int | None) -> int | None: + return _to_timestamp(value) + + +class DeletedTool(ResponseModel): + type: str + tool_name: str + provider_id: str + + +class AppPartial(ResponseModel): + id: str + name: str + max_active_requests: int | None = None + description: str | None = Field(default=None, validation_alias=AliasChoices("desc_or_prompt", "description")) + mode: str = Field(validation_alias="mode_compatible_with_agent") + icon_type: str | None = None + icon: str | None = None + icon_background: str | None = None + model_config_: ModelConfigPartial | None = Field( + default=None, + validation_alias=AliasChoices("app_model_config", "model_config"), + alias="model_config", + ) + workflow: WorkflowPartial | None = None + use_icon_as_answer_icon: bool | None = None + created_by: str | None = None + created_at: int | None = None + updated_by: str | None = None + updated_at: int | None = None + tags: list[Tag] = Field(default_factory=list) + access_mode: str | None = None + create_user_name: str | None = None + author_name: str | None = None + has_draft_trigger: bool | None = None + + @computed_field(return_type=str | None) # type: ignore + @property + def icon_url(self) -> str | None: + return _build_icon_url(self.icon_type, self.icon) + + @field_validator("created_at", "updated_at", mode="before") + @classmethod + def _normalize_timestamp(cls, value: datetime | int | None) -> int | None: + return _to_timestamp(value) + + +class AppDetail(ResponseModel): + id: str + name: str + description: str | None = None + mode: str = Field(validation_alias="mode_compatible_with_agent") + icon: str | None = None + icon_background: str | None = None + enable_site: bool + enable_api: bool + model_config_: ModelConfig | None = Field( + default=None, + validation_alias=AliasChoices("app_model_config", "model_config"), + alias="model_config", + ) + workflow: WorkflowPartial | None = None + tracing: JSONValue | None = None + use_icon_as_answer_icon: bool | None = None + created_by: str | None = None + created_at: int | None = None + updated_by: str | None = None + updated_at: int | None = None + access_mode: str | None = None + tags: list[Tag] = Field(default_factory=list) + + @field_validator("created_at", "updated_at", mode="before") + @classmethod + def _normalize_timestamp(cls, value: datetime | int | None) -> int | None: + return _to_timestamp(value) + + +class AppDetailWithSite(AppDetail): + icon_type: str | None = None + api_base_url: str | None = None + max_active_requests: int | None = None + deleted_tools: list[DeletedTool] = Field(default_factory=list) + site: Site | None = None + + @computed_field(return_type=str | None) # type: ignore + @property + def icon_url(self) -> str | None: + return _build_icon_url(self.icon_type, self.icon) + + +class AppPagination(ResponseModel): + page: int + limit: int = Field(validation_alias=AliasChoices("per_page", "limit")) + total: int + has_more: bool = Field(validation_alias=AliasChoices("has_next", "has_more")) + data: list[AppPartial] = Field(validation_alias=AliasChoices("items", "data")) + + +class AppExportResponse(ResponseModel): + data: str + + +register_schema_models( + console_ns, + AppListQuery, + CreateAppPayload, + UpdateAppPayload, + CopyAppPayload, + AppExportQuery, + AppNamePayload, + AppIconPayload, + AppSiteStatusPayload, + AppApiStatusPayload, + AppTracePayload, + Tag, + WorkflowPartial, + ModelConfigPartial, + ModelConfig, + Site, + DeletedTool, + AppPartial, + AppDetail, + AppDetailWithSite, + AppPagination, + AppExportResponse, ) @@ -318,7 +480,7 @@ class AppListApi(Resource): @console_ns.doc("list_apps") @console_ns.doc(description="Get list of applications with pagination and filtering") @console_ns.expect(console_ns.models[AppListQuery.__name__]) - @console_ns.response(200, "Success", app_pagination_model) + @console_ns.response(200, "Success", console_ns.models[AppPagination.__name__]) @setup_required @login_required @account_initialization_required @@ -334,7 +496,8 @@ class AppListApi(Resource): app_service = AppService() app_pagination = app_service.get_paginate_apps(current_user.id, current_tenant_id, args_dict) if not app_pagination: - return {"data": [], "total": 0, "page": 1, "limit": 20, "has_more": False} + empty = AppPagination(page=args.page, limit=args.limit, total=0, has_more=False, data=[]) + return empty.model_dump(mode="json"), 200 if FeatureService.get_system_features().webapp_auth.enabled: app_ids = [str(app.id) for app in app_pagination.items] @@ -378,18 +541,18 @@ class AppListApi(Resource): for app in app_pagination.items: app.has_draft_trigger = str(app.id) in draft_trigger_app_ids - return marshal(app_pagination, app_pagination_model), 200 + pagination_model = AppPagination.model_validate(app_pagination, from_attributes=True) + return pagination_model.model_dump(mode="json"), 200 @console_ns.doc("create_app") @console_ns.doc(description="Create a new application") @console_ns.expect(console_ns.models[CreateAppPayload.__name__]) - @console_ns.response(201, "App created successfully", app_detail_model) + @console_ns.response(201, "App created successfully", console_ns.models[AppDetail.__name__]) @console_ns.response(403, "Insufficient permissions") @console_ns.response(400, "Invalid request parameters") @setup_required @login_required @account_initialization_required - @marshal_with(app_detail_model) @cloud_edition_billing_resource_check("apps") @edit_permission_required def post(self): @@ -399,8 +562,8 @@ class AppListApi(Resource): app_service = AppService() app = app_service.create_app(current_tenant_id, args.model_dump(), current_user) - - return app, 201 + app_detail = AppDetail.model_validate(app, from_attributes=True) + return app_detail.model_dump(mode="json"), 201 @console_ns.route("/apps/") @@ -408,13 +571,12 @@ class AppApi(Resource): @console_ns.doc("get_app_detail") @console_ns.doc(description="Get application details") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Success", app_detail_with_site_model) + @console_ns.response(200, "Success", console_ns.models[AppDetailWithSite.__name__]) @setup_required @login_required @account_initialization_required @enterprise_license_required - @get_app_model - @marshal_with(app_detail_with_site_model) + @get_app_model(mode=None) def get(self, app_model): """Get app detail""" app_service = AppService() @@ -425,21 +587,21 @@ class AppApi(Resource): app_setting = EnterpriseService.WebAppAuth.get_app_access_mode_by_id(app_id=str(app_model.id)) app_model.access_mode = app_setting.access_mode - return app_model + response_model = AppDetailWithSite.model_validate(app_model, from_attributes=True) + return response_model.model_dump(mode="json") @console_ns.doc("update_app") @console_ns.doc(description="Update application details") @console_ns.doc(params={"app_id": "Application ID"}) @console_ns.expect(console_ns.models[UpdateAppPayload.__name__]) - @console_ns.response(200, "App updated successfully", app_detail_with_site_model) + @console_ns.response(200, "App updated successfully", console_ns.models[AppDetailWithSite.__name__]) @console_ns.response(403, "Insufficient permissions") @console_ns.response(400, "Invalid request parameters") @setup_required @login_required @account_initialization_required - @get_app_model + @get_app_model(mode=None) @edit_permission_required - @marshal_with(app_detail_with_site_model) def put(self, app_model): """Update app""" args = UpdateAppPayload.model_validate(console_ns.payload) @@ -456,8 +618,8 @@ class AppApi(Resource): "max_active_requests": args.max_active_requests or 0, } app_model = app_service.update_app(app_model, args_dict) - - return app_model + response_model = AppDetailWithSite.model_validate(app_model, from_attributes=True) + return response_model.model_dump(mode="json") @console_ns.doc("delete_app") @console_ns.doc(description="Delete application") @@ -483,14 +645,13 @@ class AppCopyApi(Resource): @console_ns.doc(description="Create a copy of an existing application") @console_ns.doc(params={"app_id": "Application ID to copy"}) @console_ns.expect(console_ns.models[CopyAppPayload.__name__]) - @console_ns.response(201, "App copied successfully", app_detail_with_site_model) + @console_ns.response(201, "App copied successfully", console_ns.models[AppDetailWithSite.__name__]) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @account_initialization_required - @get_app_model + @get_app_model(mode=None) @edit_permission_required - @marshal_with(app_detail_with_site_model) def post(self, app_model): """Copy app""" # The role of the current user in the ta table must be admin, owner, or editor @@ -516,7 +677,8 @@ class AppCopyApi(Resource): stmt = select(App).where(App.id == result.app_id) app = session.scalar(stmt) - return app, 201 + response_model = AppDetailWithSite.model_validate(app, from_attributes=True) + return response_model.model_dump(mode="json"), 201 @console_ns.route("/apps//export") @@ -525,11 +687,7 @@ class AppExportApi(Resource): @console_ns.doc(description="Export application configuration as DSL") @console_ns.doc(params={"app_id": "Application ID to export"}) @console_ns.expect(console_ns.models[AppExportQuery.__name__]) - @console_ns.response( - 200, - "App exported successfully", - console_ns.model("AppExportResponse", {"data": fields.String(description="DSL export data")}), - ) + @console_ns.response(200, "App exported successfully", console_ns.models[AppExportResponse.__name__]) @console_ns.response(403, "Insufficient permissions") @get_app_model @setup_required @@ -540,13 +698,14 @@ class AppExportApi(Resource): """Export app""" args = AppExportQuery.model_validate(request.args.to_dict(flat=True)) # type: ignore - return { - "data": AppDslService.export_dsl( + payload = AppExportResponse( + data=AppDslService.export_dsl( app_model=app_model, include_secret=args.include_secret, workflow_id=args.workflow_id, ) - } + ) + return payload.model_dump(mode="json") @console_ns.route("/apps//name") @@ -555,20 +714,19 @@ class AppNameApi(Resource): @console_ns.doc(description="Check if app name is available") @console_ns.doc(params={"app_id": "Application ID"}) @console_ns.expect(console_ns.models[AppNamePayload.__name__]) - @console_ns.response(200, "Name availability checked") + @console_ns.response(200, "Name availability checked", console_ns.models[AppDetail.__name__]) @setup_required @login_required @account_initialization_required - @get_app_model - @marshal_with(app_detail_model) + @get_app_model(mode=None) @edit_permission_required def post(self, app_model): args = AppNamePayload.model_validate(console_ns.payload) app_service = AppService() app_model = app_service.update_app_name(app_model, args.name) - - return app_model + response_model = AppDetail.model_validate(app_model, from_attributes=True) + return response_model.model_dump(mode="json") @console_ns.route("/apps//icon") @@ -582,16 +740,15 @@ class AppIconApi(Resource): @setup_required @login_required @account_initialization_required - @get_app_model - @marshal_with(app_detail_model) + @get_app_model(mode=None) @edit_permission_required def post(self, app_model): args = AppIconPayload.model_validate(console_ns.payload or {}) app_service = AppService() app_model = app_service.update_app_icon(app_model, args.icon or "", args.icon_background or "") - - return app_model + response_model = AppDetail.model_validate(app_model, from_attributes=True) + return response_model.model_dump(mode="json") @console_ns.route("/apps//site-enable") @@ -600,21 +757,20 @@ class AppSiteStatus(Resource): @console_ns.doc(description="Enable or disable app site") @console_ns.doc(params={"app_id": "Application ID"}) @console_ns.expect(console_ns.models[AppSiteStatusPayload.__name__]) - @console_ns.response(200, "Site status updated successfully", app_detail_model) + @console_ns.response(200, "Site status updated successfully", console_ns.models[AppDetail.__name__]) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @account_initialization_required - @get_app_model - @marshal_with(app_detail_model) + @get_app_model(mode=None) @edit_permission_required def post(self, app_model): args = AppSiteStatusPayload.model_validate(console_ns.payload) app_service = AppService() app_model = app_service.update_app_site_status(app_model, args.enable_site) - - return app_model + response_model = AppDetail.model_validate(app_model, from_attributes=True) + return response_model.model_dump(mode="json") @console_ns.route("/apps//api-enable") @@ -623,21 +779,20 @@ class AppApiStatus(Resource): @console_ns.doc(description="Enable or disable app API") @console_ns.doc(params={"app_id": "Application ID"}) @console_ns.expect(console_ns.models[AppApiStatusPayload.__name__]) - @console_ns.response(200, "API status updated successfully", app_detail_model) + @console_ns.response(200, "API status updated successfully", console_ns.models[AppDetail.__name__]) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @is_admin_or_owner_required @account_initialization_required - @get_app_model - @marshal_with(app_detail_model) + @get_app_model(mode=None) def post(self, app_model): args = AppApiStatusPayload.model_validate(console_ns.payload) app_service = AppService() app_model = app_service.update_app_api_status(app_model, args.enable_api) - - return app_model + response_model = AppDetail.model_validate(app_model, from_attributes=True) + return response_model.model_dump(mode="json") @console_ns.route("/apps//trace") diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index ef2f86d4be..56816dd462 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -348,10 +348,13 @@ class CompletionConversationApi(Resource): ) if args.keyword: + from libs.helper import escape_like_pattern + + escaped_keyword = escape_like_pattern(args.keyword) query = query.join(Message, Message.conversation_id == Conversation.id).where( or_( - Message.query.ilike(f"%{args.keyword}%"), - Message.answer.ilike(f"%{args.keyword}%"), + Message.query.ilike(f"%{escaped_keyword}%", escape="\\"), + Message.answer.ilike(f"%{escaped_keyword}%", escape="\\"), ) ) @@ -460,7 +463,10 @@ class ChatConversationApi(Resource): query = sa.select(Conversation).where(Conversation.app_id == app_model.id, Conversation.is_deleted.is_(False)) if args.keyword: - keyword_filter = f"%{args.keyword}%" + from libs.helper import escape_like_pattern + + escaped_keyword = escape_like_pattern(args.keyword) + keyword_filter = f"%{escaped_keyword}%" query = ( query.join( Message, @@ -469,11 +475,11 @@ class ChatConversationApi(Resource): .join(subquery, subquery.c.conversation_id == Conversation.id) .where( or_( - Message.query.ilike(keyword_filter), - Message.answer.ilike(keyword_filter), - Conversation.name.ilike(keyword_filter), - Conversation.introduction.ilike(keyword_filter), - subquery.c.from_end_user_session_id.ilike(keyword_filter), + Message.query.ilike(keyword_filter, escape="\\"), + Message.answer.ilike(keyword_filter, escape="\\"), + Conversation.name.ilike(keyword_filter, escape="\\"), + Conversation.introduction.ilike(keyword_filter, escape="\\"), + subquery.c.from_end_user_session_id.ilike(keyword_filter, escape="\\"), ), ) .group_by(Conversation.id) diff --git a/api/controllers/console/auth/login.py b/api/controllers/console/auth/login.py index 772d98822e..4a52bf8abe 100644 --- a/api/controllers/console/auth/login.py +++ b/api/controllers/console/auth/login.py @@ -1,3 +1,5 @@ +from typing import Any + import flask_login from flask import make_response, request from flask_restx import Resource @@ -96,14 +98,13 @@ class LoginApi(Resource): if is_login_error_rate_limit: raise EmailPasswordLoginLimitError() - # TODO: why invitation is re-assigned with different type? - invitation = args.invite_token # type: ignore - if invitation: - invitation = RegisterService.get_invitation_if_token_valid(None, args.email, invitation) # type: ignore + invitation_data: dict[str, Any] | None = None + if args.invite_token: + invitation_data = RegisterService.get_invitation_if_token_valid(None, args.email, args.invite_token) try: - if invitation: - data = invitation.get("data", {}) # type: ignore + if invitation_data: + data = invitation_data.get("data", {}) invitee_email = data.get("email") if data else None if invitee_email != args.email: raise InvalidEmailError() diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index e94768f985..ac78d3854b 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -751,12 +751,12 @@ class DocumentApi(DocumentResource): elif metadata == "without": dataset_process_rules = DatasetService.get_process_rules(dataset_id) document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {} - data_source_info = document.data_source_detail_dict response = { "id": document.id, "position": document.position, "data_source_type": document.data_source_type, - "data_source_info": data_source_info, + "data_source_info": document.data_source_info_dict, + "data_source_detail_dict": document.data_source_detail_dict, "dataset_process_rule_id": document.dataset_process_rule_id, "dataset_process_rule": dataset_process_rules, "document_process_rule": document_process_rules, @@ -784,12 +784,12 @@ class DocumentApi(DocumentResource): else: dataset_process_rules = DatasetService.get_process_rules(dataset_id) document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {} - data_source_info = document.data_source_detail_dict response = { "id": document.id, "position": document.position, "data_source_type": document.data_source_type, - "data_source_info": data_source_info, + "data_source_info": document.data_source_info_dict, + "data_source_detail_dict": document.data_source_detail_dict, "dataset_process_rule_id": document.dataset_process_rule_id, "dataset_process_rule": dataset_process_rules, "document_process_rule": document_process_rules, diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 5a536af6d2..16fecb41c6 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -30,6 +30,7 @@ from core.model_runtime.entities.model_entities import ModelType from extensions.ext_database import db from extensions.ext_redis import redis_client from fields.segment_fields import child_chunk_fields, segment_fields +from libs.helper import escape_like_pattern from libs.login import current_account_with_tenant, login_required from models.dataset import ChildChunk, DocumentSegment from models.model import UploadFile @@ -145,6 +146,8 @@ class DatasetDocumentSegmentListApi(Resource): query = query.where(DocumentSegment.hit_count >= hit_count_gte) if keyword: + # Escape special characters in keyword to prevent SQL injection via LIKE wildcards + escaped_keyword = escape_like_pattern(keyword) # Search in both content and keywords fields # Use database-specific methods for JSON array search if dify_config.SQLALCHEMY_DATABASE_URI_SCHEME == "postgresql": @@ -156,15 +159,15 @@ class DatasetDocumentSegmentListApi(Resource): .scalar_subquery() ), ",", - ).ilike(f"%{keyword}%") + ).ilike(f"%{escaped_keyword}%", escape="\\") else: # MySQL: Cast JSON to string for pattern matching # MySQL stores Chinese text directly in JSON without Unicode escaping - keywords_condition = cast(DocumentSegment.keywords, String).ilike(f"%{keyword}%") + keywords_condition = cast(DocumentSegment.keywords, String).ilike(f"%{escaped_keyword}%", escape="\\") query = query.where( or_( - DocumentSegment.content.ilike(f"%{keyword}%"), + DocumentSegment.content.ilike(f"%{escaped_keyword}%", escape="\\"), keywords_condition, ) ) diff --git a/api/controllers/console/datasets/hit_testing_base.py b/api/controllers/console/datasets/hit_testing_base.py index db7c50f422..db1a874437 100644 --- a/api/controllers/console/datasets/hit_testing_base.py +++ b/api/controllers/console/datasets/hit_testing_base.py @@ -1,7 +1,7 @@ import logging from typing import Any -from flask_restx import marshal, reqparse +from flask_restx import marshal from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden, InternalServerError, NotFound @@ -56,15 +56,10 @@ class DatasetsHitTestingBase: HitTestingService.hit_testing_args_check(args) @staticmethod - def parse_args(): - parser = ( - reqparse.RequestParser() - .add_argument("query", type=str, required=False, location="json") - .add_argument("attachment_ids", type=list, required=False, location="json") - .add_argument("retrieval_model", type=dict, required=False, location="json") - .add_argument("external_retrieval_model", type=dict, required=False, location="json") - ) - return parser.parse_args() + def parse_args(payload: dict[str, Any]) -> dict[str, Any]: + """Validate and return hit-testing arguments from an incoming payload.""" + hit_testing_payload = HitTestingPayload.model_validate(payload or {}) + return hit_testing_payload.model_dump(exclude_none=True) @staticmethod def perform_hit_testing(dataset, args): diff --git a/api/controllers/console/workspace/trigger_providers.py b/api/controllers/console/workspace/trigger_providers.py index 497e62b790..c13bfd986e 100644 --- a/api/controllers/console/workspace/trigger_providers.py +++ b/api/controllers/console/workspace/trigger_providers.py @@ -4,12 +4,11 @@ from typing import Any from flask import make_response, redirect, request from flask_restx import Resource, reqparse -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from sqlalchemy.orm import Session from werkzeug.exceptions import BadRequest, Forbidden from configs import dify_config -from constants import HIDDEN_VALUE, UNKNOWN_VALUE from controllers.web.error import NotFoundError from core.model_runtime.utils.encoders import jsonable_encoder from core.plugin.entities.plugin_daemon import CredentialType @@ -44,6 +43,12 @@ class TriggerSubscriptionUpdateRequest(BaseModel): parameters: Mapping[str, Any] | None = Field(default=None, description="The parameters for the subscription") properties: Mapping[str, Any] | None = Field(default=None, description="The properties for the subscription") + @model_validator(mode="after") + def check_at_least_one_field(self): + if all(v is None for v in (self.name, self.credentials, self.parameters, self.properties)): + raise ValueError("At least one of name, credentials, parameters, or properties must be provided") + return self + class TriggerSubscriptionVerifyRequest(BaseModel): """Request payload for verifying subscription credentials.""" @@ -333,7 +338,7 @@ class TriggerSubscriptionUpdateApi(Resource): user = current_user assert user.current_tenant_id is not None - args = TriggerSubscriptionUpdateRequest.model_validate(console_ns.payload) + request = TriggerSubscriptionUpdateRequest.model_validate(console_ns.payload) subscription = TriggerProviderService.get_subscription_by_id( tenant_id=user.current_tenant_id, @@ -345,50 +350,32 @@ class TriggerSubscriptionUpdateApi(Resource): provider_id = TriggerProviderID(subscription.provider_id) try: - # rename only - if ( - args.name is not None - and args.credentials is None - and args.parameters is None - and args.properties is None - ): + # For rename only, just update the name + rename = request.name is not None and not any((request.credentials, request.parameters, request.properties)) + # When credential type is UNAUTHORIZED, it indicates the subscription was manually created + # For Manually created subscription, they dont have credentials, parameters + # They only have name and properties(which is input by user) + manually_created = subscription.credential_type == CredentialType.UNAUTHORIZED + if rename or manually_created: TriggerProviderService.update_trigger_subscription( tenant_id=user.current_tenant_id, subscription_id=subscription_id, - name=args.name, + name=request.name, + properties=request.properties, ) return 200 - # rebuild for create automatically by the provider - match subscription.credential_type: - case CredentialType.UNAUTHORIZED: - TriggerProviderService.update_trigger_subscription( - tenant_id=user.current_tenant_id, - subscription_id=subscription_id, - name=args.name, - properties=args.properties, - ) - return 200 - case CredentialType.API_KEY | CredentialType.OAUTH2: - if args.credentials: - new_credentials: dict[str, Any] = { - key: value if value != HIDDEN_VALUE else subscription.credentials.get(key, UNKNOWN_VALUE) - for key, value in args.credentials.items() - } - else: - new_credentials = subscription.credentials - - TriggerProviderService.rebuild_trigger_subscription( - tenant_id=user.current_tenant_id, - name=args.name, - provider_id=provider_id, - subscription_id=subscription_id, - credentials=new_credentials, - parameters=args.parameters or subscription.parameters, - ) - return 200 - case _: - raise BadRequest("Invalid credential type") + # For the rest cases(API_KEY, OAUTH2) + # we need to call third party provider(e.g. GitHub) to rebuild the subscription + TriggerProviderService.rebuild_trigger_subscription( + tenant_id=user.current_tenant_id, + name=request.name, + provider_id=provider_id, + subscription_id=subscription_id, + credentials=request.credentials or subscription.credentials, + parameters=request.parameters or subscription.parameters, + ) + return 200 except ValueError as e: raise BadRequest(str(e)) except Exception as e: diff --git a/api/controllers/service_api/dataset/hit_testing.py b/api/controllers/service_api/dataset/hit_testing.py index d81287d56f..8dbb690901 100644 --- a/api/controllers/service_api/dataset/hit_testing.py +++ b/api/controllers/service_api/dataset/hit_testing.py @@ -24,7 +24,7 @@ class HitTestingApi(DatasetApiResource, DatasetsHitTestingBase): dataset_id_str = str(dataset_id) dataset = self.get_and_validate_dataset(dataset_id_str) - args = self.parse_args() + args = self.parse_args(service_api_ns.payload) self.hit_testing_args_check(args) return self.perform_hit_testing(dataset, args) diff --git a/api/controllers/web/conversation.py b/api/controllers/web/conversation.py index 527eef6094..e76649495a 100644 --- a/api/controllers/web/conversation.py +++ b/api/controllers/web/conversation.py @@ -1,9 +1,11 @@ -from flask_restx import reqparse -from flask_restx.inputs import int_range -from pydantic import TypeAdapter +from typing import Literal + +from flask import request +from pydantic import BaseModel, Field, TypeAdapter, field_validator, model_validator from sqlalchemy.orm import Session from werkzeug.exceptions import NotFound +from controllers.common.schema import register_schema_models from controllers.web import web_ns from controllers.web.error import NotChatAppError from controllers.web.wraps import WebApiResource @@ -21,6 +23,35 @@ from services.errors.conversation import ConversationNotExistsError, LastConvers from services.web_conversation_service import WebConversationService +class ConversationListQuery(BaseModel): + last_id: str | None = None + limit: int = Field(default=20, ge=1, le=100) + pinned: bool | None = None + sort_by: Literal["created_at", "-created_at", "updated_at", "-updated_at"] = "-updated_at" + + @field_validator("last_id") + @classmethod + def validate_last_id(cls, value: str | None) -> str | None: + if value is None: + return value + return uuid_value(value) + + +class ConversationRenamePayload(BaseModel): + name: str | None = None + auto_generate: bool = False + + @model_validator(mode="after") + def validate_name_requirement(self): + if not self.auto_generate: + if self.name is None or not self.name.strip(): + raise ValueError("name is required when auto_generate is false") + return self + + +register_schema_models(web_ns, ConversationListQuery, ConversationRenamePayload) + + @web_ns.route("/conversations") class ConversationListApi(WebApiResource): @web_ns.doc("Get Conversation List") @@ -64,25 +95,8 @@ class ConversationListApi(WebApiResource): if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}: raise NotChatAppError() - parser = ( - reqparse.RequestParser() - .add_argument("last_id", type=uuid_value, location="args") - .add_argument("limit", type=int_range(1, 100), required=False, default=20, location="args") - .add_argument("pinned", type=str, choices=["true", "false", None], location="args") - .add_argument( - "sort_by", - type=str, - choices=["created_at", "-created_at", "updated_at", "-updated_at"], - required=False, - default="-updated_at", - location="args", - ) - ) - args = parser.parse_args() - - pinned = None - if "pinned" in args and args["pinned"] is not None: - pinned = args["pinned"] == "true" + raw_args = request.args.to_dict() + query = ConversationListQuery.model_validate(raw_args) try: with Session(db.engine) as session: @@ -90,11 +104,11 @@ class ConversationListApi(WebApiResource): session=session, app_model=app_model, user=end_user, - last_id=args["last_id"], - limit=args["limit"], + last_id=query.last_id, + limit=query.limit, invoke_from=InvokeFrom.WEB_APP, - pinned=pinned, - sort_by=args["sort_by"], + pinned=query.pinned, + sort_by=query.sort_by, ) adapter = TypeAdapter(SimpleConversation) conversations = [adapter.validate_python(item, from_attributes=True) for item in pagination.data] @@ -168,16 +182,11 @@ class ConversationRenameApi(WebApiResource): conversation_id = str(c_id) - parser = ( - reqparse.RequestParser() - .add_argument("name", type=str, required=False, location="json") - .add_argument("auto_generate", type=bool, required=False, default=False, location="json") - ) - args = parser.parse_args() + payload = ConversationRenamePayload.model_validate(web_ns.payload or {}) try: conversation = ConversationService.rename( - app_model, conversation_id, end_user, args["name"], args["auto_generate"] + app_model, conversation_id, end_user, payload.name, payload.auto_generate ) return ( TypeAdapter(SimpleConversation) diff --git a/api/controllers/web/saved_message.py b/api/controllers/web/saved_message.py index 4e20690e9e..29993100f6 100644 --- a/api/controllers/web/saved_message.py +++ b/api/controllers/web/saved_message.py @@ -1,18 +1,30 @@ -from flask_restx import reqparse -from flask_restx.inputs import int_range -from pydantic import TypeAdapter +from flask import request +from pydantic import BaseModel, Field, TypeAdapter from werkzeug.exceptions import NotFound +from controllers.common.schema import register_schema_models from controllers.web import web_ns from controllers.web.error import NotCompletionAppError from controllers.web.wraps import WebApiResource from fields.conversation_fields import ResultResponse from fields.message_fields import SavedMessageInfiniteScrollPagination, SavedMessageItem -from libs.helper import uuid_value +from libs.helper import UUIDStrOrEmpty from services.errors.message import MessageNotExistsError from services.saved_message_service import SavedMessageService +class SavedMessageListQuery(BaseModel): + last_id: UUIDStrOrEmpty | None = None + limit: int = Field(default=20, ge=1, le=100) + + +class SavedMessageCreatePayload(BaseModel): + message_id: UUIDStrOrEmpty + + +register_schema_models(web_ns, SavedMessageListQuery, SavedMessageCreatePayload) + + @web_ns.route("/saved-messages") class SavedMessageListApi(WebApiResource): @web_ns.doc("Get Saved Messages") @@ -42,14 +54,10 @@ class SavedMessageListApi(WebApiResource): if app_model.mode != "completion": raise NotCompletionAppError() - parser = ( - reqparse.RequestParser() - .add_argument("last_id", type=uuid_value, location="args") - .add_argument("limit", type=int_range(1, 100), required=False, default=20, location="args") - ) - args = parser.parse_args() + raw_args = request.args.to_dict() + query = SavedMessageListQuery.model_validate(raw_args) - pagination = SavedMessageService.pagination_by_last_id(app_model, end_user, args["last_id"], args["limit"]) + pagination = SavedMessageService.pagination_by_last_id(app_model, end_user, query.last_id, query.limit) adapter = TypeAdapter(SavedMessageItem) items = [adapter.validate_python(message, from_attributes=True) for message in pagination.data] return SavedMessageInfiniteScrollPagination( @@ -79,11 +87,10 @@ class SavedMessageListApi(WebApiResource): if app_model.mode != "completion": raise NotCompletionAppError() - parser = reqparse.RequestParser().add_argument("message_id", type=uuid_value, required=True, location="json") - args = parser.parse_args() + payload = SavedMessageCreatePayload.model_validate(web_ns.payload or {}) try: - SavedMessageService.save(app_model, end_user, args["message_id"]) + SavedMessageService.save(app_model, end_user, payload.message_id) except MessageNotExistsError: raise NotFound("Message Not Exists.") diff --git a/api/core/app/apps/advanced_chat/app_runner.py b/api/core/app/apps/advanced_chat/app_runner.py index ee092e55c5..a2ae8dec5b 100644 --- a/api/core/app/apps/advanced_chat/app_runner.py +++ b/api/core/app/apps/advanced_chat/app_runner.py @@ -20,6 +20,8 @@ from core.app.entities.queue_entities import ( QueueTextChunkEvent, ) from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature +from core.app.layers.conversation_variable_persist_layer import ConversationVariablePersistenceLayer +from core.db.session_factory import session_factory from core.moderation.base import ModerationError from core.moderation.input_moderation import InputModeration from core.variables.variables import VariableUnion @@ -40,6 +42,7 @@ from models import Workflow from models.enums import UserFrom from models.model import App, Conversation, Message, MessageAnnotation from models.workflow import ConversationVariable +from services.conversation_variable_updater import ConversationVariableUpdater logger = logging.getLogger(__name__) @@ -200,6 +203,10 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner): ) workflow_entry.graph_engine.layer(persistence_layer) + conversation_variable_layer = ConversationVariablePersistenceLayer( + ConversationVariableUpdater(session_factory.get_session_maker()) + ) + workflow_entry.graph_engine.layer(conversation_variable_layer) for layer in self._graph_engine_layers: workflow_entry.graph_engine.layer(layer) diff --git a/api/core/app/features/annotation_reply/annotation_reply.py b/api/core/app/features/annotation_reply/annotation_reply.py index 79fbafe39e..3f9f3da9b2 100644 --- a/api/core/app/features/annotation_reply/annotation_reply.py +++ b/api/core/app/features/annotation_reply/annotation_reply.py @@ -75,7 +75,7 @@ class AnnotationReplyFeature: AppAnnotationService.add_annotation_history( annotation.id, app_record.id, - annotation.question, + annotation.question_text, annotation.content, query, user_id, diff --git a/api/core/app/layers/conversation_variable_persist_layer.py b/api/core/app/layers/conversation_variable_persist_layer.py new file mode 100644 index 0000000000..77cc00bdc9 --- /dev/null +++ b/api/core/app/layers/conversation_variable_persist_layer.py @@ -0,0 +1,60 @@ +import logging + +from core.variables import Variable +from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID +from core.workflow.conversation_variable_updater import ConversationVariableUpdater +from core.workflow.enums import NodeType +from core.workflow.graph_engine.layers.base import GraphEngineLayer +from core.workflow.graph_events import GraphEngineEvent, NodeRunSucceededEvent +from core.workflow.nodes.variable_assigner.common import helpers as common_helpers + +logger = logging.getLogger(__name__) + + +class ConversationVariablePersistenceLayer(GraphEngineLayer): + def __init__(self, conversation_variable_updater: ConversationVariableUpdater) -> None: + super().__init__() + self._conversation_variable_updater = conversation_variable_updater + + def on_graph_start(self) -> None: + pass + + def on_event(self, event: GraphEngineEvent) -> None: + if not isinstance(event, NodeRunSucceededEvent): + return + if event.node_type != NodeType.VARIABLE_ASSIGNER: + return + if self.graph_runtime_state is None: + return + + updated_variables = common_helpers.get_updated_variables(event.node_run_result.process_data) or [] + if not updated_variables: + return + + conversation_id = self.graph_runtime_state.system_variable.conversation_id + if conversation_id is None: + return + + updated_any = False + for item in updated_variables: + selector = item.selector + if len(selector) < 2: + logger.warning("Conversation variable selector invalid. selector=%s", selector) + continue + if selector[0] != CONVERSATION_VARIABLE_NODE_ID: + continue + variable = self.graph_runtime_state.variable_pool.get(selector) + if not isinstance(variable, Variable): + logger.warning( + "Conversation variable not found in variable pool. selector=%s", + selector, + ) + continue + self._conversation_variable_updater.update(conversation_id=conversation_id, variable=variable) + updated_any = True + + if updated_any: + self._conversation_variable_updater.flush() + + def on_graph_end(self, error: Exception | None) -> None: + pass diff --git a/api/core/app/layers/pause_state_persist_layer.py b/api/core/app/layers/pause_state_persist_layer.py index 61a3e1baca..bf76ae8178 100644 --- a/api/core/app/layers/pause_state_persist_layer.py +++ b/api/core/app/layers/pause_state_persist_layer.py @@ -66,6 +66,7 @@ class PauseStatePersistenceLayer(GraphEngineLayer): """ if isinstance(session_factory, Engine): session_factory = sessionmaker(session_factory) + super().__init__() self._session_maker = session_factory self._state_owner_user_id = state_owner_user_id self._generate_entity = generate_entity @@ -98,8 +99,6 @@ class PauseStatePersistenceLayer(GraphEngineLayer): if not isinstance(event, GraphRunPausedEvent): return - assert self.graph_runtime_state is not None - entity_wrapper: _GenerateEntityUnion if isinstance(self._generate_entity, WorkflowAppGenerateEntity): entity_wrapper = _WorkflowGenerateEntityWrapper(entity=self._generate_entity) diff --git a/api/core/app/layers/trigger_post_layer.py b/api/core/app/layers/trigger_post_layer.py index fe1a46a945..225b758fcb 100644 --- a/api/core/app/layers/trigger_post_layer.py +++ b/api/core/app/layers/trigger_post_layer.py @@ -33,6 +33,7 @@ class TriggerPostLayer(GraphEngineLayer): trigger_log_id: str, session_maker: sessionmaker[Session], ): + super().__init__() self.trigger_log_id = trigger_log_id self.start_time = start_time self.cfs_plan_scheduler_entity = cfs_plan_scheduler_entity @@ -57,10 +58,6 @@ class TriggerPostLayer(GraphEngineLayer): elapsed_time = (datetime.now(UTC) - self.start_time).total_seconds() # Extract relevant data from result - if not self.graph_runtime_state: - logger.exception("Graph runtime state is not set") - return - outputs = self.graph_runtime_state.outputs # BASICLY, workflow_execution_id is the same as workflow_run_id diff --git a/api/core/db/session_factory.py b/api/core/db/session_factory.py index 1dae2eafd4..45d4bc4594 100644 --- a/api/core/db/session_factory.py +++ b/api/core/db/session_factory.py @@ -1,7 +1,7 @@ from sqlalchemy import Engine from sqlalchemy.orm import Session, sessionmaker -_session_maker: sessionmaker | None = None +_session_maker: sessionmaker[Session] | None = None def configure_session_factory(engine: Engine, expire_on_commit: bool = False): @@ -10,7 +10,7 @@ def configure_session_factory(engine: Engine, expire_on_commit: bool = False): _session_maker = sessionmaker(bind=engine, expire_on_commit=expire_on_commit) -def get_session_maker() -> sessionmaker: +def get_session_maker() -> sessionmaker[Session]: if _session_maker is None: raise RuntimeError("Session factory not configured. Call configure_session_factory() first.") return _session_maker @@ -27,7 +27,7 @@ class SessionFactory: configure_session_factory(engine, expire_on_commit) @staticmethod - def get_session_maker() -> sessionmaker: + def get_session_maker() -> sessionmaker[Session]: return get_session_maker() @staticmethod diff --git a/api/core/file/helpers.py b/api/core/file/helpers.py index 6d553d7dc6..2ac483673a 100644 --- a/api/core/file/helpers.py +++ b/api/core/file/helpers.py @@ -8,8 +8,9 @@ import urllib.parse from configs import dify_config -def get_signed_file_url(upload_file_id: str, as_attachment=False) -> str: - url = f"{dify_config.FILES_URL}/files/{upload_file_id}/file-preview" +def get_signed_file_url(upload_file_id: str, as_attachment=False, for_external: bool = True) -> str: + base_url = dify_config.FILES_URL if for_external else (dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL) + url = f"{base_url}/files/{upload_file_id}/file-preview" timestamp = str(int(time.time())) nonce = os.urandom(16).hex() diff --git a/api/core/file/models.py b/api/core/file/models.py index d149205d77..6324523b22 100644 --- a/api/core/file/models.py +++ b/api/core/file/models.py @@ -112,17 +112,17 @@ class File(BaseModel): return text - def generate_url(self) -> str | None: + def generate_url(self, for_external: bool = True) -> str | None: if self.transfer_method == FileTransferMethod.REMOTE_URL: return self.remote_url elif self.transfer_method == FileTransferMethod.LOCAL_FILE: if self.related_id is None: raise ValueError("Missing file related_id") - return helpers.get_signed_file_url(upload_file_id=self.related_id) + return helpers.get_signed_file_url(upload_file_id=self.related_id, for_external=for_external) elif self.transfer_method in [FileTransferMethod.TOOL_FILE, FileTransferMethod.DATASOURCE_FILE]: assert self.related_id is not None assert self.extension is not None - return sign_tool_file(tool_file_id=self.related_id, extension=self.extension) + return sign_tool_file(tool_file_id=self.related_id, extension=self.extension, for_external=for_external) return None def to_plugin_parameter(self) -> dict[str, Any]: @@ -133,7 +133,7 @@ class File(BaseModel): "extension": self.extension, "size": self.size, "type": self.type, - "url": self.generate_url(), + "url": self.generate_url(for_external=False), } @model_validator(mode="after") diff --git a/api/core/helper/code_executor/template_transformer.py b/api/core/helper/code_executor/template_transformer.py index 6fda073913..5cdea19a8d 100644 --- a/api/core/helper/code_executor/template_transformer.py +++ b/api/core/helper/code_executor/template_transformer.py @@ -76,7 +76,7 @@ class TemplateTransformer(ABC): Post-process the result to convert scientific notation strings back to numbers """ - def convert_scientific_notation(value): + def convert_scientific_notation(value: Any) -> Any: if isinstance(value, str): # Check if the string looks like scientific notation if re.match(r"^-?\d+\.?\d*e[+-]\d+$", value, re.IGNORECASE): @@ -90,7 +90,7 @@ class TemplateTransformer(ABC): return [convert_scientific_notation(v) for v in value] return value - return convert_scientific_notation(result) # type: ignore[no-any-return] + return convert_scientific_notation(result) @classmethod @abstractmethod diff --git a/api/core/rag/datasource/vdb/clickzetta/clickzetta_vector.py b/api/core/rag/datasource/vdb/clickzetta/clickzetta_vector.py index a306f9ba0c..e05b70ba22 100644 --- a/api/core/rag/datasource/vdb/clickzetta/clickzetta_vector.py +++ b/api/core/rag/datasource/vdb/clickzetta/clickzetta_vector.py @@ -984,9 +984,11 @@ class ClickzettaVector(BaseVector): # No need for dataset_id filter since each dataset has its own table - # Use simple quote escaping for LIKE clause - escaped_query = query.replace("'", "''") - filter_clauses.append(f"{Field.CONTENT_KEY} LIKE '%{escaped_query}%'") + # Escape special characters for LIKE clause to prevent SQL injection + from libs.helper import escape_like_pattern + + escaped_query = escape_like_pattern(query).replace("'", "''") + filter_clauses.append(f"{Field.CONTENT_KEY} LIKE '%{escaped_query}%' ESCAPE '\\\\'") where_clause = " AND ".join(filter_clauses) search_sql = f""" diff --git a/api/core/rag/datasource/vdb/iris/iris_vector.py b/api/core/rag/datasource/vdb/iris/iris_vector.py index b1bfabb76e..5bdb0af0b3 100644 --- a/api/core/rag/datasource/vdb/iris/iris_vector.py +++ b/api/core/rag/datasource/vdb/iris/iris_vector.py @@ -287,11 +287,15 @@ class IrisVector(BaseVector): cursor.execute(sql, (query,)) else: # Fallback to LIKE search (inefficient for large datasets) - query_pattern = f"%{query}%" + # Escape special characters for LIKE clause to prevent SQL injection + from libs.helper import escape_like_pattern + + escaped_query = escape_like_pattern(query) + query_pattern = f"%{escaped_query}%" sql = f""" SELECT TOP {top_k} id, text, meta FROM {self.schema}.{self.table_name} - WHERE text LIKE ? + WHERE text LIKE ? ESCAPE '\\' """ cursor.execute(sql, (query_pattern,)) diff --git a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py index 84d1e26b34..b48dd93f04 100644 --- a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py +++ b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py @@ -66,6 +66,8 @@ class WeaviateVector(BaseVector): in a Weaviate collection. """ + _DOCUMENT_ID_PROPERTY = "document_id" + def __init__(self, collection_name: str, config: WeaviateConfig, attributes: list): """ Initializes the Weaviate vector store. @@ -353,15 +355,12 @@ class WeaviateVector(BaseVector): return [] col = self._client.collections.use(self._collection_name) - props = list({*self._attributes, "document_id", Field.TEXT_KEY.value}) + props = list({*self._attributes, self._DOCUMENT_ID_PROPERTY, Field.TEXT_KEY.value}) where = None doc_ids = kwargs.get("document_ids_filter") or [] if doc_ids: - ors = [Filter.by_property("document_id").equal(x) for x in doc_ids] - where = ors[0] - for f in ors[1:]: - where = where | f + where = Filter.by_property(self._DOCUMENT_ID_PROPERTY).contains_any(doc_ids) top_k = int(kwargs.get("top_k", 4)) score_threshold = float(kwargs.get("score_threshold") or 0.0) @@ -408,10 +407,7 @@ class WeaviateVector(BaseVector): where = None doc_ids = kwargs.get("document_ids_filter") or [] if doc_ids: - ors = [Filter.by_property("document_id").equal(x) for x in doc_ids] - where = ors[0] - for f in ors[1:]: - where = where | f + where = Filter.by_property(self._DOCUMENT_ID_PROPERTY).contains_any(doc_ids) top_k = int(kwargs.get("top_k", 4)) diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index f67f613e9d..511f5a698d 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -7,10 +7,11 @@ import re import tempfile import uuid from urllib.parse import urlparse -from xml.etree import ElementTree import httpx from docx import Document as DocxDocument +from docx.oxml.ns import qn +from docx.text.run import Run from configs import dify_config from core.helper import ssrf_proxy @@ -229,44 +230,20 @@ class WordExtractor(BaseExtractor): image_map = self._extract_images_from_docx(doc) - hyperlinks_url = None - url_pattern = re.compile(r"http://[^\s+]+//|https://[^\s+]+") - for para in doc.paragraphs: - for run in para.runs: - if run.text and hyperlinks_url: - result = f" [{run.text}]({hyperlinks_url}) " - run.text = result - hyperlinks_url = None - if "HYPERLINK" in run.element.xml: - try: - xml = ElementTree.XML(run.element.xml) - x_child = [c for c in xml.iter() if c is not None] - for x in x_child: - if x is None: - continue - if x.tag.endswith("instrText"): - if x.text is None: - continue - for i in url_pattern.findall(x.text): - hyperlinks_url = str(i) - except Exception: - logger.exception("Failed to parse HYPERLINK xml") - def parse_paragraph(paragraph): - paragraph_content = [] - - def append_image_link(image_id, has_drawing): + def append_image_link(image_id, has_drawing, target_buffer): """Helper to append image link from image_map based on relationship type.""" rel = doc.part.rels[image_id] if rel.is_external: if image_id in image_map and not has_drawing: - paragraph_content.append(image_map[image_id]) + target_buffer.append(image_map[image_id]) else: image_part = rel.target_part if image_part in image_map and not has_drawing: - paragraph_content.append(image_map[image_part]) + target_buffer.append(image_map[image_part]) - for run in paragraph.runs: + def process_run(run, target_buffer): + # Helper to extract text and embedded images from a run element and append them to target_buffer if hasattr(run.element, "tag") and isinstance(run.element.tag, str) and run.element.tag.endswith("r"): # Process drawing type images drawing_elements = run.element.findall( @@ -287,13 +264,13 @@ class WordExtractor(BaseExtractor): # External image: use embed_id as key if embed_id in image_map: has_drawing = True - paragraph_content.append(image_map[embed_id]) + target_buffer.append(image_map[embed_id]) else: # Internal image: use target_part as key image_part = doc.part.related_parts.get(embed_id) if image_part in image_map: has_drawing = True - paragraph_content.append(image_map[image_part]) + target_buffer.append(image_map[image_part]) # Process pict type images shape_elements = run.element.findall( ".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}pict" @@ -308,7 +285,7 @@ class WordExtractor(BaseExtractor): "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id" ) if image_id and image_id in doc.part.rels: - append_image_link(image_id, has_drawing) + append_image_link(image_id, has_drawing, target_buffer) # Find imagedata element in VML image_data = shape.find(".//{urn:schemas-microsoft-com:vml}imagedata") if image_data is not None: @@ -316,9 +293,93 @@ class WordExtractor(BaseExtractor): "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id" ) if image_id and image_id in doc.part.rels: - append_image_link(image_id, has_drawing) + append_image_link(image_id, has_drawing, target_buffer) if run.text.strip(): - paragraph_content.append(run.text.strip()) + target_buffer.append(run.text.strip()) + + def process_hyperlink(hyperlink_elem, target_buffer): + # Helper to extract text from a hyperlink element and append it to target_buffer + r_id = hyperlink_elem.get(qn("r:id")) + + # Extract text from runs inside the hyperlink + link_text_parts = [] + for run_elem in hyperlink_elem.findall(qn("w:r")): + run = Run(run_elem, paragraph) + # Hyperlink text may be split across multiple runs (e.g., with different formatting), + # so collect all run texts first + if run.text: + link_text_parts.append(run.text) + + link_text = "".join(link_text_parts).strip() + + # Resolve URL + if r_id: + try: + rel = doc.part.rels.get(r_id) + if rel and rel.is_external: + link_text = f"[{link_text or rel.target_ref}]({rel.target_ref})" + except Exception: + logger.exception("Failed to resolve URL for hyperlink with r:id: %s", r_id) + + if link_text: + target_buffer.append(link_text) + + paragraph_content = [] + # State for legacy HYPERLINK fields + hyperlink_field_url = None + hyperlink_field_text_parts: list = [] + is_collecting_field_text = False + # Iterate through paragraph elements in document order + for child in paragraph._element: + tag = child.tag + if tag == qn("w:r"): + # Regular run + run = Run(child, paragraph) + + # Check for fldChar (begin/end/separate) and instrText for legacy hyperlinks + fld_chars = child.findall(qn("w:fldChar")) + instr_texts = child.findall(qn("w:instrText")) + + # Handle Fields + if fld_chars or instr_texts: + # Process instrText to find HYPERLINK "url" + for instr in instr_texts: + if instr.text and "HYPERLINK" in instr.text: + # Quick regex to extract URL + match = re.search(r'HYPERLINK\s+"([^"]+)"', instr.text, re.IGNORECASE) + if match: + hyperlink_field_url = match.group(1) + + # Process fldChar + for fld_char in fld_chars: + fld_char_type = fld_char.get(qn("w:fldCharType")) + if fld_char_type == "begin": + # Start of a field: reset legacy link state + hyperlink_field_url = None + hyperlink_field_text_parts = [] + is_collecting_field_text = False + elif fld_char_type == "separate": + # Separator: if we found a URL, start collecting visible text + if hyperlink_field_url: + is_collecting_field_text = True + elif fld_char_type == "end": + # End of field + if is_collecting_field_text and hyperlink_field_url: + # Create markdown link and append to main content + display_text = "".join(hyperlink_field_text_parts).strip() + if display_text: + link_md = f"[{display_text}]({hyperlink_field_url})" + paragraph_content.append(link_md) + # Reset state + hyperlink_field_url = None + hyperlink_field_text_parts = [] + is_collecting_field_text = False + + # Decide where to append content + target_buffer = hyperlink_field_text_parts if is_collecting_field_text else paragraph_content + process_run(run, target_buffer) + elif tag == qn("w:hyperlink"): + process_hyperlink(child, paragraph_content) return "".join(paragraph_content) if paragraph_content else "" paragraphs = doc.paragraphs.copy() diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index a5fa77365f..f8f85d141a 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -1198,18 +1198,24 @@ class DatasetRetrieval: json_field = DatasetDocument.doc_metadata[metadata_name].as_string() + from libs.helper import escape_like_pattern + match condition: case "contains": - filters.append(json_field.like(f"%{value}%")) + escaped_value = escape_like_pattern(str(value)) + filters.append(json_field.like(f"%{escaped_value}%", escape="\\")) case "not contains": - filters.append(json_field.notlike(f"%{value}%")) + escaped_value = escape_like_pattern(str(value)) + filters.append(json_field.notlike(f"%{escaped_value}%", escape="\\")) case "start with": - filters.append(json_field.like(f"{value}%")) + escaped_value = escape_like_pattern(str(value)) + filters.append(json_field.like(f"{escaped_value}%", escape="\\")) case "end with": - filters.append(json_field.like(f"%{value}")) + escaped_value = escape_like_pattern(str(value)) + filters.append(json_field.like(f"%{escaped_value}", escape="\\")) case "is" | "=": if isinstance(value, str): @@ -1474,38 +1480,38 @@ class DatasetRetrieval: if cancel_event and cancel_event.is_set(): break - # Skip second reranking when there is only one dataset - if reranking_enable and dataset_count > 1: - # do rerank for searched documents - data_post_processor = DataPostProcessor(tenant_id, reranking_mode, reranking_model, weights, False) - if query: - all_documents_item = data_post_processor.invoke( - query=query, - documents=all_documents_item, - score_threshold=score_threshold, - top_n=top_k, - query_type=QueryType.TEXT_QUERY, - ) - if attachment_id: - all_documents_item = data_post_processor.invoke( - documents=all_documents_item, - score_threshold=score_threshold, - top_n=top_k, - query_type=QueryType.IMAGE_QUERY, - query=attachment_id, - ) - else: - if index_type == IndexTechniqueType.ECONOMY: - if not query: - all_documents_item = [] - else: - all_documents_item = self.calculate_keyword_score(query, all_documents_item, top_k) - elif index_type == IndexTechniqueType.HIGH_QUALITY: - all_documents_item = self.calculate_vector_score(all_documents_item, top_k, score_threshold) + # Skip second reranking when there is only one dataset + if reranking_enable and dataset_count > 1: + # do rerank for searched documents + data_post_processor = DataPostProcessor(tenant_id, reranking_mode, reranking_model, weights, False) + if query: + all_documents_item = data_post_processor.invoke( + query=query, + documents=all_documents_item, + score_threshold=score_threshold, + top_n=top_k, + query_type=QueryType.TEXT_QUERY, + ) + if attachment_id: + all_documents_item = data_post_processor.invoke( + documents=all_documents_item, + score_threshold=score_threshold, + top_n=top_k, + query_type=QueryType.IMAGE_QUERY, + query=attachment_id, + ) else: - all_documents_item = all_documents_item[:top_k] if top_k else all_documents_item - if all_documents_item: - all_documents.extend(all_documents_item) + if index_type == IndexTechniqueType.ECONOMY: + if not query: + all_documents_item = [] + else: + all_documents_item = self.calculate_keyword_score(query, all_documents_item, top_k) + elif index_type == IndexTechniqueType.HIGH_QUALITY: + all_documents_item = self.calculate_vector_score(all_documents_item, top_k, score_threshold) + else: + all_documents_item = all_documents_item[:top_k] if top_k else all_documents_item + if all_documents_item: + all_documents.extend(all_documents_item) except Exception as e: if cancel_event: cancel_event.set() diff --git a/api/core/tools/signature.py b/api/core/tools/signature.py index fef3157f27..22e099deba 100644 --- a/api/core/tools/signature.py +++ b/api/core/tools/signature.py @@ -7,12 +7,12 @@ import time from configs import dify_config -def sign_tool_file(tool_file_id: str, extension: str) -> str: +def sign_tool_file(tool_file_id: str, extension: str, for_external: bool = True) -> str: """ sign file to get a temporary url for plugin access """ - # Use internal URL for plugin/tool file access in Docker environments - base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL + # Use internal URL for plugin/tool file access in Docker environments, unless for_external is True + base_url = dify_config.FILES_URL if for_external else (dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL) file_preview_url = f"{base_url}/files/tools/{tool_file_id}{extension}" timestamp = str(int(time.time())) diff --git a/api/core/workflow/README.md b/api/core/workflow/README.md index 72f5dbe1e2..9a39f976a6 100644 --- a/api/core/workflow/README.md +++ b/api/core/workflow/README.md @@ -64,6 +64,9 @@ engine.layer(DebugLoggingLayer(level="INFO")) engine.layer(ExecutionLimitsLayer(max_nodes=100)) ``` +`engine.layer()` binds the read-only runtime state before execution, so layer hooks +can assume `graph_runtime_state` is available. + ### Event-Driven Architecture All node executions emit events for monitoring and integration: diff --git a/api/core/workflow/graph_engine/command_channels/redis_channel.py b/api/core/workflow/graph_engine/command_channels/redis_channel.py index 4be3adb8f8..0fccd4a0fd 100644 --- a/api/core/workflow/graph_engine/command_channels/redis_channel.py +++ b/api/core/workflow/graph_engine/command_channels/redis_channel.py @@ -9,7 +9,7 @@ Each instance uses a unique key for its command queue. import json from typing import TYPE_CHECKING, Any, final -from ..entities.commands import AbortCommand, CommandType, GraphEngineCommand, PauseCommand +from ..entities.commands import AbortCommand, CommandType, GraphEngineCommand, PauseCommand, UpdateVariablesCommand if TYPE_CHECKING: from extensions.ext_redis import RedisClientWrapper @@ -113,6 +113,8 @@ class RedisChannel: return AbortCommand.model_validate(data) if command_type == CommandType.PAUSE: return PauseCommand.model_validate(data) + if command_type == CommandType.UPDATE_VARIABLES: + return UpdateVariablesCommand.model_validate(data) # For other command types, use base class return GraphEngineCommand.model_validate(data) diff --git a/api/core/workflow/graph_engine/command_processing/__init__.py b/api/core/workflow/graph_engine/command_processing/__init__.py index 837f5e55fd..7b4f0dfff7 100644 --- a/api/core/workflow/graph_engine/command_processing/__init__.py +++ b/api/core/workflow/graph_engine/command_processing/__init__.py @@ -5,11 +5,12 @@ This package handles external commands sent to the engine during execution. """ -from .command_handlers import AbortCommandHandler, PauseCommandHandler +from .command_handlers import AbortCommandHandler, PauseCommandHandler, UpdateVariablesCommandHandler from .command_processor import CommandProcessor __all__ = [ "AbortCommandHandler", "CommandProcessor", "PauseCommandHandler", + "UpdateVariablesCommandHandler", ] diff --git a/api/core/workflow/graph_engine/command_processing/command_handlers.py b/api/core/workflow/graph_engine/command_processing/command_handlers.py index e9f109c88c..cfe856d9e8 100644 --- a/api/core/workflow/graph_engine/command_processing/command_handlers.py +++ b/api/core/workflow/graph_engine/command_processing/command_handlers.py @@ -4,9 +4,10 @@ from typing import final from typing_extensions import override from core.workflow.entities.pause_reason import SchedulingPause +from core.workflow.runtime import VariablePool from ..domain.graph_execution import GraphExecution -from ..entities.commands import AbortCommand, GraphEngineCommand, PauseCommand +from ..entities.commands import AbortCommand, GraphEngineCommand, PauseCommand, UpdateVariablesCommand from .command_processor import CommandHandler logger = logging.getLogger(__name__) @@ -31,3 +32,25 @@ class PauseCommandHandler(CommandHandler): reason = command.reason pause_reason = SchedulingPause(message=reason) execution.pause(pause_reason) + + +@final +class UpdateVariablesCommandHandler(CommandHandler): + def __init__(self, variable_pool: VariablePool) -> None: + self._variable_pool = variable_pool + + @override + def handle(self, command: GraphEngineCommand, execution: GraphExecution) -> None: + assert isinstance(command, UpdateVariablesCommand) + for update in command.updates: + try: + variable = update.value + self._variable_pool.add(variable.selector, variable) + logger.debug("Updated variable %s for workflow %s", variable.selector, execution.workflow_id) + except ValueError as exc: + logger.warning( + "Skipping invalid variable selector %s for workflow %s: %s", + getattr(update.value, "selector", None), + execution.workflow_id, + exc, + ) diff --git a/api/core/workflow/graph_engine/entities/commands.py b/api/core/workflow/graph_engine/entities/commands.py index 0d51b2b716..6dce03c94d 100644 --- a/api/core/workflow/graph_engine/entities/commands.py +++ b/api/core/workflow/graph_engine/entities/commands.py @@ -5,17 +5,21 @@ This module defines command types that can be sent to a running GraphEngine instance to control its execution flow. """ -from enum import StrEnum +from collections.abc import Sequence +from enum import StrEnum, auto from typing import Any from pydantic import BaseModel, Field +from core.variables.variables import VariableUnion + class CommandType(StrEnum): """Types of commands that can be sent to GraphEngine.""" - ABORT = "abort" - PAUSE = "pause" + ABORT = auto() + PAUSE = auto() + UPDATE_VARIABLES = auto() class GraphEngineCommand(BaseModel): @@ -37,3 +41,16 @@ class PauseCommand(GraphEngineCommand): command_type: CommandType = Field(default=CommandType.PAUSE, description="Type of command") reason: str = Field(default="unknown reason", description="reason for pause") + + +class VariableUpdate(BaseModel): + """Represents a single variable update instruction.""" + + value: VariableUnion = Field(description="New variable value") + + +class UpdateVariablesCommand(GraphEngineCommand): + """Command to update a group of variables in the variable pool.""" + + command_type: CommandType = Field(default=CommandType.UPDATE_VARIABLES, description="Type of command") + updates: Sequence[VariableUpdate] = Field(default_factory=list, description="Variable updates") diff --git a/api/core/workflow/graph_engine/graph_engine.py b/api/core/workflow/graph_engine/graph_engine.py index 2e8b8f345f..500ba4487b 100644 --- a/api/core/workflow/graph_engine/graph_engine.py +++ b/api/core/workflow/graph_engine/graph_engine.py @@ -8,6 +8,7 @@ Domain-Driven Design principles for improved maintainability and testability. import contextvars import logging import queue +import threading from collections.abc import Generator from typing import TYPE_CHECKING, cast, final @@ -30,8 +31,13 @@ from core.workflow.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWr if TYPE_CHECKING: # pragma: no cover - used only for static analysis from core.workflow.runtime.graph_runtime_state import GraphProtocol -from .command_processing import AbortCommandHandler, CommandProcessor, PauseCommandHandler -from .entities.commands import AbortCommand, PauseCommand +from .command_processing import ( + AbortCommandHandler, + CommandProcessor, + PauseCommandHandler, + UpdateVariablesCommandHandler, +) +from .entities.commands import AbortCommand, PauseCommand, UpdateVariablesCommand from .error_handler import ErrorHandler from .event_management import EventHandler, EventManager from .graph_state_manager import GraphStateManager @@ -70,10 +76,13 @@ class GraphEngine: scale_down_idle_time: float | None = None, ) -> None: """Initialize the graph engine with all subsystems and dependencies.""" + # stop event + self._stop_event = threading.Event() # Bind runtime state to current workflow context self._graph = graph self._graph_runtime_state = graph_runtime_state + self._graph_runtime_state.stop_event = self._stop_event self._graph_runtime_state.configure(graph=cast("GraphProtocol", graph)) self._command_channel = command_channel @@ -140,6 +149,9 @@ class GraphEngine: pause_handler = PauseCommandHandler() self._command_processor.register_handler(PauseCommand, pause_handler) + update_variables_handler = UpdateVariablesCommandHandler(self._graph_runtime_state.variable_pool) + self._command_processor.register_handler(UpdateVariablesCommand, update_variables_handler) + # === Extensibility === # Layers allow plugins to extend engine functionality self._layers: list[GraphEngineLayer] = [] @@ -169,6 +181,7 @@ class GraphEngine: max_workers=self._max_workers, scale_up_threshold=self._scale_up_threshold, scale_down_idle_time=self._scale_down_idle_time, + stop_event=self._stop_event, ) # === Orchestration === @@ -199,6 +212,7 @@ class GraphEngine: event_handler=self._event_handler_registry, execution_coordinator=self._execution_coordinator, event_emitter=self._event_manager, + stop_event=self._stop_event, ) # === Validation === @@ -212,9 +226,16 @@ class GraphEngine: if id(node.graph_runtime_state) != expected_state_id: raise ValueError(f"GraphRuntimeState consistency violation: Node '{node.id}' has a different instance") + def _bind_layer_context( + self, + layer: GraphEngineLayer, + ) -> None: + layer.initialize(ReadOnlyGraphRuntimeStateWrapper(self._graph_runtime_state), self._command_channel) + def layer(self, layer: GraphEngineLayer) -> "GraphEngine": """Add a layer for extending functionality.""" self._layers.append(layer) + self._bind_layer_context(layer) return self def run(self) -> Generator[GraphEngineEvent, None, None]: @@ -301,14 +322,7 @@ class GraphEngine: def _initialize_layers(self) -> None: """Initialize layers with context.""" self._event_manager.set_layers(self._layers) - # Create a read-only wrapper for the runtime state - read_only_state = ReadOnlyGraphRuntimeStateWrapper(self._graph_runtime_state) for layer in self._layers: - try: - layer.initialize(read_only_state, self._command_channel) - except Exception as e: - logger.warning("Failed to initialize layer %s: %s", layer.__class__.__name__, e) - try: layer.on_graph_start() except Exception as e: @@ -316,6 +330,7 @@ class GraphEngine: def _start_execution(self, *, resume: bool = False) -> None: """Start execution subsystems.""" + self._stop_event.clear() paused_nodes: list[str] = [] if resume: paused_nodes = self._graph_runtime_state.consume_paused_nodes() @@ -343,13 +358,12 @@ class GraphEngine: def _stop_execution(self) -> None: """Stop execution subsystems.""" + self._stop_event.set() self._dispatcher.stop() self._worker_pool.stop() # Don't mark complete here as the dispatcher already does it # Notify layers - logger = logging.getLogger(__name__) - for layer in self._layers: try: layer.on_graph_end(self._graph_execution.error) diff --git a/api/core/workflow/graph_engine/layers/README.md b/api/core/workflow/graph_engine/layers/README.md index 17845ee1f0..b0f295037c 100644 --- a/api/core/workflow/graph_engine/layers/README.md +++ b/api/core/workflow/graph_engine/layers/README.md @@ -8,7 +8,7 @@ Pluggable middleware for engine extensions. Abstract base class for layers. -- `initialize()` - Receive runtime context +- `initialize()` - Receive runtime context (runtime state is bound here and always available to hooks) - `on_graph_start()` - Execution start hook - `on_event()` - Process all events - `on_graph_end()` - Execution end hook @@ -34,6 +34,9 @@ engine.layer(debug_layer) engine.run() ``` +`engine.layer()` binds the read-only runtime state before execution, so +`graph_runtime_state` is always available inside layer hooks. + ## Custom Layers ```python diff --git a/api/core/workflow/graph_engine/layers/base.py b/api/core/workflow/graph_engine/layers/base.py index 780f92a0f4..89293b9b30 100644 --- a/api/core/workflow/graph_engine/layers/base.py +++ b/api/core/workflow/graph_engine/layers/base.py @@ -13,6 +13,14 @@ from core.workflow.nodes.base.node import Node from core.workflow.runtime import ReadOnlyGraphRuntimeState +class GraphEngineLayerNotInitializedError(Exception): + """Raised when a layer's runtime state is accessed before initialization.""" + + def __init__(self, layer_name: str | None = None) -> None: + name = layer_name or "GraphEngineLayer" + super().__init__(f"{name} runtime state is not initialized. Bind the layer to a GraphEngine before access.") + + class GraphEngineLayer(ABC): """ Abstract base class for GraphEngine layers. @@ -28,22 +36,27 @@ class GraphEngineLayer(ABC): def __init__(self) -> None: """Initialize the layer. Subclasses can override with custom parameters.""" - self.graph_runtime_state: ReadOnlyGraphRuntimeState | None = None + self._graph_runtime_state: ReadOnlyGraphRuntimeState | None = None self.command_channel: CommandChannel | None = None + @property + def graph_runtime_state(self) -> ReadOnlyGraphRuntimeState: + if self._graph_runtime_state is None: + raise GraphEngineLayerNotInitializedError(type(self).__name__) + return self._graph_runtime_state + def initialize(self, graph_runtime_state: ReadOnlyGraphRuntimeState, command_channel: CommandChannel) -> None: """ Initialize the layer with engine dependencies. - Called by GraphEngine before execution starts to inject the read-only runtime state - and command channel. This allows layers to observe engine context and send - commands, but prevents direct state modification. - + Called by GraphEngine to inject the read-only runtime state and command channel. + This is invoked when the layer is registered with a `GraphEngine` instance. + Implementations should be idempotent. Args: graph_runtime_state: Read-only view of the runtime state command_channel: Channel for sending commands to the engine """ - self.graph_runtime_state = graph_runtime_state + self._graph_runtime_state = graph_runtime_state self.command_channel = command_channel @abstractmethod diff --git a/api/core/workflow/graph_engine/layers/debug_logging.py b/api/core/workflow/graph_engine/layers/debug_logging.py index 034ebcf54f..e0402cd09c 100644 --- a/api/core/workflow/graph_engine/layers/debug_logging.py +++ b/api/core/workflow/graph_engine/layers/debug_logging.py @@ -109,10 +109,8 @@ class DebugLoggingLayer(GraphEngineLayer): self.logger.info("=" * 80) self.logger.info("🚀 GRAPH EXECUTION STARTED") self.logger.info("=" * 80) - - if self.graph_runtime_state: - # Log initial state - self.logger.info("Initial State:") + # Log initial state + self.logger.info("Initial State:") @override def on_event(self, event: GraphEngineEvent) -> None: @@ -243,8 +241,7 @@ class DebugLoggingLayer(GraphEngineLayer): self.logger.info(" Node retries: %s", self.retry_count) # Log final state if available - if self.graph_runtime_state and self.include_outputs: - if self.graph_runtime_state.outputs: - self.logger.info("Final outputs: %s", self._format_dict(self.graph_runtime_state.outputs)) + if self.include_outputs and self.graph_runtime_state.outputs: + self.logger.info("Final outputs: %s", self._format_dict(self.graph_runtime_state.outputs)) self.logger.info("=" * 80) diff --git a/api/core/workflow/graph_engine/layers/persistence.py b/api/core/workflow/graph_engine/layers/persistence.py index b70f36ec9e..e81df4f3b7 100644 --- a/api/core/workflow/graph_engine/layers/persistence.py +++ b/api/core/workflow/graph_engine/layers/persistence.py @@ -337,8 +337,6 @@ class WorkflowPersistenceLayer(GraphEngineLayer): if update_finished: execution.finished_at = naive_utc_now() runtime_state = self.graph_runtime_state - if runtime_state is None: - return execution.total_tokens = runtime_state.total_tokens execution.total_steps = runtime_state.node_run_steps execution.outputs = execution.outputs or runtime_state.outputs @@ -404,6 +402,4 @@ class WorkflowPersistenceLayer(GraphEngineLayer): def _system_variables(self) -> Mapping[str, Any]: runtime_state = self.graph_runtime_state - if runtime_state is None: - return {} return runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID) diff --git a/api/core/workflow/graph_engine/manager.py b/api/core/workflow/graph_engine/manager.py index 0577ba8f02..d2cfa755d9 100644 --- a/api/core/workflow/graph_engine/manager.py +++ b/api/core/workflow/graph_engine/manager.py @@ -3,14 +3,20 @@ GraphEngine Manager for sending control commands via Redis channel. This module provides a simplified interface for controlling workflow executions using the new Redis command channel, without requiring user permission checks. -Supports stop, pause, and resume operations. """ import logging +from collections.abc import Sequence from typing import final from core.workflow.graph_engine.command_channels.redis_channel import RedisChannel -from core.workflow.graph_engine.entities.commands import AbortCommand, GraphEngineCommand, PauseCommand +from core.workflow.graph_engine.entities.commands import ( + AbortCommand, + GraphEngineCommand, + PauseCommand, + UpdateVariablesCommand, + VariableUpdate, +) from extensions.ext_redis import redis_client logger = logging.getLogger(__name__) @@ -23,7 +29,6 @@ class GraphEngineManager: This class provides a simple interface for controlling workflow executions by sending commands through Redis channels, without user validation. - Supports stop and pause operations. """ @staticmethod @@ -45,6 +50,16 @@ class GraphEngineManager: pause_command = PauseCommand(reason=reason or "User requested pause") GraphEngineManager._send_command(task_id, pause_command) + @staticmethod + def send_update_variables_command(task_id: str, updates: Sequence[VariableUpdate]) -> None: + """Send a command to update variables in a running workflow.""" + + if not updates: + return + + update_command = UpdateVariablesCommand(updates=updates) + GraphEngineManager._send_command(task_id, update_command) + @staticmethod def _send_command(task_id: str, command: GraphEngineCommand) -> None: """Send a command to the workflow-specific Redis channel.""" diff --git a/api/core/workflow/graph_engine/orchestration/dispatcher.py b/api/core/workflow/graph_engine/orchestration/dispatcher.py index 334a3f77bf..27439a2412 100644 --- a/api/core/workflow/graph_engine/orchestration/dispatcher.py +++ b/api/core/workflow/graph_engine/orchestration/dispatcher.py @@ -44,6 +44,7 @@ class Dispatcher: event_queue: queue.Queue[GraphNodeEventBase], event_handler: "EventHandler", execution_coordinator: ExecutionCoordinator, + stop_event: threading.Event, event_emitter: EventManager | None = None, ) -> None: """ @@ -61,7 +62,7 @@ class Dispatcher: self._event_emitter = event_emitter self._thread: threading.Thread | None = None - self._stop_event = threading.Event() + self._stop_event = stop_event self._start_time: float | None = None def start(self) -> None: @@ -69,16 +70,14 @@ class Dispatcher: if self._thread and self._thread.is_alive(): return - self._stop_event.clear() self._start_time = time.time() self._thread = threading.Thread(target=self._dispatcher_loop, name="GraphDispatcher", daemon=True) self._thread.start() def stop(self) -> None: """Stop the dispatcher thread.""" - self._stop_event.set() if self._thread and self._thread.is_alive(): - self._thread.join(timeout=10.0) + self._thread.join(timeout=2.0) def _dispatcher_loop(self) -> None: """Main dispatcher loop.""" diff --git a/api/core/workflow/graph_engine/worker.py b/api/core/workflow/graph_engine/worker.py index e37a08ae47..83419830b6 100644 --- a/api/core/workflow/graph_engine/worker.py +++ b/api/core/workflow/graph_engine/worker.py @@ -42,6 +42,7 @@ class Worker(threading.Thread): event_queue: queue.Queue[GraphNodeEventBase], graph: Graph, layers: Sequence[GraphEngineLayer], + stop_event: threading.Event, worker_id: int = 0, flask_app: Flask | None = None, context_vars: contextvars.Context | None = None, @@ -65,13 +66,16 @@ class Worker(threading.Thread): self._worker_id = worker_id self._flask_app = flask_app self._context_vars = context_vars - self._stop_event = threading.Event() self._last_task_time = time.time() + self._stop_event = stop_event self._layers = layers if layers is not None else [] def stop(self) -> None: - """Signal the worker to stop processing.""" - self._stop_event.set() + """Worker is controlled via shared stop_event from GraphEngine. + + This method is a no-op retained for backward compatibility. + """ + pass @property def is_idle(self) -> bool: diff --git a/api/core/workflow/graph_engine/worker_management/worker_pool.py b/api/core/workflow/graph_engine/worker_management/worker_pool.py index 5b9234586b..df76ebe882 100644 --- a/api/core/workflow/graph_engine/worker_management/worker_pool.py +++ b/api/core/workflow/graph_engine/worker_management/worker_pool.py @@ -41,6 +41,7 @@ class WorkerPool: event_queue: queue.Queue[GraphNodeEventBase], graph: Graph, layers: list[GraphEngineLayer], + stop_event: threading.Event, flask_app: "Flask | None" = None, context_vars: "Context | None" = None, min_workers: int | None = None, @@ -81,6 +82,7 @@ class WorkerPool: self._worker_counter = 0 self._lock = threading.RLock() self._running = False + self._stop_event = stop_event # No longer tracking worker states with callbacks to avoid lock contention @@ -135,7 +137,7 @@ class WorkerPool: # Wait for workers to finish for worker in self._workers: if worker.is_alive(): - worker.join(timeout=10.0) + worker.join(timeout=2.0) self._workers.clear() @@ -152,6 +154,7 @@ class WorkerPool: worker_id=worker_id, flask_app=self._flask_app, context_vars=self._context_vars, + stop_event=self._stop_event, ) worker.start() diff --git a/api/core/workflow/nodes/base/node.py b/api/core/workflow/nodes/base/node.py index 8ebba3659c..e7282313b6 100644 --- a/api/core/workflow/nodes/base/node.py +++ b/api/core/workflow/nodes/base/node.py @@ -264,6 +264,10 @@ class Node(Generic[NodeDataT]): """ raise NotImplementedError + def _should_stop(self) -> bool: + """Check if execution should be stopped.""" + return self.graph_runtime_state.stop_event.is_set() + def run(self) -> Generator[GraphNodeEventBase, None, None]: execution_id = self.ensure_execution_id() self._start_at = naive_utc_now() @@ -332,6 +336,21 @@ class Node(Generic[NodeDataT]): yield event else: yield event + + if self._should_stop(): + error_message = "Execution cancelled" + yield NodeRunFailedEvent( + id=self.execution_id, + node_id=self._node_id, + node_type=self.node_type, + start_at=self._start_at, + node_run_result=NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + error=error_message, + ), + error=error_message, + ) + return except Exception as e: logger.exception("Node %s failed to run", self._node_id) result = NodeRunResult( diff --git a/api/core/workflow/nodes/node_factory.py b/api/core/workflow/nodes/node_factory.py index 1ba0494259..557d3a330d 100644 --- a/api/core/workflow/nodes/node_factory.py +++ b/api/core/workflow/nodes/node_factory.py @@ -11,6 +11,11 @@ from core.workflow.graph import NodeFactory from core.workflow.nodes.base.node import Node from core.workflow.nodes.code.code_node import CodeNode from core.workflow.nodes.code.limits import CodeNodeLimits +from core.workflow.nodes.template_transform.template_renderer import ( + CodeExecutorJinja2TemplateRenderer, + Jinja2TemplateRenderer, +) +from core.workflow.nodes.template_transform.template_transform_node import TemplateTransformNode from libs.typing import is_str, is_str_dict from .node_mapping import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING @@ -37,6 +42,7 @@ class DifyNodeFactory(NodeFactory): code_executor: type[CodeExecutor] | None = None, code_providers: Sequence[type[CodeNodeProvider]] | None = None, code_limits: CodeNodeLimits | None = None, + template_renderer: Jinja2TemplateRenderer | None = None, ) -> None: self.graph_init_params = graph_init_params self.graph_runtime_state = graph_runtime_state @@ -54,6 +60,7 @@ class DifyNodeFactory(NodeFactory): max_string_array_length=dify_config.CODE_MAX_STRING_ARRAY_LENGTH, max_object_array_length=dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH, ) + self._template_renderer = template_renderer or CodeExecutorJinja2TemplateRenderer() @override def create_node(self, node_config: dict[str, object]) -> Node: @@ -106,6 +113,14 @@ class DifyNodeFactory(NodeFactory): code_providers=self._code_providers, code_limits=self._code_limits, ) + if node_type == NodeType.TEMPLATE_TRANSFORM: + return TemplateTransformNode( + id=node_id, + config=node_config, + graph_init_params=self.graph_init_params, + graph_runtime_state=self.graph_runtime_state, + template_renderer=self._template_renderer, + ) return node_class( id=node_id, diff --git a/api/core/workflow/nodes/template_transform/template_renderer.py b/api/core/workflow/nodes/template_transform/template_renderer.py new file mode 100644 index 0000000000..a5f06bf2bb --- /dev/null +++ b/api/core/workflow/nodes/template_transform/template_renderer.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol + +from core.helper.code_executor.code_executor import CodeExecutionError, CodeExecutor, CodeLanguage + + +class TemplateRenderError(ValueError): + """Raised when rendering a Jinja2 template fails.""" + + +class Jinja2TemplateRenderer(Protocol): + """Render Jinja2 templates for template transform nodes.""" + + def render_template(self, template: str, variables: Mapping[str, Any]) -> str: + """Render a Jinja2 template with provided variables.""" + raise NotImplementedError + + +class CodeExecutorJinja2TemplateRenderer(Jinja2TemplateRenderer): + """Adapter that renders Jinja2 templates via CodeExecutor.""" + + _code_executor: type[CodeExecutor] + + def __init__(self, code_executor: type[CodeExecutor] | None = None) -> None: + self._code_executor = code_executor or CodeExecutor + + def render_template(self, template: str, variables: Mapping[str, Any]) -> str: + try: + result = self._code_executor.execute_workflow_code_template( + language=CodeLanguage.JINJA2, code=template, inputs=variables + ) + except CodeExecutionError as exc: + raise TemplateRenderError(str(exc)) from exc + + rendered = result.get("result") + if not isinstance(rendered, str): + raise TemplateRenderError("Template render result must be a string.") + return rendered diff --git a/api/core/workflow/nodes/template_transform/template_transform_node.py b/api/core/workflow/nodes/template_transform/template_transform_node.py index 2274323960..f7e0bccccf 100644 --- a/api/core/workflow/nodes/template_transform/template_transform_node.py +++ b/api/core/workflow/nodes/template_transform/template_transform_node.py @@ -1,18 +1,44 @@ from collections.abc import Mapping, Sequence -from typing import Any +from typing import TYPE_CHECKING, Any from configs import dify_config -from core.helper.code_executor.code_executor import CodeExecutionError, CodeExecutor, CodeLanguage from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base.node import Node from core.workflow.nodes.template_transform.entities import TemplateTransformNodeData +from core.workflow.nodes.template_transform.template_renderer import ( + CodeExecutorJinja2TemplateRenderer, + Jinja2TemplateRenderer, + TemplateRenderError, +) + +if TYPE_CHECKING: + from core.workflow.entities import GraphInitParams + from core.workflow.runtime import GraphRuntimeState MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH = dify_config.TEMPLATE_TRANSFORM_MAX_LENGTH class TemplateTransformNode(Node[TemplateTransformNodeData]): node_type = NodeType.TEMPLATE_TRANSFORM + _template_renderer: Jinja2TemplateRenderer + + def __init__( + self, + id: str, + config: Mapping[str, Any], + graph_init_params: "GraphInitParams", + graph_runtime_state: "GraphRuntimeState", + *, + template_renderer: Jinja2TemplateRenderer | None = None, + ) -> None: + super().__init__( + id=id, + config=config, + graph_init_params=graph_init_params, + graph_runtime_state=graph_runtime_state, + ) + self._template_renderer = template_renderer or CodeExecutorJinja2TemplateRenderer() @classmethod def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]: @@ -39,13 +65,11 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]): variables[variable_name] = value.to_object() if value else None # Run code try: - result = CodeExecutor.execute_workflow_code_template( - language=CodeLanguage.JINJA2, code=self.node_data.template, inputs=variables - ) - except CodeExecutionError as e: + rendered = self._template_renderer.render_template(self.node_data.template, variables) + except TemplateRenderError as e: return NodeRunResult(inputs=variables, status=WorkflowNodeExecutionStatus.FAILED, error=str(e)) - if len(result["result"]) > MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH: + if len(rendered) > MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH: return NodeRunResult( inputs=variables, status=WorkflowNodeExecutionStatus.FAILED, @@ -53,7 +77,7 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]): ) return NodeRunResult( - status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs=variables, outputs={"output": result["result"]} + status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs=variables, outputs={"output": rendered} ) @classmethod diff --git a/api/core/workflow/nodes/variable_assigner/common/impl.py b/api/core/workflow/nodes/variable_assigner/common/impl.py deleted file mode 100644 index 050e213535..0000000000 --- a/api/core/workflow/nodes/variable_assigner/common/impl.py +++ /dev/null @@ -1,28 +0,0 @@ -from sqlalchemy import select -from sqlalchemy.orm import Session - -from core.variables.variables import Variable -from extensions.ext_database import db -from models import ConversationVariable - -from .exc import VariableOperatorNodeError - - -class ConversationVariableUpdaterImpl: - def update(self, conversation_id: str, variable: Variable): - stmt = select(ConversationVariable).where( - ConversationVariable.id == variable.id, ConversationVariable.conversation_id == conversation_id - ) - with Session(db.engine) as session: - row = session.scalar(stmt) - if not row: - raise VariableOperatorNodeError("conversation variable not found in the database") - row.data = variable.model_dump_json() - session.commit() - - def flush(self): - pass - - -def conversation_variable_updater_factory() -> ConversationVariableUpdaterImpl: - return ConversationVariableUpdaterImpl() diff --git a/api/core/workflow/nodes/variable_assigner/v1/node.py b/api/core/workflow/nodes/variable_assigner/v1/node.py index da23207b62..d2ea7d94ea 100644 --- a/api/core/workflow/nodes/variable_assigner/v1/node.py +++ b/api/core/workflow/nodes/variable_assigner/v1/node.py @@ -1,9 +1,8 @@ -from collections.abc import Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, TypeAlias +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Any from core.variables import SegmentType, Variable from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID -from core.workflow.conversation_variable_updater import ConversationVariableUpdater from core.workflow.entities import GraphInitParams from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult @@ -11,19 +10,14 @@ from core.workflow.nodes.base.node import Node from core.workflow.nodes.variable_assigner.common import helpers as common_helpers from core.workflow.nodes.variable_assigner.common.exc import VariableOperatorNodeError -from ..common.impl import conversation_variable_updater_factory from .node_data import VariableAssignerData, WriteMode if TYPE_CHECKING: from core.workflow.runtime import GraphRuntimeState -_CONV_VAR_UPDATER_FACTORY: TypeAlias = Callable[[], ConversationVariableUpdater] - - class VariableAssignerNode(Node[VariableAssignerData]): node_type = NodeType.VARIABLE_ASSIGNER - _conv_var_updater_factory: _CONV_VAR_UPDATER_FACTORY def __init__( self, @@ -31,7 +25,6 @@ class VariableAssignerNode(Node[VariableAssignerData]): config: Mapping[str, Any], graph_init_params: "GraphInitParams", graph_runtime_state: "GraphRuntimeState", - conv_var_updater_factory: _CONV_VAR_UPDATER_FACTORY = conversation_variable_updater_factory, ): super().__init__( id=id, @@ -39,7 +32,6 @@ class VariableAssignerNode(Node[VariableAssignerData]): graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - self._conv_var_updater_factory = conv_var_updater_factory @classmethod def version(cls) -> str: @@ -96,16 +88,7 @@ class VariableAssignerNode(Node[VariableAssignerData]): # Over write the variable. self.graph_runtime_state.variable_pool.add(assigned_variable_selector, updated_variable) - # TODO: Move database operation to the pipeline. - # Update conversation variable. - conversation_id = self.graph_runtime_state.variable_pool.get(["sys", "conversation_id"]) - if not conversation_id: - raise VariableOperatorNodeError("conversation_id not found") - conv_var_updater = self._conv_var_updater_factory() - conv_var_updater.update(conversation_id=conversation_id.text, variable=updated_variable) - conv_var_updater.flush() updated_variables = [common_helpers.variable_to_processed_data(assigned_variable_selector, updated_variable)] - return NodeRunResult( status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs={ diff --git a/api/core/workflow/nodes/variable_assigner/v2/node.py b/api/core/workflow/nodes/variable_assigner/v2/node.py index 389fb54d35..486e6bb6a7 100644 --- a/api/core/workflow/nodes/variable_assigner/v2/node.py +++ b/api/core/workflow/nodes/variable_assigner/v2/node.py @@ -1,24 +1,20 @@ import json from collections.abc import Mapping, MutableMapping, Sequence -from typing import Any, cast +from typing import TYPE_CHECKING, Any -from core.app.entities.app_invoke_entities import InvokeFrom from core.variables import SegmentType, Variable from core.variables.consts import SELECTORS_LENGTH from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID -from core.workflow.conversation_variable_updater import ConversationVariableUpdater from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base.node import Node from core.workflow.nodes.variable_assigner.common import helpers as common_helpers from core.workflow.nodes.variable_assigner.common.exc import VariableOperatorNodeError -from core.workflow.nodes.variable_assigner.common.impl import conversation_variable_updater_factory from . import helpers from .entities import VariableAssignerNodeData, VariableOperationItem from .enums import InputType, Operation from .exc import ( - ConversationIDNotFoundError, InputTypeNotSupportedError, InvalidDataError, InvalidInputValueError, @@ -26,6 +22,10 @@ from .exc import ( VariableNotFoundError, ) +if TYPE_CHECKING: + from core.workflow.entities import GraphInitParams + from core.workflow.runtime import GraphRuntimeState + def _target_mapping_from_item(mapping: MutableMapping[str, Sequence[str]], node_id: str, item: VariableOperationItem): selector_node_id = item.variable_selector[0] @@ -53,6 +53,20 @@ def _source_mapping_from_item(mapping: MutableMapping[str, Sequence[str]], node_ class VariableAssignerNode(Node[VariableAssignerNodeData]): node_type = NodeType.VARIABLE_ASSIGNER + def __init__( + self, + id: str, + config: Mapping[str, Any], + graph_init_params: "GraphInitParams", + graph_runtime_state: "GraphRuntimeState", + ): + super().__init__( + id=id, + config=config, + graph_init_params=graph_init_params, + graph_runtime_state=graph_runtime_state, + ) + def blocks_variable_output(self, variable_selectors: set[tuple[str, ...]]) -> bool: """ Check if this Variable Assigner node blocks the output of specific variables. @@ -70,9 +84,6 @@ class VariableAssignerNode(Node[VariableAssignerNodeData]): return False - def _conv_var_updater_factory(self) -> ConversationVariableUpdater: - return conversation_variable_updater_factory() - @classmethod def version(cls) -> str: return "2" @@ -179,26 +190,12 @@ class VariableAssignerNode(Node[VariableAssignerNodeData]): # remove the duplicated items first. updated_variable_selectors = list(set(map(tuple, updated_variable_selectors))) - conv_var_updater = self._conv_var_updater_factory() - # Update variables for selector in updated_variable_selectors: variable = self.graph_runtime_state.variable_pool.get(selector) if not isinstance(variable, Variable): raise VariableNotFoundError(variable_selector=selector) process_data[variable.name] = variable.value - if variable.selector[0] == CONVERSATION_VARIABLE_NODE_ID: - conversation_id = self.graph_runtime_state.variable_pool.get(["sys", "conversation_id"]) - if not conversation_id: - if self.invoke_from != InvokeFrom.DEBUGGER: - raise ConversationIDNotFoundError - else: - conversation_id = conversation_id.value - conv_var_updater.update( - conversation_id=cast(str, conversation_id), - variable=variable, - ) - conv_var_updater.flush() updated_variables = [ common_helpers.variable_to_processed_data(selector, seg) for selector in updated_variable_selectors diff --git a/api/core/workflow/runtime/graph_runtime_state.py b/api/core/workflow/runtime/graph_runtime_state.py index 1561b789df..401cecc162 100644 --- a/api/core/workflow/runtime/graph_runtime_state.py +++ b/api/core/workflow/runtime/graph_runtime_state.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib import json +import threading from collections.abc import Mapping, Sequence from copy import deepcopy from dataclasses import dataclass @@ -168,6 +169,7 @@ class GraphRuntimeState: self._pending_response_coordinator_dump: str | None = None self._pending_graph_execution_workflow_id: str | None = None self._paused_nodes: set[str] = set() + self.stop_event: threading.Event = threading.Event() if graph is not None: self.attach_graph(graph) diff --git a/api/core/workflow/runtime/graph_runtime_state_protocol.py b/api/core/workflow/runtime/graph_runtime_state_protocol.py index 5e0878e873..bfbb5ba704 100644 --- a/api/core/workflow/runtime/graph_runtime_state_protocol.py +++ b/api/core/workflow/runtime/graph_runtime_state_protocol.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, Protocol from core.model_runtime.entities.llm_entities import LLMUsage @@ -9,7 +9,7 @@ from core.workflow.system_variable import SystemVariableReadOnlyView class ReadOnlyVariablePool(Protocol): """Read-only interface for VariablePool.""" - def get(self, node_id: str, variable_key: str) -> Segment | None: + def get(self, selector: Sequence[str], /) -> Segment | None: """Get a variable value (read-only).""" ... diff --git a/api/core/workflow/runtime/read_only_wrappers.py b/api/core/workflow/runtime/read_only_wrappers.py index 8539727fd6..d3e4c60d9b 100644 --- a/api/core/workflow/runtime/read_only_wrappers.py +++ b/api/core/workflow/runtime/read_only_wrappers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from copy import deepcopy from typing import Any @@ -18,9 +18,9 @@ class ReadOnlyVariablePoolWrapper: def __init__(self, variable_pool: VariablePool) -> None: self._variable_pool = variable_pool - def get(self, node_id: str, variable_key: str) -> Segment | None: + def get(self, selector: Sequence[str], /) -> Segment | None: """Return a copy of a variable value if present.""" - value = self._variable_pool.get([node_id, variable_key]) + value = self._variable_pool.get(selector) return deepcopy(value) if value is not None else None def get_all_by_node(self, node_id: str) -> Mapping[str, object]: diff --git a/api/docker/entrypoint.sh b/api/docker/entrypoint.sh index 5a69eb15ac..c0279f893b 100755 --- a/api/docker/entrypoint.sh +++ b/api/docker/entrypoint.sh @@ -3,8 +3,9 @@ set -e # Set UTF-8 encoding to address potential encoding issues in containerized environments -export LANG=${LANG:-en_US.UTF-8} -export LC_ALL=${LC_ALL:-en_US.UTF-8} +# Use C.UTF-8 which is universally available in all containers +export LANG=${LANG:-C.UTF-8} +export LC_ALL=${LC_ALL:-C.UTF-8} export PYTHONIOENCODING=${PYTHONIOENCODING:-utf-8} if [[ "${MIGRATION_ENABLED}" == "true" ]]; then diff --git a/api/extensions/ext_blueprints.py b/api/extensions/ext_blueprints.py index cf994c11df..7d13f0c061 100644 --- a/api/extensions/ext_blueprints.py +++ b/api/extensions/ext_blueprints.py @@ -6,6 +6,7 @@ BASE_CORS_HEADERS: tuple[str, ...] = ("Content-Type", HEADER_NAME_APP_CODE, HEAD SERVICE_API_HEADERS: tuple[str, ...] = (*BASE_CORS_HEADERS, "Authorization") AUTHENTICATED_HEADERS: tuple[str, ...] = (*SERVICE_API_HEADERS, HEADER_NAME_CSRF_TOKEN) FILES_HEADERS: tuple[str, ...] = (*BASE_CORS_HEADERS, HEADER_NAME_CSRF_TOKEN) +EMBED_HEADERS: tuple[str, ...] = ("Content-Type", HEADER_NAME_APP_CODE) EXPOSED_HEADERS: tuple[str, ...] = ("X-Version", "X-Env", "X-Trace-Id") @@ -42,10 +43,28 @@ def init_app(app: DifyApp): _apply_cors_once( web_bp, - resources={r"/*": {"origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS}}, - supports_credentials=True, - allow_headers=list(AUTHENTICATED_HEADERS), - methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"], + resources={ + # Embedded bot endpoints (unauthenticated, cross-origin safe) + r"^/chat-messages$": { + "origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS, + "supports_credentials": False, + "allow_headers": list(EMBED_HEADERS), + "methods": ["GET", "POST", "OPTIONS"], + }, + r"^/chat-messages/.*": { + "origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS, + "supports_credentials": False, + "allow_headers": list(EMBED_HEADERS), + "methods": ["GET", "POST", "OPTIONS"], + }, + # Default web application endpoints (authenticated) + r"/*": { + "origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS, + "supports_credentials": True, + "allow_headers": list(AUTHENTICATED_HEADERS), + "methods": ["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"], + }, + }, expose_headers=list(EXPOSED_HEADERS), ) app.register_blueprint(web_bp) diff --git a/api/extensions/ext_commands.py b/api/extensions/ext_commands.py index 71a63168a5..daa3756dba 100644 --- a/api/extensions/ext_commands.py +++ b/api/extensions/ext_commands.py @@ -11,6 +11,7 @@ def init_app(app: DifyApp): create_tenant, extract_plugins, extract_unique_plugins, + file_usage, fix_app_site_missing, install_plugins, install_rag_pipeline_plugins, @@ -47,6 +48,7 @@ def init_app(app: DifyApp): clear_free_plan_tenant_expired_logs, clear_orphaned_file_records, remove_orphaned_files_on_storage, + file_usage, setup_system_tool_oauth_client, setup_system_trigger_oauth_client, cleanup_orphaned_draft_variables, diff --git a/api/extensions/ext_database.py b/api/extensions/ext_database.py index c90b1d0a9f..2e0d4c889a 100644 --- a/api/extensions/ext_database.py +++ b/api/extensions/ext_database.py @@ -53,3 +53,10 @@ def _setup_gevent_compatibility(): def init_app(app: DifyApp): db.init_app(app) _setup_gevent_compatibility() + + # Eagerly build the engine so pool_size/max_overflow/etc. come from config + try: + with app.app_context(): + _ = db.engine # triggers engine creation with the configured options + except Exception: + logger.exception("Failed to initialize SQLAlchemy engine during app startup") diff --git a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py index 6e6631cfef..1119534d52 100644 --- a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py +++ b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py @@ -22,6 +22,18 @@ from models.enums import WorkflowRunTriggeredFrom logger = logging.getLogger(__name__) +def to_serializable(obj): + """ + Convert non-JSON-serializable objects into JSON-compatible formats. + + - Uses `to_dict()` if it's a callable method. + - Falls back to string representation. + """ + if hasattr(obj, "to_dict") and callable(obj.to_dict): + return obj.to_dict() + return str(obj) + + class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository): def __init__( self, @@ -69,6 +81,11 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository): # Set to True to enable dual-write for safe migration, False to use LogStore only self._enable_dual_write = os.environ.get("LOGSTORE_DUAL_WRITE_ENABLED", "true").lower() == "true" + # Control flag for whether to write the `graph` field to LogStore. + # If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; + # otherwise write an empty {} instead. Defaults to writing the `graph` field. + self._enable_put_graph_field = os.environ.get("LOGSTORE_ENABLE_PUT_GRAPH_FIELD", "true").lower() == "true" + def _to_logstore_model(self, domain_model: WorkflowExecution) -> list[tuple[str, str]]: """ Convert a domain model to a logstore model (List[Tuple[str, str]]). @@ -108,9 +125,24 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository): ), ("type", domain_model.workflow_type.value), ("version", domain_model.workflow_version), - ("graph", json.dumps(domain_model.graph, ensure_ascii=False) if domain_model.graph else "{}"), - ("inputs", json.dumps(domain_model.inputs, ensure_ascii=False) if domain_model.inputs else "{}"), - ("outputs", json.dumps(domain_model.outputs, ensure_ascii=False) if domain_model.outputs else "{}"), + ( + "graph", + json.dumps(domain_model.graph, ensure_ascii=False, default=to_serializable) + if domain_model.graph and self._enable_put_graph_field + else "{}", + ), + ( + "inputs", + json.dumps(domain_model.inputs, ensure_ascii=False, default=to_serializable) + if domain_model.inputs + else "{}", + ), + ( + "outputs", + json.dumps(domain_model.outputs, ensure_ascii=False, default=to_serializable) + if domain_model.outputs + else "{}", + ), ("status", domain_model.status.value), ("error_message", domain_model.error_message or ""), ("total_tokens", str(domain_model.total_tokens)), diff --git a/api/libs/helper.py b/api/libs/helper.py index 74e1808e49..07c4823727 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -32,6 +32,38 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def escape_like_pattern(pattern: str) -> str: + """ + Escape special characters in a string for safe use in SQL LIKE patterns. + + This function escapes the special characters used in SQL LIKE patterns: + - Backslash (\\) -> \\ + - Percent (%) -> \\% + - Underscore (_) -> \\_ + + The escaped pattern can then be safely used in SQL LIKE queries with the + ESCAPE '\\' clause to prevent SQL injection via LIKE wildcards. + + Args: + pattern: The string pattern to escape + + Returns: + Escaped string safe for use in SQL LIKE queries + + Examples: + >>> escape_like_pattern("50% discount") + '50\\% discount' + >>> escape_like_pattern("test_data") + 'test\\_data' + >>> escape_like_pattern("path\\to\\file") + 'path\\\\to\\\\file' + """ + if not pattern: + return pattern + # Escape backslash first, then percent and underscore + return pattern.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + def extract_tenant_id(user: Union["Account", "EndUser"]) -> str | None: """ Extract tenant_id from Account or EndUser object. diff --git a/api/models/model.py b/api/models/model.py index 88cb945b3f..b6f2751a72 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -70,6 +70,7 @@ class AppMode(StrEnum): class IconType(StrEnum): IMAGE = auto() EMOJI = auto() + LINK = auto() class App(Base): @@ -81,7 +82,7 @@ class App(Base): name: Mapped[str] = mapped_column(String(255)) description: Mapped[str] = mapped_column(LongText, default=sa.text("''")) mode: Mapped[str] = mapped_column(String(255)) - icon_type: Mapped[str | None] = mapped_column(String(255)) # image, emoji + icon_type: Mapped[str | None] = mapped_column(String(255)) # image, emoji, link icon = mapped_column(String(255)) icon_background: Mapped[str | None] = mapped_column(String(255)) app_model_config_id = mapped_column(StringUUID, nullable=True) @@ -1419,15 +1420,20 @@ class MessageAnnotation(Base): app_id: Mapped[str] = mapped_column(StringUUID) conversation_id: Mapped[str | None] = mapped_column(StringUUID, sa.ForeignKey("conversations.id")) message_id: Mapped[str | None] = mapped_column(StringUUID) - question = mapped_column(LongText, nullable=True) - content = mapped_column(LongText, nullable=False) + question: Mapped[str | None] = mapped_column(LongText, nullable=True) + content: Mapped[str] = mapped_column(LongText, nullable=False) hit_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0")) - account_id = mapped_column(StringUUID, nullable=False) - created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at = mapped_column( + account_id: Mapped[str] = mapped_column(StringUUID, nullable=False) + created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) + updated_at: Mapped[datetime] = mapped_column( sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp() ) + @property + def question_text(self) -> str: + """Return a non-null question string, falling back to the answer content.""" + return self.question or self.content + @property def account(self): account = db.session.query(Account).where(Account.id == self.account_id).first() diff --git a/api/models/workflow.py b/api/models/workflow.py index 853d5afefc..c04e4e9762 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -1506,6 +1506,7 @@ class WorkflowDraftVariable(Base): file_id: str | None = None, ) -> "WorkflowDraftVariable": variable = WorkflowDraftVariable() + variable.id = str(uuid4()) variable.created_at = naive_utc_now() variable.updated_at = naive_utc_now() variable.description = description diff --git a/api/services/annotation_service.py b/api/services/annotation_service.py index d03cbddceb..b73302508a 100644 --- a/api/services/annotation_service.py +++ b/api/services/annotation_service.py @@ -77,7 +77,7 @@ class AppAnnotationService: if annotation_setting: add_annotation_to_index_task.delay( annotation.id, - annotation.question, + question, current_tenant_id, app_id, annotation_setting.collection_binding_id, @@ -137,13 +137,16 @@ class AppAnnotationService: if not app: raise NotFound("App not found") if keyword: + from libs.helper import escape_like_pattern + + escaped_keyword = escape_like_pattern(keyword) stmt = ( select(MessageAnnotation) .where(MessageAnnotation.app_id == app_id) .where( or_( - MessageAnnotation.question.ilike(f"%{keyword}%"), - MessageAnnotation.content.ilike(f"%{keyword}%"), + MessageAnnotation.question.ilike(f"%{escaped_keyword}%", escape="\\"), + MessageAnnotation.content.ilike(f"%{escaped_keyword}%", escape="\\"), ) ) .order_by(MessageAnnotation.created_at.desc(), MessageAnnotation.id.desc()) @@ -253,7 +256,7 @@ class AppAnnotationService: if app_annotation_setting: update_annotation_to_index_task.delay( annotation.id, - annotation.question, + annotation.question_text, current_tenant_id, app_id, app_annotation_setting.collection_binding_id, diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index deba0b79e8..acd2a25a86 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -32,7 +32,7 @@ from extensions.ext_redis import redis_client from factories import variable_factory from libs.datetime_utils import naive_utc_now from models import Account, App, AppMode -from models.model import AppModelConfig +from models.model import AppModelConfig, IconType from models.workflow import Workflow from services.plugin.dependencies_analysis import DependenciesAnalysisService from services.workflow_draft_variable_service import WorkflowDraftVariableService @@ -428,10 +428,10 @@ class AppDslService: # Set icon type icon_type_value = icon_type or app_data.get("icon_type") - if icon_type_value in ["emoji", "link", "image"]: + if icon_type_value in [IconType.EMOJI.value, IconType.IMAGE.value, IconType.LINK.value]: icon_type = icon_type_value else: - icon_type = "emoji" + icon_type = IconType.EMOJI.value icon = icon or str(app_data.get("icon", "")) if app: diff --git a/api/services/app_service.py b/api/services/app_service.py index ef89a4fd10..02ebfbace0 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -55,8 +55,11 @@ class AppService: if args.get("is_created_by_me", False): filters.append(App.created_by == user_id) if args.get("name"): + from libs.helper import escape_like_pattern + name = args["name"][:30] - filters.append(App.name.ilike(f"%{name}%")) + escaped_name = escape_like_pattern(name) + filters.append(App.name.ilike(f"%{escaped_name}%", escape="\\")) # Check if tag_ids is not empty to avoid WHERE false condition if args.get("tag_ids") and len(args["tag_ids"]) > 0: target_ids = TagService.get_target_ids_by_tag_ids("app", tenant_id, args["tag_ids"]) diff --git a/api/services/conversation_service.py b/api/services/conversation_service.py index 659e7406fb..295d48d8a1 100644 --- a/api/services/conversation_service.py +++ b/api/services/conversation_service.py @@ -11,13 +11,13 @@ from core.app.entities.app_invoke_entities import InvokeFrom from core.db.session_factory import session_factory from core.llm_generator.llm_generator import LLMGenerator from core.variables.types import SegmentType -from core.workflow.nodes.variable_assigner.common.impl import conversation_variable_updater_factory from extensions.ext_database import db from factories import variable_factory from libs.datetime_utils import naive_utc_now from libs.infinite_scroll_pagination import InfiniteScrollPagination from models import Account, ConversationVariable from models.model import App, Conversation, EndUser, Message +from services.conversation_variable_updater import ConversationVariableUpdater from services.errors.conversation import ( ConversationNotExistsError, ConversationVariableNotExistsError, @@ -218,7 +218,9 @@ class ConversationService: # Apply variable_name filter if provided if variable_name: # Filter using JSON extraction to match variable names case-insensitively - escaped_variable_name = variable_name.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + from libs.helper import escape_like_pattern + + escaped_variable_name = escape_like_pattern(variable_name) # Filter using JSON extraction to match variable names case-insensitively if dify_config.DB_TYPE in ["mysql", "oceanbase", "seekdb"]: stmt = stmt.where( @@ -335,7 +337,7 @@ class ConversationService: updated_variable = variable_factory.build_conversation_variable_from_mapping(updated_variable_dict) # Use the conversation variable updater to persist the changes - updater = conversation_variable_updater_factory() + updater = ConversationVariableUpdater(session_factory.get_session_maker()) updater.update(conversation_id, updated_variable) updater.flush() diff --git a/api/services/conversation_variable_updater.py b/api/services/conversation_variable_updater.py new file mode 100644 index 0000000000..acc0ec2b22 --- /dev/null +++ b/api/services/conversation_variable_updater.py @@ -0,0 +1,28 @@ +from sqlalchemy import select +from sqlalchemy.orm import Session, sessionmaker + +from core.variables.variables import Variable +from models import ConversationVariable + + +class ConversationVariableNotFoundError(Exception): + pass + + +class ConversationVariableUpdater: + def __init__(self, session_maker: sessionmaker[Session]) -> None: + self._session_maker: sessionmaker[Session] = session_maker + + def update(self, conversation_id: str, variable: Variable) -> None: + stmt = select(ConversationVariable).where( + ConversationVariable.id == variable.id, ConversationVariable.conversation_id == conversation_id + ) + with self._session_maker() as session: + row = session.scalar(stmt) + if not row: + raise ConversationVariableNotFoundError("conversation variable not found in the database") + row.data = variable.model_dump_json() + session.commit() + + def flush(self) -> None: + pass diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index ac4b25c5dc..18e5613438 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -144,7 +144,8 @@ class DatasetService: query = query.where(Dataset.permission == DatasetPermissionEnum.ALL_TEAM) if search: - query = query.where(Dataset.name.ilike(f"%{search}%")) + escaped_search = helper.escape_like_pattern(search) + query = query.where(Dataset.name.ilike(f"%{escaped_search}%", escape="\\")) # Check if tag_ids is not empty to avoid WHERE false condition if tag_ids and len(tag_ids) > 0: @@ -3423,7 +3424,8 @@ class SegmentService: .order_by(ChildChunk.position.asc()) ) if keyword: - query = query.where(ChildChunk.content.ilike(f"%{keyword}%")) + escaped_keyword = helper.escape_like_pattern(keyword) + query = query.where(ChildChunk.content.ilike(f"%{escaped_keyword}%", escape="\\")) return db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False) @classmethod @@ -3456,7 +3458,8 @@ class SegmentService: query = query.where(DocumentSegment.status.in_(status_list)) if keyword: - query = query.where(DocumentSegment.content.ilike(f"%{keyword}%")) + escaped_keyword = helper.escape_like_pattern(keyword) + query = query.where(DocumentSegment.content.ilike(f"%{escaped_keyword}%", escape="\\")) query = query.order_by(DocumentSegment.position.asc(), DocumentSegment.id.asc()) paginated_segments = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False) diff --git a/api/services/external_knowledge_service.py b/api/services/external_knowledge_service.py index 40faa85b9a..65dd41af43 100644 --- a/api/services/external_knowledge_service.py +++ b/api/services/external_knowledge_service.py @@ -35,7 +35,10 @@ class ExternalDatasetService: .order_by(ExternalKnowledgeApis.created_at.desc()) ) if search: - query = query.where(ExternalKnowledgeApis.name.ilike(f"%{search}%")) + from libs.helper import escape_like_pattern + + escaped_search = escape_like_pattern(search) + query = query.where(ExternalKnowledgeApis.name.ilike(f"%{escaped_search}%", escape="\\")) external_knowledge_apis = db.paginate( select=query, page=page, per_page=per_page, max_per_page=100, error_out=False diff --git a/api/services/tag_service.py b/api/services/tag_service.py index 937e6593fe..bd3585acf4 100644 --- a/api/services/tag_service.py +++ b/api/services/tag_service.py @@ -19,7 +19,10 @@ class TagService: .where(Tag.type == tag_type, Tag.tenant_id == current_tenant_id) ) if keyword: - query = query.where(sa.and_(Tag.name.ilike(f"%{keyword}%"))) + from libs.helper import escape_like_pattern + + escaped_keyword = escape_like_pattern(keyword) + query = query.where(sa.and_(Tag.name.ilike(f"%{escaped_keyword}%", escape="\\"))) query = query.group_by(Tag.id, Tag.type, Tag.name, Tag.created_at) results: list = query.order_by(Tag.created_at.desc()).all() return results diff --git a/api/services/trigger/trigger_provider_service.py b/api/services/trigger/trigger_provider_service.py index ef77c33c1b..4131d75145 100644 --- a/api/services/trigger/trigger_provider_service.py +++ b/api/services/trigger/trigger_provider_service.py @@ -853,7 +853,7 @@ class TriggerProviderService: """ Create a subscription builder for rebuilding an existing subscription. - This method creates a builder pre-filled with data from the rebuild request, + This method rebuild the subscription by call DELETE and CREATE API of the third party provider(e.g. GitHub) keeping the same subscription_id and endpoint_id so the webhook URL remains unchanged. :param tenant_id: Tenant ID @@ -868,111 +868,50 @@ class TriggerProviderService: if not provider_controller: raise ValueError(f"Provider {provider_id} not found") - # Use distributed lock to prevent race conditions on the same subscription - lock_key = f"trigger_subscription_rebuild_lock:{tenant_id}_{subscription_id}" - with redis_client.lock(lock_key, timeout=20): - with Session(db.engine, expire_on_commit=False) as session: - try: - # Get subscription within the transaction - subscription: TriggerSubscription | None = ( - session.query(TriggerSubscription).filter_by(tenant_id=tenant_id, id=subscription_id).first() - ) - if not subscription: - raise ValueError(f"Subscription {subscription_id} not found") + subscription = TriggerProviderService.get_subscription_by_id( + tenant_id=tenant_id, + subscription_id=subscription_id, + ) + if not subscription: + raise ValueError(f"Subscription {subscription_id} not found") - credential_type = CredentialType.of(subscription.credential_type) - if credential_type not in [CredentialType.OAUTH2, CredentialType.API_KEY]: - raise ValueError("Credential type not supported for rebuild") + credential_type = CredentialType.of(subscription.credential_type) + if credential_type not in {CredentialType.OAUTH2, CredentialType.API_KEY}: + raise ValueError(f"Credential type {credential_type} not supported for auto creation") - # Decrypt existing credentials for merging - credential_encrypter, _ = create_trigger_provider_encrypter_for_subscription( - tenant_id=tenant_id, - controller=provider_controller, - subscription=subscription, - ) - decrypted_credentials = dict(credential_encrypter.decrypt(subscription.credentials)) + # Delete the previous subscription + user_id = subscription.user_id + unsubscribe_result = TriggerManager.unsubscribe_trigger( + tenant_id=tenant_id, + user_id=user_id, + provider_id=provider_id, + subscription=subscription.to_entity(), + credentials=subscription.credentials, + credential_type=credential_type, + ) + if not unsubscribe_result.success: + raise ValueError(f"Failed to delete previous subscription: {unsubscribe_result.message}") - # Merge credentials: if caller passed HIDDEN_VALUE, retain existing decrypted value - merged_credentials: dict[str, Any] = { - key: value if value != HIDDEN_VALUE else decrypted_credentials.get(key, UNKNOWN_VALUE) - for key, value in credentials.items() - } - - user_id = subscription.user_id - - # TODO: Trying to invoke update api of the plugin trigger provider - - # FALLBACK: If the update api is not implemented, - # delete the previous subscription and create a new one - - # Unsubscribe the previous subscription (external call, but we'll handle errors) - try: - TriggerManager.unsubscribe_trigger( - tenant_id=tenant_id, - user_id=user_id, - provider_id=provider_id, - subscription=subscription.to_entity(), - credentials=decrypted_credentials, - credential_type=credential_type, - ) - except Exception as e: - logger.exception("Error unsubscribing trigger during rebuild", exc_info=e) - # Continue anyway - the subscription might already be deleted externally - - # Create a new subscription with the same subscription_id and endpoint_id (external call) - new_subscription: TriggerSubscriptionEntity = TriggerManager.subscribe_trigger( - tenant_id=tenant_id, - user_id=user_id, - provider_id=provider_id, - endpoint=generate_plugin_trigger_endpoint_url(subscription.endpoint_id), - parameters=parameters, - credentials=merged_credentials, - credential_type=credential_type, - ) - - # Update the subscription in the same transaction - # Inline update logic to reuse the same session - if name is not None and name != subscription.name: - existing = ( - session.query(TriggerSubscription) - .filter_by(tenant_id=tenant_id, provider_id=str(provider_id), name=name) - .first() - ) - if existing and existing.id != subscription.id: - raise ValueError(f"Subscription name '{name}' already exists for this provider") - subscription.name = name - - # Update parameters - subscription.parameters = dict(parameters) - - # Update credentials with merged (and encrypted) values - subscription.credentials = dict(credential_encrypter.encrypt(merged_credentials)) - - # Update properties - if new_subscription.properties: - properties_encrypter, _ = create_provider_encrypter( - tenant_id=tenant_id, - config=provider_controller.get_properties_schema(), - cache=NoOpProviderCredentialCache(), - ) - subscription.properties = dict(properties_encrypter.encrypt(dict(new_subscription.properties))) - - # Update expiration timestamp - if new_subscription.expires_at is not None: - subscription.expires_at = new_subscription.expires_at - - # Commit the transaction - session.commit() - - # Clear subscription cache - delete_cache_for_subscription( - tenant_id=tenant_id, - provider_id=subscription.provider_id, - subscription_id=subscription.id, - ) - - except Exception as e: - # Rollback on any error - session.rollback() - logger.exception("Failed to rebuild trigger subscription", exc_info=e) - raise + # Create a new subscription with the same subscription_id and endpoint_id + new_credentials: dict[str, Any] = { + key: value if value != HIDDEN_VALUE else subscription.credentials.get(key, UNKNOWN_VALUE) + for key, value in credentials.items() + } + new_subscription: TriggerSubscriptionEntity = TriggerManager.subscribe_trigger( + tenant_id=tenant_id, + user_id=user_id, + provider_id=provider_id, + endpoint=generate_plugin_trigger_endpoint_url(subscription.endpoint_id), + parameters=parameters, + credentials=new_credentials, + credential_type=credential_type, + ) + TriggerProviderService.update_trigger_subscription( + tenant_id=tenant_id, + subscription_id=subscription.id, + name=name, + parameters=parameters, + credentials=new_credentials, + properties=new_subscription.properties, + expires_at=new_subscription.expires_at, + ) diff --git a/api/services/workflow_app_service.py b/api/services/workflow_app_service.py index 01f0c7a55a..8574d30255 100644 --- a/api/services/workflow_app_service.py +++ b/api/services/workflow_app_service.py @@ -86,12 +86,19 @@ class WorkflowAppService: # Join to workflow run for filtering when needed. if keyword: - keyword_like_val = f"%{keyword[:30].encode('unicode_escape').decode('utf-8')}%".replace(r"\u", r"\\u") + from libs.helper import escape_like_pattern + + # Escape special characters in keyword to prevent SQL injection via LIKE wildcards + escaped_keyword = escape_like_pattern(keyword[:30]) + keyword_like_val = f"%{escaped_keyword}%" keyword_conditions = [ - WorkflowRun.inputs.ilike(keyword_like_val), - WorkflowRun.outputs.ilike(keyword_like_val), + WorkflowRun.inputs.ilike(keyword_like_val, escape="\\"), + WorkflowRun.outputs.ilike(keyword_like_val, escape="\\"), # filter keyword by end user session id if created by end user role - and_(WorkflowRun.created_by_role == "end_user", EndUser.session_id.ilike(keyword_like_val)), + and_( + WorkflowRun.created_by_role == "end_user", + EndUser.session_id.ilike(keyword_like_val, escape="\\"), + ), ] # filter keyword by workflow run id diff --git a/api/services/workflow_draft_variable_service.py b/api/services/workflow_draft_variable_service.py index f299ce3baa..9407a2b3f0 100644 --- a/api/services/workflow_draft_variable_service.py +++ b/api/services/workflow_draft_variable_service.py @@ -679,6 +679,7 @@ def _batch_upsert_draft_variable( def _model_to_insertion_dict(model: WorkflowDraftVariable) -> dict[str, Any]: d: dict[str, Any] = { + "id": model.id, "app_id": model.app_id, "last_edited_at": None, "node_id": model.node_id, diff --git a/api/tasks/annotation/enable_annotation_reply_task.py b/api/tasks/annotation/enable_annotation_reply_task.py index cdc07c77a8..be1de3cdd2 100644 --- a/api/tasks/annotation/enable_annotation_reply_task.py +++ b/api/tasks/annotation/enable_annotation_reply_task.py @@ -98,7 +98,7 @@ def enable_annotation_reply_task( if annotations: for annotation in annotations: document = Document( - page_content=annotation.question, + page_content=annotation.question_text, metadata={"annotation_id": annotation.id, "app_id": app_id, "doc_id": annotation.id}, ) documents.append(document) diff --git a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py index 72469ad646..dcf31aeca7 100644 --- a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py @@ -35,6 +35,7 @@ from core.model_runtime.entities.llm_entities import LLMUsage from core.workflow.entities.pause_reason import SchedulingPause from core.workflow.enums import WorkflowExecutionStatus from core.workflow.graph_engine.entities.commands import GraphEngineCommand +from core.workflow.graph_engine.layers.base import GraphEngineLayerNotInitializedError from core.workflow.graph_events.graph import GraphRunPausedEvent from core.workflow.runtime.graph_runtime_state import GraphRuntimeState from core.workflow.runtime.graph_runtime_state_protocol import ReadOnlyGraphRuntimeState @@ -569,10 +570,10 @@ class TestPauseStatePersistenceLayerTestContainers: """Test that layer requires proper initialization before handling events.""" # Arrange layer = self._create_pause_state_persistence_layer() - # Don't initialize - graph_runtime_state should not be set + # Don't initialize - graph_runtime_state should be uninitialized event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")]) - # Act & Assert - Should raise AttributeError - with pytest.raises(AttributeError): + # Act & Assert - Should raise GraphEngineLayerNotInitializedError + with pytest.raises(GraphEngineLayerNotInitializedError): layer.on_event(event) diff --git a/api/tests/test_containers_integration_tests/services/test_annotation_service.py b/api/tests/test_containers_integration_tests/services/test_annotation_service.py index da73122cd7..5555400ca6 100644 --- a/api/tests/test_containers_integration_tests/services/test_annotation_service.py +++ b/api/tests/test_containers_integration_tests/services/test_annotation_service.py @@ -444,6 +444,78 @@ class TestAnnotationService: assert total == 1 assert unique_keyword in annotation_list[0].question or unique_keyword in annotation_list[0].content + def test_get_annotation_list_by_app_id_with_special_characters_in_keyword( + self, db_session_with_containers, mock_external_service_dependencies + ): + r""" + Test retrieval of annotation list with special characters in keyword to verify SQL injection prevention. + + This test verifies: + - Special characters (%, _, \) in keyword are properly escaped + - Search treats special characters as literal characters, not wildcards + - SQL injection via LIKE wildcards is prevented + """ + fake = Faker() + app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) + + # Create annotations with special characters in content + annotation_with_percent = { + "question": "Question with 50% discount", + "answer": "Answer about 50% discount offer", + } + AppAnnotationService.insert_app_annotation_directly(annotation_with_percent, app.id) + + annotation_with_underscore = { + "question": "Question with test_data", + "answer": "Answer about test_data value", + } + AppAnnotationService.insert_app_annotation_directly(annotation_with_underscore, app.id) + + annotation_with_backslash = { + "question": "Question with path\\to\\file", + "answer": "Answer about path\\to\\file location", + } + AppAnnotationService.insert_app_annotation_directly(annotation_with_backslash, app.id) + + # Create annotation that should NOT match (contains % but as part of different text) + annotation_no_match = { + "question": "Question with 100% different", + "answer": "Answer about 100% different content", + } + AppAnnotationService.insert_app_annotation_directly(annotation_no_match, app.id) + + # Test 1: Search with % character - should find exact match only + annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id( + app.id, page=1, limit=10, keyword="50%" + ) + assert total == 1 + assert len(annotation_list) == 1 + assert "50%" in annotation_list[0].question or "50%" in annotation_list[0].content + + # Test 2: Search with _ character - should find exact match only + annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id( + app.id, page=1, limit=10, keyword="test_data" + ) + assert total == 1 + assert len(annotation_list) == 1 + assert "test_data" in annotation_list[0].question or "test_data" in annotation_list[0].content + + # Test 3: Search with \ character - should find exact match only + annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id( + app.id, page=1, limit=10, keyword="path\\to\\file" + ) + assert total == 1 + assert len(annotation_list) == 1 + assert "path\\to\\file" in annotation_list[0].question or "path\\to\\file" in annotation_list[0].content + + # Test 4: Search with % should NOT match 100% (verifies escaping works) + annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id( + app.id, page=1, limit=10, keyword="50%" + ) + # Should only find the 50% annotation, not the 100% one + assert total == 1 + assert all("50%" in (item.question or "") or "50%" in (item.content or "") for item in annotation_list) + def test_get_annotation_list_by_app_id_app_not_found( self, db_session_with_containers, mock_external_service_dependencies ): diff --git a/api/tests/test_containers_integration_tests/services/test_app_service.py b/api/tests/test_containers_integration_tests/services/test_app_service.py index e53392bcef..745d6c97b0 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_service.py @@ -7,7 +7,9 @@ from constants.model_template import default_app_templates from models import Account from models.model import App, Site from services.account_service import AccountService, TenantService -from services.app_service import AppService + +# Delay import of AppService to avoid circular dependency +# from services.app_service import AppService class TestAppService: @@ -71,6 +73,9 @@ class TestAppService: } # Create app + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -109,6 +114,9 @@ class TestAppService: TenantService.create_owner_tenant_if_not_exist(account, name=fake.company()) tenant = account.current_tenant + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() # Test different app modes @@ -159,6 +167,9 @@ class TestAppService: "icon_background": "#45B7D1", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() created_app = app_service.create_app(tenant.id, app_args, account) @@ -194,6 +205,9 @@ class TestAppService: TenantService.create_owner_tenant_if_not_exist(account, name=fake.company()) tenant = account.current_tenant + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() # Create multiple apps @@ -245,6 +259,9 @@ class TestAppService: TenantService.create_owner_tenant_if_not_exist(account, name=fake.company()) tenant = account.current_tenant + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() # Create apps with different modes @@ -315,6 +332,9 @@ class TestAppService: TenantService.create_owner_tenant_if_not_exist(account, name=fake.company()) tenant = account.current_tenant + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() # Create an app @@ -392,6 +412,9 @@ class TestAppService: "icon_background": "#45B7D1", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -458,6 +481,9 @@ class TestAppService: "icon_background": "#45B7D1", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -508,6 +534,9 @@ class TestAppService: "icon_background": "#45B7D1", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -562,6 +591,9 @@ class TestAppService: "icon_background": "#74B9FF", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -617,6 +649,9 @@ class TestAppService: "icon_background": "#A29BFE", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -672,6 +707,9 @@ class TestAppService: "icon_background": "#FD79A8", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -720,6 +758,9 @@ class TestAppService: "icon_background": "#E17055", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -768,6 +809,9 @@ class TestAppService: "icon_background": "#00B894", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -826,6 +870,9 @@ class TestAppService: "icon_background": "#6C5CE7", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -862,6 +909,9 @@ class TestAppService: "icon_background": "#FDCB6E", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -899,6 +949,9 @@ class TestAppService: "icon_background": "#E84393", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -947,8 +1000,132 @@ class TestAppService: "icon_background": "#D63031", } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() # Attempt to create app with invalid mode with pytest.raises(ValueError, match="invalid mode value"): app_service.create_app(tenant.id, app_args, account) + + def test_get_apps_with_special_characters_in_name( + self, db_session_with_containers, mock_external_service_dependencies + ): + r""" + Test app retrieval with special characters in name search to verify SQL injection prevention. + + This test verifies: + - Special characters (%, _, \) in name search are properly escaped + - Search treats special characters as literal characters, not wildcards + - SQL injection via LIKE wildcards is prevented + """ + fake = Faker() + + # Create account and tenant first + account = AccountService.create_account( + email=fake.email(), + name=fake.name(), + interface_language="en-US", + password=fake.password(length=12), + ) + TenantService.create_owner_tenant_if_not_exist(account, name=fake.company()) + tenant = account.current_tenant + + # Import here to avoid circular dependency + from services.app_service import AppService + + app_service = AppService() + + # Create apps with special characters in names + app_with_percent = app_service.create_app( + tenant.id, + { + "name": "App with 50% discount", + "description": fake.text(max_nb_chars=100), + "mode": "chat", + "icon_type": "emoji", + "icon": "🤖", + "icon_background": "#FF6B6B", + "api_rph": 100, + "api_rpm": 10, + }, + account, + ) + + app_with_underscore = app_service.create_app( + tenant.id, + { + "name": "test_data_app", + "description": fake.text(max_nb_chars=100), + "mode": "chat", + "icon_type": "emoji", + "icon": "🤖", + "icon_background": "#FF6B6B", + "api_rph": 100, + "api_rpm": 10, + }, + account, + ) + + app_with_backslash = app_service.create_app( + tenant.id, + { + "name": "path\\to\\app", + "description": fake.text(max_nb_chars=100), + "mode": "chat", + "icon_type": "emoji", + "icon": "🤖", + "icon_background": "#FF6B6B", + "api_rph": 100, + "api_rpm": 10, + }, + account, + ) + + # Create app that should NOT match + app_no_match = app_service.create_app( + tenant.id, + { + "name": "100% different", + "description": fake.text(max_nb_chars=100), + "mode": "chat", + "icon_type": "emoji", + "icon": "🤖", + "icon_background": "#FF6B6B", + "api_rph": 100, + "api_rpm": 10, + }, + account, + ) + + # Test 1: Search with % character + args = {"name": "50%", "mode": "chat", "page": 1, "limit": 10} + paginated_apps = app_service.get_paginate_apps(account.id, tenant.id, args) + assert paginated_apps is not None + assert paginated_apps.total == 1 + assert len(paginated_apps.items) == 1 + assert paginated_apps.items[0].name == "App with 50% discount" + + # Test 2: Search with _ character + args = {"name": "test_data", "mode": "chat", "page": 1, "limit": 10} + paginated_apps = app_service.get_paginate_apps(account.id, tenant.id, args) + assert paginated_apps is not None + assert paginated_apps.total == 1 + assert len(paginated_apps.items) == 1 + assert paginated_apps.items[0].name == "test_data_app" + + # Test 3: Search with \ character + args = {"name": "path\\to\\app", "mode": "chat", "page": 1, "limit": 10} + paginated_apps = app_service.get_paginate_apps(account.id, tenant.id, args) + assert paginated_apps is not None + assert paginated_apps.total == 1 + assert len(paginated_apps.items) == 1 + assert paginated_apps.items[0].name == "path\\to\\app" + + # Test 4: Search with % should NOT match 100% (verifies escaping works) + args = {"name": "50%", "mode": "chat", "page": 1, "limit": 10} + paginated_apps = app_service.get_paginate_apps(account.id, tenant.id, args) + assert paginated_apps is not None + assert paginated_apps.total == 1 + assert all("50%" in app.name for app in paginated_apps.items) diff --git a/api/tests/test_containers_integration_tests/services/test_tag_service.py b/api/tests/test_containers_integration_tests/services/test_tag_service.py index 6732b8d558..e8c7f17e0b 100644 --- a/api/tests/test_containers_integration_tests/services/test_tag_service.py +++ b/api/tests/test_containers_integration_tests/services/test_tag_service.py @@ -1,3 +1,4 @@ +import uuid from unittest.mock import create_autospec, patch import pytest @@ -312,6 +313,85 @@ class TestTagService: result_no_match = TagService.get_tags("app", tenant.id, keyword="nonexistent") assert len(result_no_match) == 0 + def test_get_tags_with_special_characters_in_keyword( + self, db_session_with_containers, mock_external_service_dependencies + ): + r""" + Test tag retrieval with special characters in keyword to verify SQL injection prevention. + + This test verifies: + - Special characters (%, _, \) in keyword are properly escaped + - Search treats special characters as literal characters, not wildcards + - SQL injection via LIKE wildcards is prevented + """ + # Arrange: Create test data + fake = Faker() + account, tenant = self._create_test_account_and_tenant( + db_session_with_containers, mock_external_service_dependencies + ) + + from extensions.ext_database import db + + # Create tags with special characters in names + tag_with_percent = Tag( + name="50% discount", + type="app", + tenant_id=tenant.id, + created_by=account.id, + ) + tag_with_percent.id = str(uuid.uuid4()) + db.session.add(tag_with_percent) + + tag_with_underscore = Tag( + name="test_data_tag", + type="app", + tenant_id=tenant.id, + created_by=account.id, + ) + tag_with_underscore.id = str(uuid.uuid4()) + db.session.add(tag_with_underscore) + + tag_with_backslash = Tag( + name="path\\to\\tag", + type="app", + tenant_id=tenant.id, + created_by=account.id, + ) + tag_with_backslash.id = str(uuid.uuid4()) + db.session.add(tag_with_backslash) + + # Create tag that should NOT match + tag_no_match = Tag( + name="100% different", + type="app", + tenant_id=tenant.id, + created_by=account.id, + ) + tag_no_match.id = str(uuid.uuid4()) + db.session.add(tag_no_match) + + db.session.commit() + + # Act & Assert: Test 1 - Search with % character + result = TagService.get_tags("app", tenant.id, keyword="50%") + assert len(result) == 1 + assert result[0].name == "50% discount" + + # Test 2 - Search with _ character + result = TagService.get_tags("app", tenant.id, keyword="test_data") + assert len(result) == 1 + assert result[0].name == "test_data_tag" + + # Test 3 - Search with \ character + result = TagService.get_tags("app", tenant.id, keyword="path\\to\\tag") + assert len(result) == 1 + assert result[0].name == "path\\to\\tag" + + # Test 4 - Search with % should NOT match 100% (verifies escaping works) + result = TagService.get_tags("app", tenant.id, keyword="50%") + assert len(result) == 1 + assert all("50%" in item.name for item in result) + def test_get_tags_empty_result(self, db_session_with_containers, mock_external_service_dependencies): """ Test tag retrieval when no tags exist. diff --git a/api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py b/api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py index 8322b9414e..5315960d73 100644 --- a/api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py +++ b/api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py @@ -474,64 +474,6 @@ class TestTriggerProviderService: assert subscription.name == original_name assert subscription.parameters == original_parameters - def test_rebuild_trigger_subscription_unsubscribe_error_continues( - self, db_session_with_containers, mock_external_service_dependencies - ): - """ - Test that unsubscribe errors are handled gracefully and operation continues. - - This test verifies: - - Unsubscribe errors are caught and logged but don't stop the rebuild - - Rebuild continues even if unsubscribe fails - """ - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - provider_id = TriggerProviderID("test_org/test_plugin/test_provider") - credential_type = CredentialType.API_KEY - - original_credentials = {"api_key": "original-key"} - subscription = self._create_test_subscription( - db_session_with_containers, - tenant.id, - account.id, - provider_id, - credential_type, - original_credentials, - mock_external_service_dependencies, - ) - - # Make unsubscribe_trigger raise an error (should be caught and continue) - mock_external_service_dependencies["trigger_manager"].unsubscribe_trigger.side_effect = ValueError( - "Unsubscribe failed" - ) - - new_subscription_entity = TriggerSubscriptionEntity( - endpoint=subscription.endpoint_id, - parameters={}, - properties={}, - expires_at=-1, - ) - mock_external_service_dependencies["trigger_manager"].subscribe_trigger.return_value = new_subscription_entity - - # Execute rebuild - should succeed despite unsubscribe error - TriggerProviderService.rebuild_trigger_subscription( - tenant_id=tenant.id, - provider_id=provider_id, - subscription_id=subscription.id, - credentials={"api_key": "new-key"}, - parameters={}, - ) - - # Verify subscribe was still called (operation continued) - mock_external_service_dependencies["trigger_manager"].subscribe_trigger.assert_called_once() - - # Verify subscription was updated - db.session.refresh(subscription) - assert subscription.parameters == {} - def test_rebuild_trigger_subscription_subscription_not_found( self, db_session_with_containers, mock_external_service_dependencies ): @@ -558,70 +500,6 @@ class TestTriggerProviderService: parameters={}, ) - def test_rebuild_trigger_subscription_provider_not_found( - self, db_session_with_containers, mock_external_service_dependencies - ): - """ - Test error when provider is not found. - - This test verifies: - - Proper error is raised when provider doesn't exist - """ - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - provider_id = TriggerProviderID("non_existent_org/non_existent_plugin/non_existent_provider") - - # Make get_trigger_provider return None - mock_external_service_dependencies["trigger_manager"].get_trigger_provider.return_value = None - - with pytest.raises(ValueError, match="Provider.*not found"): - TriggerProviderService.rebuild_trigger_subscription( - tenant_id=tenant.id, - provider_id=provider_id, - subscription_id=fake.uuid4(), - credentials={}, - parameters={}, - ) - - def test_rebuild_trigger_subscription_unsupported_credential_type( - self, db_session_with_containers, mock_external_service_dependencies - ): - """ - Test error when credential type is not supported for rebuild. - - This test verifies: - - Proper error is raised for unsupported credential types (not OAUTH2 or API_KEY) - """ - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - provider_id = TriggerProviderID("test_org/test_plugin/test_provider") - credential_type = CredentialType.UNAUTHORIZED # Not supported - - subscription = self._create_test_subscription( - db_session_with_containers, - tenant.id, - account.id, - provider_id, - credential_type, - {}, - mock_external_service_dependencies, - ) - - with pytest.raises(ValueError, match="Credential type not supported for rebuild"): - TriggerProviderService.rebuild_trigger_subscription( - tenant_id=tenant.id, - provider_id=provider_id, - subscription_id=subscription.id, - credentials={}, - parameters={}, - ) - def test_rebuild_trigger_subscription_name_uniqueness_check( self, db_session_with_containers, mock_external_service_dependencies ): diff --git a/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py index 7b95944bbe..040fb826e1 100644 --- a/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py +++ b/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py @@ -10,7 +10,9 @@ from core.workflow.entities.workflow_execution import WorkflowExecutionStatus from models import EndUser, Workflow, WorkflowAppLog, WorkflowRun from models.enums import CreatorUserRole from services.account_service import AccountService, TenantService -from services.app_service import AppService + +# Delay import of AppService to avoid circular dependency +# from services.app_service import AppService from services.workflow_app_service import WorkflowAppService @@ -86,6 +88,9 @@ class TestWorkflowAppService: "api_rpm": 10, } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -147,6 +152,9 @@ class TestWorkflowAppService: "api_rpm": 10, } + # Import here to avoid circular dependency + from services.app_service import AppService + app_service = AppService() app = app_service.create_app(tenant.id, app_args, account) @@ -308,6 +316,156 @@ class TestWorkflowAppService: assert result_no_match["total"] == 0 assert len(result_no_match["data"]) == 0 + def test_get_paginate_workflow_app_logs_with_special_characters_in_keyword( + self, db_session_with_containers, mock_external_service_dependencies + ): + r""" + Test workflow app logs pagination with special characters in keyword to verify SQL injection prevention. + + This test verifies: + - Special characters (%, _) in keyword are properly escaped + - Search treats special characters as literal characters, not wildcards + - SQL injection via LIKE wildcards is prevented + """ + # Arrange: Create test data + fake = Faker() + app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) + workflow, _, _ = self._create_test_workflow_data(db_session_with_containers, app, account) + + from extensions.ext_database import db + + service = WorkflowAppService() + + # Test 1: Search with % character + workflow_run_1 = WorkflowRun( + id=str(uuid.uuid4()), + tenant_id=app.tenant_id, + app_id=app.id, + workflow_id=workflow.id, + type="workflow", + triggered_from="app-run", + version="1.0.0", + graph=json.dumps({"nodes": [], "edges": []}), + status="succeeded", + inputs=json.dumps({"search_term": "50% discount", "input2": "other_value"}), + outputs=json.dumps({"result": "50% discount applied", "status": "success"}), + created_by_role=CreatorUserRole.ACCOUNT, + created_by=account.id, + created_at=datetime.now(UTC), + ) + db.session.add(workflow_run_1) + db.session.flush() + + workflow_app_log_1 = WorkflowAppLog( + tenant_id=app.tenant_id, + app_id=app.id, + workflow_id=workflow.id, + workflow_run_id=workflow_run_1.id, + created_from="service-api", + created_by_role=CreatorUserRole.ACCOUNT, + created_by=account.id, + ) + workflow_app_log_1.id = str(uuid.uuid4()) + workflow_app_log_1.created_at = datetime.now(UTC) + db.session.add(workflow_app_log_1) + db.session.commit() + + result = service.get_paginate_workflow_app_logs( + session=db_session_with_containers, app_model=app, keyword="50%", page=1, limit=20 + ) + # Should find the workflow_run_1 entry + assert result["total"] >= 1 + assert len(result["data"]) >= 1 + assert any(log.workflow_run_id == workflow_run_1.id for log in result["data"]) + + # Test 2: Search with _ character + workflow_run_2 = WorkflowRun( + id=str(uuid.uuid4()), + tenant_id=app.tenant_id, + app_id=app.id, + workflow_id=workflow.id, + type="workflow", + triggered_from="app-run", + version="1.0.0", + graph=json.dumps({"nodes": [], "edges": []}), + status="succeeded", + inputs=json.dumps({"search_term": "test_data_value", "input2": "other_value"}), + outputs=json.dumps({"result": "test_data_value found", "status": "success"}), + created_by_role=CreatorUserRole.ACCOUNT, + created_by=account.id, + created_at=datetime.now(UTC), + ) + db.session.add(workflow_run_2) + db.session.flush() + + workflow_app_log_2 = WorkflowAppLog( + tenant_id=app.tenant_id, + app_id=app.id, + workflow_id=workflow.id, + workflow_run_id=workflow_run_2.id, + created_from="service-api", + created_by_role=CreatorUserRole.ACCOUNT, + created_by=account.id, + ) + workflow_app_log_2.id = str(uuid.uuid4()) + workflow_app_log_2.created_at = datetime.now(UTC) + db.session.add(workflow_app_log_2) + db.session.commit() + + result = service.get_paginate_workflow_app_logs( + session=db_session_with_containers, app_model=app, keyword="test_data", page=1, limit=20 + ) + # Should find the workflow_run_2 entry + assert result["total"] >= 1 + assert len(result["data"]) >= 1 + assert any(log.workflow_run_id == workflow_run_2.id for log in result["data"]) + + # Test 3: Search with % should NOT match 100% (verifies escaping works correctly) + workflow_run_4 = WorkflowRun( + id=str(uuid.uuid4()), + tenant_id=app.tenant_id, + app_id=app.id, + workflow_id=workflow.id, + type="workflow", + triggered_from="app-run", + version="1.0.0", + graph=json.dumps({"nodes": [], "edges": []}), + status="succeeded", + inputs=json.dumps({"search_term": "100% different", "input2": "other_value"}), + outputs=json.dumps({"result": "100% different result", "status": "success"}), + created_by_role=CreatorUserRole.ACCOUNT, + created_by=account.id, + created_at=datetime.now(UTC), + ) + db.session.add(workflow_run_4) + db.session.flush() + + workflow_app_log_4 = WorkflowAppLog( + tenant_id=app.tenant_id, + app_id=app.id, + workflow_id=workflow.id, + workflow_run_id=workflow_run_4.id, + created_from="service-api", + created_by_role=CreatorUserRole.ACCOUNT, + created_by=account.id, + ) + workflow_app_log_4.id = str(uuid.uuid4()) + workflow_app_log_4.created_at = datetime.now(UTC) + db.session.add(workflow_app_log_4) + db.session.commit() + + result = service.get_paginate_workflow_app_logs( + session=db_session_with_containers, app_model=app, keyword="50%", page=1, limit=20 + ) + # Should only find the 50% entry (workflow_run_1), not the 100% entry (workflow_run_4) + # This verifies that escaping works correctly - 50% should not match 100% + assert result["total"] >= 1 + assert len(result["data"]) >= 1 + # Verify that we found workflow_run_1 (50% discount) but not workflow_run_4 (100% different) + found_run_ids = [log.workflow_run_id for log in result["data"]] + assert workflow_run_1.id in found_run_ids + assert workflow_run_4.id not in found_run_ids + def test_get_paginate_workflow_app_logs_with_status_filter( self, db_session_with_containers, mock_external_service_dependencies ): diff --git a/api/tests/unit_tests/controllers/__init__.py b/api/tests/unit_tests/controllers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/controllers/console/__init__.py b/api/tests/unit_tests/controllers/console/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/controllers/console/app/test_app_response_models.py b/api/tests/unit_tests/controllers/console/app/test_app_response_models.py new file mode 100644 index 0000000000..40eb59a8f4 --- /dev/null +++ b/api/tests/unit_tests/controllers/console/app/test_app_response_models.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import builtins +import sys +from datetime import datetime +from importlib import util +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest +from flask.views import MethodView + +# kombu references MethodView as a global when importing celery/kombu pools. +if not hasattr(builtins, "MethodView"): + builtins.MethodView = MethodView # type: ignore[attr-defined] + + +def _load_app_module(): + module_name = "controllers.console.app.app" + if module_name in sys.modules: + return sys.modules[module_name] + + root = Path(__file__).resolve().parents[5] + module_path = root / "controllers" / "console" / "app" / "app.py" + + class _StubNamespace: + def __init__(self): + self.models: dict[str, Any] = {} + self.payload = None + + def schema_model(self, name, schema): + self.models[name] = schema + + def _decorator(self, obj): + return obj + + def doc(self, *args, **kwargs): + return self._decorator + + def expect(self, *args, **kwargs): + return self._decorator + + def response(self, *args, **kwargs): + return self._decorator + + def route(self, *args, **kwargs): + def decorator(obj): + return obj + + return decorator + + stub_namespace = _StubNamespace() + + original_console = sys.modules.get("controllers.console") + original_app_pkg = sys.modules.get("controllers.console.app") + stubbed_modules: list[tuple[str, ModuleType | None]] = [] + + console_module = ModuleType("controllers.console") + console_module.__path__ = [str(root / "controllers" / "console")] + console_module.console_ns = stub_namespace + console_module.api = None + console_module.bp = None + sys.modules["controllers.console"] = console_module + + app_package = ModuleType("controllers.console.app") + app_package.__path__ = [str(root / "controllers" / "console" / "app")] + sys.modules["controllers.console.app"] = app_package + console_module.app = app_package + + def _stub_module(name: str, attrs: dict[str, Any]): + original = sys.modules.get(name) + module = ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + sys.modules[name] = module + stubbed_modules.append((name, original)) + + class _OpsTraceManager: + @staticmethod + def get_app_tracing_config(app_id: str) -> dict[str, Any]: + return {} + + @staticmethod + def update_app_tracing_config(app_id: str, **kwargs) -> None: + return None + + _stub_module( + "core.ops.ops_trace_manager", + { + "OpsTraceManager": _OpsTraceManager, + "TraceQueueManager": object, + "TraceTask": object, + }, + ) + + spec = util.spec_from_file_location(module_name, module_path) + module = util.module_from_spec(spec) + sys.modules[module_name] = module + + try: + assert spec.loader is not None + spec.loader.exec_module(module) + finally: + for name, original in reversed(stubbed_modules): + if original is not None: + sys.modules[name] = original + else: + sys.modules.pop(name, None) + if original_console is not None: + sys.modules["controllers.console"] = original_console + else: + sys.modules.pop("controllers.console", None) + if original_app_pkg is not None: + sys.modules["controllers.console.app"] = original_app_pkg + else: + sys.modules.pop("controllers.console.app", None) + + return module + + +_app_module = _load_app_module() +AppDetailWithSite = _app_module.AppDetailWithSite +AppPagination = _app_module.AppPagination +AppPartial = _app_module.AppPartial + + +@pytest.fixture(autouse=True) +def patch_signed_url(monkeypatch): + """Ensure icon URL generation uses a deterministic helper for tests.""" + + def _fake_signed_url(key: str | None) -> str | None: + if not key: + return None + return f"signed:{key}" + + monkeypatch.setattr(_app_module.file_helpers, "get_signed_file_url", _fake_signed_url) + + +def _ts(hour: int = 12) -> datetime: + return datetime(2024, 1, 1, hour, 0, 0) + + +def _dummy_model_config(): + return SimpleNamespace( + model_dict={"provider": "openai", "name": "gpt-4o"}, + pre_prompt="hello", + created_by="config-author", + created_at=_ts(9), + updated_by="config-editor", + updated_at=_ts(10), + ) + + +def _dummy_workflow(): + return SimpleNamespace( + id="wf-1", + created_by="workflow-author", + created_at=_ts(8), + updated_by="workflow-editor", + updated_at=_ts(9), + ) + + +def test_app_partial_serialization_uses_aliases(): + created_at = _ts() + app_obj = SimpleNamespace( + id="app-1", + name="My App", + desc_or_prompt="Prompt snippet", + mode_compatible_with_agent="chat", + icon_type="image", + icon="icon-key", + icon_background="#fff", + app_model_config=_dummy_model_config(), + workflow=_dummy_workflow(), + created_by="creator", + created_at=created_at, + updated_by="editor", + updated_at=created_at, + tags=[SimpleNamespace(id="tag-1", name="Utilities", type="app")], + access_mode="private", + create_user_name="Creator", + author_name="Author", + has_draft_trigger=True, + ) + + serialized = AppPartial.model_validate(app_obj, from_attributes=True).model_dump(mode="json") + + assert serialized["description"] == "Prompt snippet" + assert serialized["mode"] == "chat" + assert serialized["icon_url"] == "signed:icon-key" + assert serialized["created_at"] == int(created_at.timestamp()) + assert serialized["updated_at"] == int(created_at.timestamp()) + assert serialized["model_config"]["model"] == {"provider": "openai", "name": "gpt-4o"} + assert serialized["workflow"]["id"] == "wf-1" + assert serialized["tags"][0]["name"] == "Utilities" + + +def test_app_detail_with_site_includes_nested_serialization(): + timestamp = _ts(14) + site = SimpleNamespace( + code="site-code", + title="Public Site", + icon_type="image", + icon="site-icon", + created_at=timestamp, + updated_at=timestamp, + ) + app_obj = SimpleNamespace( + id="app-2", + name="Detailed App", + description="Desc", + mode_compatible_with_agent="advanced-chat", + icon_type="image", + icon="detail-icon", + icon_background="#123456", + enable_site=True, + enable_api=True, + app_model_config={ + "opening_statement": "hi", + "model": {"provider": "openai", "name": "gpt-4o"}, + "retriever_resource": {"enabled": True}, + }, + workflow=_dummy_workflow(), + tracing={"enabled": True}, + use_icon_as_answer_icon=True, + created_by="creator", + created_at=timestamp, + updated_by="editor", + updated_at=timestamp, + access_mode="public", + tags=[SimpleNamespace(id="tag-2", name="Prod", type="app")], + api_base_url="https://api.example.com/v1", + max_active_requests=5, + deleted_tools=[{"type": "api", "tool_name": "search", "provider_id": "prov"}], + site=site, + ) + + serialized = AppDetailWithSite.model_validate(app_obj, from_attributes=True).model_dump(mode="json") + + assert serialized["icon_url"] == "signed:detail-icon" + assert serialized["model_config"]["retriever_resource"] == {"enabled": True} + assert serialized["deleted_tools"][0]["tool_name"] == "search" + assert serialized["site"]["icon_url"] == "signed:site-icon" + assert serialized["site"]["created_at"] == int(timestamp.timestamp()) + + +def test_app_pagination_aliases_per_page_and_has_next(): + item_one = SimpleNamespace( + id="app-10", + name="Paginated One", + desc_or_prompt="Summary", + mode_compatible_with_agent="chat", + icon_type="image", + icon="first-icon", + created_at=_ts(15), + updated_at=_ts(15), + ) + item_two = SimpleNamespace( + id="app-11", + name="Paginated Two", + desc_or_prompt="Summary", + mode_compatible_with_agent="agent-chat", + icon_type="emoji", + icon="🙂", + created_at=_ts(16), + updated_at=_ts(16), + ) + pagination = SimpleNamespace( + page=2, + per_page=10, + total=50, + has_next=True, + items=[item_one, item_two], + ) + + serialized = AppPagination.model_validate(pagination, from_attributes=True).model_dump(mode="json") + + assert serialized["page"] == 2 + assert serialized["limit"] == 10 + assert serialized["has_more"] is True + assert len(serialized["data"]) == 2 + assert serialized["data"][0]["icon_url"] == "signed:first-icon" + assert serialized["data"][1]["icon_url"] is None diff --git a/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py new file mode 100644 index 0000000000..f8dd98fdb2 --- /dev/null +++ b/api/tests/unit_tests/controllers/console/test_document_detail_api_data_source_info.py @@ -0,0 +1,145 @@ +""" +Test for document detail API data_source_info serialization fix. + +This test verifies that the document detail API returns both data_source_info +and data_source_detail_dict for all data_source_type values, including "local_file". +""" + +import json +from typing import Generic, Literal, NotRequired, TypedDict, TypeVar, Union + +from models.dataset import Document + + +class LocalFileInfo(TypedDict): + file_path: str + size: int + created_at: NotRequired[str] + + +class UploadFileInfo(TypedDict): + upload_file_id: str + + +class NotionImportInfo(TypedDict): + notion_page_id: str + workspace_id: str + + +class WebsiteCrawlInfo(TypedDict): + url: str + job_id: str + + +RawInfo = Union[LocalFileInfo, UploadFileInfo, NotionImportInfo, WebsiteCrawlInfo] +T_type = TypeVar("T_type", bound=str) +T_info = TypeVar("T_info", bound=Union[LocalFileInfo, UploadFileInfo, NotionImportInfo, WebsiteCrawlInfo]) + + +class Case(TypedDict, Generic[T_type, T_info]): + data_source_type: T_type + data_source_info: str + expected_raw: T_info + + +LocalFileCase = Case[Literal["local_file"], LocalFileInfo] +UploadFileCase = Case[Literal["upload_file"], UploadFileInfo] +NotionImportCase = Case[Literal["notion_import"], NotionImportInfo] +WebsiteCrawlCase = Case[Literal["website_crawl"], WebsiteCrawlInfo] + +AnyCase = Union[LocalFileCase, UploadFileCase, NotionImportCase, WebsiteCrawlCase] + + +case_1: LocalFileCase = { + "data_source_type": "local_file", + "data_source_info": json.dumps({"file_path": "/tmp/test.txt", "size": 1024}), + "expected_raw": {"file_path": "/tmp/test.txt", "size": 1024}, +} + + +# ERROR: Expected LocalFileInfo, but got WebsiteCrawlInfo +case_2: LocalFileCase = { + "data_source_type": "local_file", + "data_source_info": "...", + "expected_raw": {"file_path": "https://google.com", "size": 123}, +} + +cases: list[AnyCase] = [case_1] + + +class TestDocumentDetailDataSourceInfo: + """Test cases for document detail API data_source_info serialization.""" + + def test_data_source_info_dict_returns_raw_data(self): + """Test that data_source_info_dict returns raw JSON data for all data_source_type values.""" + # Test data for different data_source_type values + for case in cases: + document = Document( + data_source_type=case["data_source_type"], + data_source_info=case["data_source_info"], + ) + + # Test data_source_info_dict (raw data) + raw_result = document.data_source_info_dict + assert raw_result == case["expected_raw"], f"Failed for {case['data_source_type']}" + + # Verify raw_result is always a valid dict + assert isinstance(raw_result, dict) + + def test_local_file_data_source_info_without_db_context(self): + """Test that local_file type data_source_info_dict works without database context.""" + test_data: LocalFileInfo = { + "file_path": "/local/path/document.txt", + "size": 512, + "created_at": "2024-01-01T00:00:00Z", + } + + document = Document( + data_source_type="local_file", + data_source_info=json.dumps(test_data), + ) + + # data_source_info_dict should return the raw data (this doesn't need DB context) + raw_data = document.data_source_info_dict + assert raw_data == test_data + assert isinstance(raw_data, dict) + + # Verify the data contains expected keys for pipeline mode + assert "file_path" in raw_data + assert "size" in raw_data + + def test_notion_and_website_crawl_data_source_detail(self): + """Test that notion_import and website_crawl return raw data in data_source_detail_dict.""" + # Test notion_import + notion_data: NotionImportInfo = {"notion_page_id": "page-123", "workspace_id": "ws-456"} + document = Document( + data_source_type="notion_import", + data_source_info=json.dumps(notion_data), + ) + + # data_source_detail_dict should return raw data for notion_import + detail_result = document.data_source_detail_dict + assert detail_result == notion_data + + # Test website_crawl + website_data: WebsiteCrawlInfo = {"url": "https://example.com", "job_id": "job-789"} + document = Document( + data_source_type="website_crawl", + data_source_info=json.dumps(website_data), + ) + + # data_source_detail_dict should return raw data for website_crawl + detail_result = document.data_source_detail_dict + assert detail_result == website_data + + def test_local_file_data_source_detail_dict_without_db(self): + """Test that local_file returns empty data_source_detail_dict (this doesn't need DB context).""" + # Test local_file - this should work without database context since it returns {} early + document = Document( + data_source_type="local_file", + data_source_info=json.dumps({"file_path": "/tmp/test.txt"}), + ) + + # Should return empty dict for local_file type (handled in the model) + detail_result = document.data_source_detail_dict + assert detail_result == {} diff --git a/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py new file mode 100644 index 0000000000..b6e8cc9c8e --- /dev/null +++ b/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py @@ -0,0 +1,144 @@ +from collections.abc import Sequence +from datetime import datetime +from unittest.mock import Mock + +from core.app.layers.conversation_variable_persist_layer import ConversationVariablePersistenceLayer +from core.variables import StringVariable +from core.variables.segments import Segment +from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus +from core.workflow.graph_engine.protocols.command_channel import CommandChannel +from core.workflow.graph_events.node import NodeRunSucceededEvent +from core.workflow.node_events import NodeRunResult +from core.workflow.nodes.variable_assigner.common import helpers as common_helpers +from core.workflow.runtime.graph_runtime_state_protocol import ReadOnlyGraphRuntimeState +from core.workflow.system_variable import SystemVariable + + +class MockReadOnlyVariablePool: + def __init__(self, variables: dict[tuple[str, str], Segment] | None = None) -> None: + self._variables = variables or {} + + def get(self, selector: Sequence[str]) -> Segment | None: + if len(selector) < 2: + return None + return self._variables.get((selector[0], selector[1])) + + def get_all_by_node(self, node_id: str) -> dict[str, object]: + return {key: value for (nid, key), value in self._variables.items() if nid == node_id} + + def get_by_prefix(self, prefix: str) -> dict[str, object]: + return {key: value for (nid, key), value in self._variables.items() if nid == prefix} + + +def _build_graph_runtime_state( + variable_pool: MockReadOnlyVariablePool, + conversation_id: str | None = None, +) -> ReadOnlyGraphRuntimeState: + graph_runtime_state = Mock(spec=ReadOnlyGraphRuntimeState) + graph_runtime_state.variable_pool = variable_pool + graph_runtime_state.system_variable = SystemVariable(conversation_id=conversation_id).as_view() + return graph_runtime_state + + +def _build_node_run_succeeded_event( + *, + node_type: NodeType, + outputs: dict[str, object] | None = None, + process_data: dict[str, object] | None = None, +) -> NodeRunSucceededEvent: + return NodeRunSucceededEvent( + id="node-exec-id", + node_id="assigner", + node_type=node_type, + start_at=datetime.utcnow(), + node_run_result=NodeRunResult( + status=WorkflowNodeExecutionStatus.SUCCEEDED, + outputs=outputs or {}, + process_data=process_data or {}, + ), + ) + + +def test_persists_conversation_variables_from_assigner_output(): + conversation_id = "conv-123" + variable = StringVariable( + id="var-1", + name="name", + value="updated", + selector=[CONVERSATION_VARIABLE_NODE_ID, "name"], + ) + process_data = common_helpers.set_updated_variables( + {}, [common_helpers.variable_to_processed_data(variable.selector, variable)] + ) + + variable_pool = MockReadOnlyVariablePool({(CONVERSATION_VARIABLE_NODE_ID, "name"): variable}) + + updater = Mock() + layer = ConversationVariablePersistenceLayer(updater) + layer.initialize(_build_graph_runtime_state(variable_pool, conversation_id), Mock(spec=CommandChannel)) + + event = _build_node_run_succeeded_event(node_type=NodeType.VARIABLE_ASSIGNER, process_data=process_data) + layer.on_event(event) + + updater.update.assert_called_once_with(conversation_id=conversation_id, variable=variable) + updater.flush.assert_called_once() + + +def test_skips_when_outputs_missing(): + conversation_id = "conv-456" + variable = StringVariable( + id="var-2", + name="name", + value="updated", + selector=[CONVERSATION_VARIABLE_NODE_ID, "name"], + ) + + variable_pool = MockReadOnlyVariablePool({(CONVERSATION_VARIABLE_NODE_ID, "name"): variable}) + + updater = Mock() + layer = ConversationVariablePersistenceLayer(updater) + layer.initialize(_build_graph_runtime_state(variable_pool, conversation_id), Mock(spec=CommandChannel)) + + event = _build_node_run_succeeded_event(node_type=NodeType.VARIABLE_ASSIGNER) + layer.on_event(event) + + updater.update.assert_not_called() + updater.flush.assert_not_called() + + +def test_skips_non_assigner_nodes(): + updater = Mock() + layer = ConversationVariablePersistenceLayer(updater) + layer.initialize(_build_graph_runtime_state(MockReadOnlyVariablePool()), Mock(spec=CommandChannel)) + + event = _build_node_run_succeeded_event(node_type=NodeType.LLM) + layer.on_event(event) + + updater.update.assert_not_called() + updater.flush.assert_not_called() + + +def test_skips_non_conversation_variables(): + conversation_id = "conv-789" + non_conversation_variable = StringVariable( + id="var-3", + name="name", + value="updated", + selector=["environment", "name"], + ) + process_data = common_helpers.set_updated_variables( + {}, [common_helpers.variable_to_processed_data(non_conversation_variable.selector, non_conversation_variable)] + ) + + variable_pool = MockReadOnlyVariablePool() + + updater = Mock() + layer = ConversationVariablePersistenceLayer(updater) + layer.initialize(_build_graph_runtime_state(variable_pool, conversation_id), Mock(spec=CommandChannel)) + + event = _build_node_run_succeeded_event(node_type=NodeType.VARIABLE_ASSIGNER, process_data=process_data) + layer.on_event(event) + + updater.update.assert_not_called() + updater.flush.assert_not_called() diff --git a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py index 534420f21e..1d885f6b2e 100644 --- a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py @@ -1,4 +1,5 @@ import json +from collections.abc import Sequence from time import time from unittest.mock import Mock @@ -15,6 +16,7 @@ from core.app.layers.pause_state_persist_layer import ( from core.variables.segments import Segment from core.workflow.entities.pause_reason import SchedulingPause from core.workflow.graph_engine.entities.commands import GraphEngineCommand +from core.workflow.graph_engine.layers.base import GraphEngineLayerNotInitializedError from core.workflow.graph_events.graph import ( GraphRunFailedEvent, GraphRunPausedEvent, @@ -66,8 +68,10 @@ class MockReadOnlyVariablePool: def __init__(self, variables: dict[tuple[str, str], object] | None = None): self._variables = variables or {} - def get(self, node_id: str, variable_key: str) -> Segment | None: - value = self._variables.get((node_id, variable_key)) + def get(self, selector: Sequence[str]) -> Segment | None: + if len(selector) < 2: + return None + value = self._variables.get((selector[0], selector[1])) if value is None: return None mock_segment = Mock(spec=Segment) @@ -209,8 +213,9 @@ class TestPauseStatePersistenceLayer: assert layer._session_maker is session_factory assert layer._state_owner_user_id == state_owner_user_id - assert not hasattr(layer, "graph_runtime_state") - assert not hasattr(layer, "command_channel") + with pytest.raises(GraphEngineLayerNotInitializedError): + _ = layer.graph_runtime_state + assert layer.command_channel is None def test_initialize_sets_dependencies(self): session_factory = Mock(name="session_factory") @@ -295,7 +300,7 @@ class TestPauseStatePersistenceLayer: mock_factory.assert_not_called() mock_repo.create_workflow_pause.assert_not_called() - def test_on_event_raises_attribute_error_when_graph_runtime_state_is_none(self): + def test_on_event_raises_when_graph_runtime_state_is_uninitialized(self): session_factory = Mock(name="session_factory") layer = PauseStatePersistenceLayer( session_factory=session_factory, @@ -305,7 +310,7 @@ class TestPauseStatePersistenceLayer: event = TestDataFactory.create_graph_run_paused_event() - with pytest.raises(AttributeError): + with pytest.raises(GraphEngineLayerNotInitializedError): layer.on_event(event) def test_on_event_asserts_when_workflow_execution_id_missing(self, monkeypatch: pytest.MonkeyPatch): diff --git a/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py index 3203aab8c3..f9e59a5f05 100644 --- a/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py +++ b/api/tests/unit_tests/core/rag/extractor/test_word_extractor.py @@ -1,8 +1,12 @@ """Primarily used for testing merged cell scenarios""" +import os +import tempfile from types import SimpleNamespace from docx import Document +from docx.oxml import OxmlElement +from docx.oxml.ns import qn import core.rag.extractor.word_extractor as we from core.rag.extractor.word_extractor import WordExtractor @@ -165,3 +169,110 @@ def test_extract_images_from_docx_uses_internal_files_url(): dify_config.FILES_URL = original_files_url if original_internal_files_url is not None: dify_config.INTERNAL_FILES_URL = original_internal_files_url + + +def test_extract_hyperlinks(monkeypatch): + # Mock db and storage to avoid issues during image extraction (even if no images are present) + monkeypatch.setattr(we, "storage", SimpleNamespace(save=lambda k, d: None)) + db_stub = SimpleNamespace(session=SimpleNamespace(add=lambda o: None, commit=lambda: None)) + monkeypatch.setattr(we, "db", db_stub) + monkeypatch.setattr(we.dify_config, "FILES_URL", "http://files.local", raising=False) + monkeypatch.setattr(we.dify_config, "STORAGE_TYPE", "local", raising=False) + + doc = Document() + p = doc.add_paragraph("Visit ") + + # Adding a hyperlink manually + r_id = "rId99" + hyperlink = OxmlElement("w:hyperlink") + hyperlink.set(qn("r:id"), r_id) + + new_run = OxmlElement("w:r") + t = OxmlElement("w:t") + t.text = "Dify" + new_run.append(t) + hyperlink.append(new_run) + p._p.append(hyperlink) + + # Add relationship to the part + doc.part.rels.add_relationship( + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink", + "https://dify.ai", + r_id, + is_external=True, + ) + + with tempfile.NamedTemporaryFile(suffix=".docx", delete=False) as tmp: + doc.save(tmp.name) + tmp_path = tmp.name + + try: + extractor = WordExtractor(tmp_path, "tenant_id", "user_id") + docs = extractor.extract() + # Verify modern hyperlink extraction + assert "Visit[Dify](https://dify.ai)" in docs[0].page_content + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + + +def test_extract_legacy_hyperlinks(monkeypatch): + # Mock db and storage + monkeypatch.setattr(we, "storage", SimpleNamespace(save=lambda k, d: None)) + db_stub = SimpleNamespace(session=SimpleNamespace(add=lambda o: None, commit=lambda: None)) + monkeypatch.setattr(we, "db", db_stub) + monkeypatch.setattr(we.dify_config, "FILES_URL", "http://files.local", raising=False) + monkeypatch.setattr(we.dify_config, "STORAGE_TYPE", "local", raising=False) + + doc = Document() + p = doc.add_paragraph() + + # Construct a legacy HYPERLINK field: + # 1. w:fldChar (begin) + # 2. w:instrText (HYPERLINK "http://example.com") + # 3. w:fldChar (separate) + # 4. w:r (visible text "Example") + # 5. w:fldChar (end) + + run1 = OxmlElement("w:r") + fldCharBegin = OxmlElement("w:fldChar") + fldCharBegin.set(qn("w:fldCharType"), "begin") + run1.append(fldCharBegin) + p._p.append(run1) + + run2 = OxmlElement("w:r") + instrText = OxmlElement("w:instrText") + instrText.text = ' HYPERLINK "http://example.com" ' + run2.append(instrText) + p._p.append(run2) + + run3 = OxmlElement("w:r") + fldCharSep = OxmlElement("w:fldChar") + fldCharSep.set(qn("w:fldCharType"), "separate") + run3.append(fldCharSep) + p._p.append(run3) + + run4 = OxmlElement("w:r") + t4 = OxmlElement("w:t") + t4.text = "Example" + run4.append(t4) + p._p.append(run4) + + run5 = OxmlElement("w:r") + fldCharEnd = OxmlElement("w:fldChar") + fldCharEnd.set(qn("w:fldCharType"), "end") + run5.append(fldCharEnd) + p._p.append(run5) + + with tempfile.NamedTemporaryFile(suffix=".docx", delete=False) as tmp: + doc.save(tmp.name) + tmp_path = tmp.name + + try: + extractor = WordExtractor(tmp_path, "tenant_id", "user_id") + docs = extractor.extract() + # Verify legacy hyperlink extraction + assert "[Example](http://example.com)" in docs[0].page_content + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) diff --git a/api/tests/unit_tests/core/rag/retrieval/test_knowledge_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_knowledge_retrieval.py new file mode 100644 index 0000000000..5f461d53ae --- /dev/null +++ b/api/tests/unit_tests/core/rag/retrieval/test_knowledge_retrieval.py @@ -0,0 +1,113 @@ +import threading +from unittest.mock import Mock, patch +from uuid import uuid4 + +import pytest +from flask import Flask, current_app + +from core.rag.models.document import Document +from core.rag.retrieval.dataset_retrieval import DatasetRetrieval +from models.dataset import Dataset + + +class TestRetrievalService: + @pytest.fixture + def mock_dataset(self) -> Dataset: + dataset = Mock(spec=Dataset) + dataset.id = str(uuid4()) + dataset.tenant_id = str(uuid4()) + dataset.name = "test_dataset" + dataset.indexing_technique = "high_quality" + dataset.provider = "dify" + return dataset + + def test_multiple_retrieve_reranking_with_app_context(self, mock_dataset): + """ + Repro test for current bug: + reranking runs after `with flask_app.app_context():` exits. + `_multiple_retrieve_thread` catches exceptions and stores them into `thread_exceptions`, + so we must assert from that list (not from an outer try/except). + """ + dataset_retrieval = DatasetRetrieval() + flask_app = Flask(__name__) + tenant_id = str(uuid4()) + + # second dataset to ensure dataset_count > 1 reranking branch + secondary_dataset = Mock(spec=Dataset) + secondary_dataset.id = str(uuid4()) + secondary_dataset.provider = "dify" + secondary_dataset.indexing_technique = "high_quality" + + # retriever returns 1 doc into internal list (all_documents_item) + document = Document( + page_content="Context aware doc", + metadata={ + "doc_id": "doc1", + "score": 0.95, + "document_id": str(uuid4()), + "dataset_id": mock_dataset.id, + }, + provider="dify", + ) + + def fake_retriever( + flask_app, dataset_id, query, top_k, all_documents, document_ids_filter, metadata_condition, attachment_ids + ): + all_documents.append(document) + + called = {"init": 0, "invoke": 0} + + class ContextRequiredPostProcessor: + def __init__(self, *args, **kwargs): + called["init"] += 1 + # will raise RuntimeError if no Flask app context exists + _ = current_app.name + + def invoke(self, *args, **kwargs): + called["invoke"] += 1 + _ = current_app.name + return kwargs.get("documents") or args[1] + + # output list from _multiple_retrieve_thread + all_documents: list[Document] = [] + + # IMPORTANT: _multiple_retrieve_thread swallows exceptions and appends them here + thread_exceptions: list[Exception] = [] + + def target(): + with patch.object(dataset_retrieval, "_retriever", side_effect=fake_retriever): + with patch( + "core.rag.retrieval.dataset_retrieval.DataPostProcessor", + ContextRequiredPostProcessor, + ): + dataset_retrieval._multiple_retrieve_thread( + flask_app=flask_app, + available_datasets=[mock_dataset, secondary_dataset], + metadata_condition=None, + metadata_filter_document_ids=None, + all_documents=all_documents, + tenant_id=tenant_id, + reranking_enable=True, + reranking_mode="reranking_model", + reranking_model={ + "reranking_provider_name": "cohere", + "reranking_model_name": "rerank-v2", + }, + weights=None, + top_k=3, + score_threshold=0.0, + query="test query", + attachment_id=None, + dataset_count=2, # force reranking branch + thread_exceptions=thread_exceptions, # ✅ key + ) + + t = threading.Thread(target=target) + t.start() + t.join() + + # Ensure reranking branch was actually executed + assert called["init"] >= 1, "DataPostProcessor was never constructed; reranking branch may not have run." + + # Current buggy code should record an exception (not raise it) + assert not thread_exceptions, thread_exceptions diff --git a/api/tests/unit_tests/core/workflow/graph_engine/command_channels/test_redis_channel.py b/api/tests/unit_tests/core/workflow/graph_engine/command_channels/test_redis_channel.py index 8677325d4e..f33fd0deeb 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/command_channels/test_redis_channel.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/command_channels/test_redis_channel.py @@ -3,8 +3,15 @@ import json from unittest.mock import MagicMock +from core.variables import IntegerVariable, StringVariable from core.workflow.graph_engine.command_channels.redis_channel import RedisChannel -from core.workflow.graph_engine.entities.commands import AbortCommand, CommandType, GraphEngineCommand +from core.workflow.graph_engine.entities.commands import ( + AbortCommand, + CommandType, + GraphEngineCommand, + UpdateVariablesCommand, + VariableUpdate, +) class TestRedisChannel: @@ -148,6 +155,43 @@ class TestRedisChannel: assert commands[0].command_type == CommandType.ABORT assert isinstance(commands[1], AbortCommand) + def test_fetch_commands_with_update_variables_command(self): + """Test fetching update variables command from Redis.""" + mock_redis = MagicMock() + pending_pipe = MagicMock() + fetch_pipe = MagicMock() + pending_context = MagicMock() + fetch_context = MagicMock() + pending_context.__enter__.return_value = pending_pipe + pending_context.__exit__.return_value = None + fetch_context.__enter__.return_value = fetch_pipe + fetch_context.__exit__.return_value = None + mock_redis.pipeline.side_effect = [pending_context, fetch_context] + + update_command = UpdateVariablesCommand( + updates=[ + VariableUpdate( + value=StringVariable(name="foo", value="bar", selector=["node1", "foo"]), + ), + VariableUpdate( + value=IntegerVariable(name="baz", value=123, selector=["node2", "baz"]), + ), + ] + ) + command_json = json.dumps(update_command.model_dump()) + + pending_pipe.execute.return_value = [b"1", 1] + fetch_pipe.execute.return_value = [[command_json.encode()], 1] + + channel = RedisChannel(mock_redis, "test:key") + commands = channel.fetch_commands() + + assert len(commands) == 1 + assert isinstance(commands[0], UpdateVariablesCommand) + assert isinstance(commands[0].updates[0].value, StringVariable) + assert list(commands[0].updates[0].value.selector) == ["node1", "foo"] + assert commands[0].updates[0].value.value == "bar" + def test_fetch_commands_skips_invalid_json(self): """Test that invalid JSON commands are skipped.""" mock_redis = MagicMock() diff --git a/api/tests/unit_tests/core/workflow/graph_engine/layers/test_layer_initialization.py b/api/tests/unit_tests/core/workflow/graph_engine/layers/test_layer_initialization.py new file mode 100644 index 0000000000..d6ba61c50c --- /dev/null +++ b/api/tests/unit_tests/core/workflow/graph_engine/layers/test_layer_initialization.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import pytest + +from core.workflow.graph_engine import GraphEngine +from core.workflow.graph_engine.command_channels import InMemoryChannel +from core.workflow.graph_engine.layers.base import ( + GraphEngineLayer, + GraphEngineLayerNotInitializedError, +) +from core.workflow.graph_events import GraphEngineEvent + +from ..test_table_runner import WorkflowRunner + + +class LayerForTest(GraphEngineLayer): + def on_graph_start(self) -> None: + pass + + def on_event(self, event: GraphEngineEvent) -> None: + pass + + def on_graph_end(self, error: Exception | None) -> None: + pass + + +def test_layer_runtime_state_raises_when_uninitialized() -> None: + layer = LayerForTest() + + with pytest.raises(GraphEngineLayerNotInitializedError): + _ = layer.graph_runtime_state + + +def test_layer_runtime_state_available_after_engine_layer() -> None: + runner = WorkflowRunner() + fixture_data = runner.load_fixture("simple_passthrough_workflow") + graph, graph_runtime_state = runner.create_graph_from_fixture( + fixture_data, + inputs={"query": "test layer state"}, + ) + engine = GraphEngine( + workflow_id="test_workflow", + graph=graph, + graph_runtime_state=graph_runtime_state, + command_channel=InMemoryChannel(), + ) + + layer = LayerForTest() + engine.layer(layer) + + outputs = layer.graph_runtime_state.outputs + ready_queue_size = layer.graph_runtime_state.ready_queue_size + + assert outputs == {} + assert isinstance(ready_queue_size, int) + assert ready_queue_size >= 0 diff --git a/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py b/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py index c1fc4acd73..fe3ea576c1 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py @@ -3,6 +3,7 @@ from __future__ import annotations import queue +import threading from unittest import mock from core.workflow.entities.pause_reason import SchedulingPause @@ -36,6 +37,7 @@ def test_dispatcher_should_consume_remains_events_after_pause(): event_queue=event_queue, event_handler=event_handler, execution_coordinator=execution_coordinator, + stop_event=threading.Event(), ) dispatcher._dispatcher_loop() assert event_queue.empty() @@ -96,6 +98,7 @@ def _run_dispatcher_for_event(event) -> int: event_queue=event_queue, event_handler=event_handler, execution_coordinator=coordinator, + stop_event=threading.Event(), ) dispatcher._dispatcher_loop() @@ -181,6 +184,7 @@ def test_dispatcher_drain_event_queue(): event_queue=event_queue, event_handler=event_handler, execution_coordinator=coordinator, + stop_event=threading.Event(), ) dispatcher._dispatcher_loop() diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py b/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py index b074a11be9..d826f7a900 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py @@ -4,12 +4,19 @@ import time from unittest.mock import MagicMock from core.app.entities.app_invoke_entities import InvokeFrom +from core.variables import IntegerVariable, StringVariable from core.workflow.entities.graph_init_params import GraphInitParams from core.workflow.entities.pause_reason import SchedulingPause from core.workflow.graph import Graph from core.workflow.graph_engine import GraphEngine from core.workflow.graph_engine.command_channels import InMemoryChannel -from core.workflow.graph_engine.entities.commands import AbortCommand, CommandType, PauseCommand +from core.workflow.graph_engine.entities.commands import ( + AbortCommand, + CommandType, + PauseCommand, + UpdateVariablesCommand, + VariableUpdate, +) from core.workflow.graph_events import GraphRunAbortedEvent, GraphRunPausedEvent, GraphRunStartedEvent from core.workflow.nodes.start.start_node import StartNode from core.workflow.runtime import GraphRuntimeState, VariablePool @@ -180,3 +187,67 @@ def test_pause_command(): graph_execution = engine.graph_runtime_state.graph_execution assert graph_execution.pause_reasons == [SchedulingPause(message="User requested pause")] + + +def test_update_variables_command_updates_pool(): + """Test that GraphEngine updates variable pool via update variables command.""" + + shared_runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + shared_runtime_state.variable_pool.add(("node1", "foo"), "old value") + + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + mock_graph.root_node.id = "start" + + start_node = StartNode( + id="start", + config={"id": "start", "data": {"title": "start", "variables": []}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=shared_runtime_state, + ) + mock_graph.nodes["start"] = start_node + + mock_graph.get_outgoing_edges = MagicMock(return_value=[]) + mock_graph.get_incoming_edges = MagicMock(return_value=[]) + + command_channel = InMemoryChannel() + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=shared_runtime_state, + command_channel=command_channel, + ) + + update_command = UpdateVariablesCommand( + updates=[ + VariableUpdate( + value=StringVariable(name="foo", value="new value", selector=["node1", "foo"]), + ), + VariableUpdate( + value=IntegerVariable(name="bar", value=123, selector=["node2", "bar"]), + ), + ] + ) + command_channel.send_command(update_command) + + list(engine.run()) + + updated_existing = shared_runtime_state.variable_pool.get(["node1", "foo"]) + added_new = shared_runtime_state.variable_pool.get(["node2", "bar"]) + + assert updated_existing is not None + assert updated_existing.value == "new value" + assert added_new is not None + assert added_new.value == 123 diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_stop_event.py b/api/tests/unit_tests/core/workflow/graph_engine/test_stop_event.py new file mode 100644 index 0000000000..ea8d3a977f --- /dev/null +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_stop_event.py @@ -0,0 +1,539 @@ +""" +Unit tests for stop_event functionality in GraphEngine. + +Tests the unified stop_event management by GraphEngine and its propagation +to WorkerPool, Worker, Dispatcher, and Nodes. +""" + +import threading +import time +from unittest.mock import MagicMock, Mock, patch + +from core.app.entities.app_invoke_entities import InvokeFrom +from core.workflow.entities.graph_init_params import GraphInitParams +from core.workflow.graph import Graph +from core.workflow.graph_engine import GraphEngine +from core.workflow.graph_engine.command_channels import InMemoryChannel +from core.workflow.graph_events import ( + GraphRunStartedEvent, + GraphRunSucceededEvent, + NodeRunStartedEvent, +) +from core.workflow.nodes.answer.answer_node import AnswerNode +from core.workflow.nodes.start.start_node import StartNode +from core.workflow.runtime import GraphRuntimeState, VariablePool +from models.enums import UserFrom + + +class TestStopEventPropagation: + """Test suite for stop_event propagation through GraphEngine components.""" + + def test_graph_engine_creates_stop_event(self): + """Test that GraphEngine creates a stop_event on initialization.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Verify stop_event was created + assert engine._stop_event is not None + assert isinstance(engine._stop_event, threading.Event) + + # Verify it was set in graph_runtime_state + assert runtime_state.stop_event is not None + assert runtime_state.stop_event is engine._stop_event + + def test_stop_event_cleared_on_start(self): + """Test that stop_event is cleared when execution starts.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + mock_graph.root_node.id = "start" # Set proper id + + start_node = StartNode( + id="start", + config={"id": "start", "data": {"title": "start", "variables": []}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + mock_graph.nodes["start"] = start_node + mock_graph.get_outgoing_edges = MagicMock(return_value=[]) + mock_graph.get_incoming_edges = MagicMock(return_value=[]) + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Set the stop_event before running + engine._stop_event.set() + assert engine._stop_event.is_set() + + # Run the engine (should clear the stop_event) + events = list(engine.run()) + + # After running, stop_event should be set again (by _stop_execution) + # But during start it was cleared + assert any(isinstance(e, GraphRunStartedEvent) for e in events) + assert any(isinstance(e, GraphRunSucceededEvent) for e in events) + + def test_stop_event_set_on_stop(self): + """Test that stop_event is set when execution stops.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + mock_graph.root_node.id = "start" # Set proper id + + start_node = StartNode( + id="start", + config={"id": "start", "data": {"title": "start", "variables": []}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + mock_graph.nodes["start"] = start_node + mock_graph.get_outgoing_edges = MagicMock(return_value=[]) + mock_graph.get_incoming_edges = MagicMock(return_value=[]) + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Initially not set + assert not engine._stop_event.is_set() + + # Run the engine + list(engine.run()) + + # After execution completes, stop_event should be set + assert engine._stop_event.is_set() + + def test_stop_event_passed_to_worker_pool(self): + """Test that stop_event is passed to WorkerPool.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Verify WorkerPool has the stop_event + assert engine._worker_pool._stop_event is not None + assert engine._worker_pool._stop_event is engine._stop_event + + def test_stop_event_passed_to_dispatcher(self): + """Test that stop_event is passed to Dispatcher.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Verify Dispatcher has the stop_event + assert engine._dispatcher._stop_event is not None + assert engine._dispatcher._stop_event is engine._stop_event + + +class TestNodeStopCheck: + """Test suite for Node._should_stop() functionality.""" + + def test_node_should_stop_checks_runtime_state(self): + """Test that Node._should_stop() checks GraphRuntimeState.stop_event.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + + answer_node = AnswerNode( + id="answer", + config={"id": "answer", "data": {"title": "answer", "answer": "{{#start.result#}}"}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + + # Initially stop_event is not set + assert not answer_node._should_stop() + + # Set the stop_event + runtime_state.stop_event.set() + + # Now _should_stop should return True + assert answer_node._should_stop() + + def test_node_run_checks_stop_event_between_yields(self): + """Test that Node.run() checks stop_event between yielding events.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + + # Create a simple node + answer_node = AnswerNode( + id="answer", + config={"id": "answer", "data": {"title": "answer", "answer": "hello"}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + + # Set stop_event BEFORE running the node + runtime_state.stop_event.set() + + # Run the node - should yield start event then detect stop + # The node should check stop_event before processing + assert answer_node._should_stop(), "stop_event should be set" + + # Run and collect events + events = list(answer_node.run()) + + # Since stop_event is set at the start, we should get: + # 1. NodeRunStartedEvent (always yielded first) + # 2. Either NodeRunFailedEvent (if detected early) or NodeRunSucceededEvent (if too fast) + assert len(events) >= 2 + assert isinstance(events[0], NodeRunStartedEvent) + + # Note: AnswerNode is very simple and might complete before stop check + # The important thing is that _should_stop() returns True when stop_event is set + assert answer_node._should_stop() + + +class TestStopEventIntegration: + """Integration tests for stop_event in workflow execution.""" + + def test_simple_workflow_respects_stop_event(self): + """Test that a simple workflow respects stop_event.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + mock_graph.root_node.id = "start" + + # Create start and answer nodes + start_node = StartNode( + id="start", + config={"id": "start", "data": {"title": "start", "variables": []}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + + answer_node = AnswerNode( + id="answer", + config={"id": "answer", "data": {"title": "answer", "answer": "hello"}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + + mock_graph.nodes["start"] = start_node + mock_graph.nodes["answer"] = answer_node + mock_graph.get_outgoing_edges = MagicMock(return_value=[]) + mock_graph.get_incoming_edges = MagicMock(return_value=[]) + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Set stop_event before running + runtime_state.stop_event.set() + + # Run the engine + events = list(engine.run()) + + # Should get started event but not succeeded (due to stop) + assert any(isinstance(e, GraphRunStartedEvent) for e in events) + # The workflow should still complete (start node runs quickly) + # but answer node might be cancelled depending on timing + + def test_stop_event_with_concurrent_nodes(self): + """Test stop_event behavior with multiple concurrent nodes.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + + # Create multiple nodes + for i in range(3): + answer_node = AnswerNode( + id=f"answer_{i}", + config={"id": f"answer_{i}", "data": {"title": f"answer_{i}", "answer": f"test{i}"}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + mock_graph.nodes[f"answer_{i}"] = answer_node + + mock_graph.get_outgoing_edges = MagicMock(return_value=[]) + mock_graph.get_incoming_edges = MagicMock(return_value=[]) + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # All nodes should share the same stop_event + for node in mock_graph.nodes.values(): + assert node.graph_runtime_state.stop_event is runtime_state.stop_event + assert node.graph_runtime_state.stop_event is engine._stop_event + + +class TestStopEventTimeoutBehavior: + """Test stop_event behavior with join timeouts.""" + + @patch("core.workflow.graph_engine.orchestration.dispatcher.threading.Thread") + def test_dispatcher_uses_shorter_timeout(self, mock_thread_cls: MagicMock): + """Test that Dispatcher uses 2s timeout instead of 10s.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + dispatcher = engine._dispatcher + dispatcher.start() # This will create and start the mocked thread + + mock_thread_instance = mock_thread_cls.return_value + mock_thread_instance.is_alive.return_value = True + + dispatcher.stop() + + mock_thread_instance.join.assert_called_once_with(timeout=2.0) + + @patch("core.workflow.graph_engine.worker_management.worker_pool.Worker") + def test_worker_pool_uses_shorter_timeout(self, mock_worker_cls: MagicMock): + """Test that WorkerPool uses 2s timeout instead of 10s.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + worker_pool = engine._worker_pool + worker_pool.start(initial_count=1) # Start with one worker + + mock_worker_instance = mock_worker_cls.return_value + mock_worker_instance.is_alive.return_value = True + + worker_pool.stop() + + mock_worker_instance.join.assert_called_once_with(timeout=2.0) + + +class TestStopEventResumeBehavior: + """Test stop_event behavior during workflow resume.""" + + def test_stop_event_cleared_on_resume(self): + """Test that stop_event is cleared when resuming a paused workflow.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + mock_graph.root_node.id = "start" # Set proper id + + start_node = StartNode( + id="start", + config={"id": "start", "data": {"title": "start", "variables": []}}, + graph_init_params=GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ), + graph_runtime_state=runtime_state, + ) + mock_graph.nodes["start"] = start_node + mock_graph.get_outgoing_edges = MagicMock(return_value=[]) + mock_graph.get_incoming_edges = MagicMock(return_value=[]) + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Simulate a previous execution that set stop_event + engine._stop_event.set() + assert engine._stop_event.is_set() + + # Run the engine (should clear stop_event in _start_execution) + events = list(engine.run()) + + # Execution should complete successfully + assert any(isinstance(e, GraphRunStartedEvent) for e in events) + assert any(isinstance(e, GraphRunSucceededEvent) for e in events) + + +class TestWorkerStopBehavior: + """Test Worker behavior with shared stop_event.""" + + def test_worker_uses_shared_stop_event(self): + """Test that Worker uses shared stop_event from GraphEngine.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + mock_graph = MagicMock(spec=Graph) + mock_graph.nodes = {} + mock_graph.edges = {} + mock_graph.root_node = MagicMock() + + engine = GraphEngine( + workflow_id="test_workflow", + graph=mock_graph, + graph_runtime_state=runtime_state, + command_channel=InMemoryChannel(), + ) + + # Get the worker pool and check workers + worker_pool = engine._worker_pool + + # Start the worker pool to create workers + worker_pool.start() + + # Check that at least one worker was created + assert len(worker_pool._workers) > 0 + + # Verify workers use the shared stop_event + for worker in worker_pool._workers: + assert worker._stop_event is engine._stop_event + + # Clean up + worker_pool.stop() + + def test_worker_stop_is_noop(self): + """Test that Worker.stop() is now a no-op.""" + runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=time.perf_counter()) + + # Create a mock worker + from core.workflow.graph_engine.ready_queue import InMemoryReadyQueue + from core.workflow.graph_engine.worker import Worker + + ready_queue = InMemoryReadyQueue() + event_queue = MagicMock() + + # Create a proper mock graph with real dict + mock_graph = Mock(spec=Graph) + mock_graph.nodes = {} # Use real dict + + stop_event = threading.Event() + + worker = Worker( + ready_queue=ready_queue, + event_queue=event_queue, + graph=mock_graph, + layers=[], + stop_event=stop_event, + ) + + # Calling stop() should do nothing (no-op) + # and should NOT set the stop_event + worker.stop() + assert not stop_event.is_set() diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py index e8f257bf2f..1e224d56a5 100644 --- a/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py +++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py @@ -78,7 +78,7 @@ class TestFileSaverImpl: file_binary=_PNG_DATA, mimetype=mime_type, ) - mocked_sign_file.assert_called_once_with(mock_tool_file.id, ".png") + mocked_sign_file.assert_called_once_with(tool_file_id=mock_tool_file.id, extension=".png", for_external=True) def test_save_remote_url_request_failed(self, monkeypatch: pytest.MonkeyPatch): _TEST_URL = "https://example.com/image.png" diff --git a/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py b/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py index 1a67d5c3e3..66d6c3c56b 100644 --- a/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py +++ b/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py @@ -5,8 +5,8 @@ from core.workflow.graph_engine.entities.graph import Graph from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState -from core.helper.code_executor.code_executor import CodeExecutionError from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.nodes.template_transform.template_renderer import TemplateRenderError from core.workflow.nodes.template_transform.template_transform_node import TemplateTransformNode from models.workflow import WorkflowType @@ -127,7 +127,9 @@ class TestTemplateTransformNode: """Test version class method.""" assert TemplateTransformNode.version() == "1" - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_simple_template( self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params ): @@ -145,7 +147,7 @@ class TestTemplateTransformNode: mock_graph_runtime_state.variable_pool.get.side_effect = lambda selector: variable_map.get(tuple(selector)) # Setup mock executor - mock_execute.return_value = {"result": "Hello Alice, you are 30 years old!"} + mock_execute.return_value = "Hello Alice, you are 30 years old!" node = TemplateTransformNode( id="test_node", @@ -162,7 +164,9 @@ class TestTemplateTransformNode: assert result.inputs["name"] == "Alice" assert result.inputs["age"] == 30 - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_with_none_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): """Test _run with None variable values.""" node_data = { @@ -172,7 +176,7 @@ class TestTemplateTransformNode: } mock_graph_runtime_state.variable_pool.get.return_value = None - mock_execute.return_value = {"result": "Value: "} + mock_execute.return_value = "Value: " node = TemplateTransformNode( id="test_node", @@ -187,13 +191,15 @@ class TestTemplateTransformNode: assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.inputs["value"] is None - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_with_code_execution_error( self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params ): """Test _run when code execution fails.""" mock_graph_runtime_state.variable_pool.get.return_value = MagicMock() - mock_execute.side_effect = CodeExecutionError("Template syntax error") + mock_execute.side_effect = TemplateRenderError("Template syntax error") node = TemplateTransformNode( id="test_node", @@ -208,14 +214,16 @@ class TestTemplateTransformNode: assert result.status == WorkflowNodeExecutionStatus.FAILED assert "Template syntax error" in result.error - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) @patch("core.workflow.nodes.template_transform.template_transform_node.MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH", 10) def test_run_output_length_exceeds_limit( self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params ): """Test _run when output exceeds maximum length.""" mock_graph_runtime_state.variable_pool.get.return_value = MagicMock() - mock_execute.return_value = {"result": "This is a very long output that exceeds the limit"} + mock_execute.return_value = "This is a very long output that exceeds the limit" node = TemplateTransformNode( id="test_node", @@ -230,7 +238,9 @@ class TestTemplateTransformNode: assert result.status == WorkflowNodeExecutionStatus.FAILED assert "Output length exceeds" in result.error - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_with_complex_jinja2_template( self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params ): @@ -257,7 +267,7 @@ class TestTemplateTransformNode: ("sys", "show_total"): mock_show_total, } mock_graph_runtime_state.variable_pool.get.side_effect = lambda selector: variable_map.get(tuple(selector)) - mock_execute.return_value = {"result": "apple, banana, orange (Total: 3)"} + mock_execute.return_value = "apple, banana, orange (Total: 3)" node = TemplateTransformNode( id="test_node", @@ -292,7 +302,9 @@ class TestTemplateTransformNode: assert mapping["node_123.var1"] == ["sys", "input1"] assert mapping["node_123.var2"] == ["sys", "input2"] - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_with_empty_variables(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): """Test _run with no variables (static template).""" node_data = { @@ -301,7 +313,7 @@ class TestTemplateTransformNode: "template": "This is a static message.", } - mock_execute.return_value = {"result": "This is a static message."} + mock_execute.return_value = "This is a static message." node = TemplateTransformNode( id="test_node", @@ -317,7 +329,9 @@ class TestTemplateTransformNode: assert result.outputs["output"] == "This is a static message." assert result.inputs == {} - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_with_numeric_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): """Test _run with numeric variable values.""" node_data = { @@ -339,7 +353,7 @@ class TestTemplateTransformNode: ("sys", "quantity"): mock_quantity, } mock_graph_runtime_state.variable_pool.get.side_effect = lambda selector: variable_map.get(tuple(selector)) - mock_execute.return_value = {"result": "Total: $31.5"} + mock_execute.return_value = "Total: $31.5" node = TemplateTransformNode( id="test_node", @@ -354,7 +368,9 @@ class TestTemplateTransformNode: assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.outputs["output"] == "Total: $31.5" - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_with_dict_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): """Test _run with dictionary variable values.""" node_data = { @@ -367,7 +383,7 @@ class TestTemplateTransformNode: mock_user.to_object.return_value = {"name": "John Doe", "email": "john@example.com"} mock_graph_runtime_state.variable_pool.get.return_value = mock_user - mock_execute.return_value = {"result": "Name: John Doe, Email: john@example.com"} + mock_execute.return_value = "Name: John Doe, Email: john@example.com" node = TemplateTransformNode( id="test_node", @@ -383,7 +399,9 @@ class TestTemplateTransformNode: assert "John Doe" in result.outputs["output"] assert "john@example.com" in result.outputs["output"] - @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch( + "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template" + ) def test_run_with_list_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): """Test _run with list variable values.""" node_data = { @@ -396,7 +414,7 @@ class TestTemplateTransformNode: mock_tags.to_object.return_value = ["python", "ai", "workflow"] mock_graph_runtime_state.variable_pool.get.return_value = mock_tags - mock_execute.return_value = {"result": "Tags: #python #ai #workflow "} + mock_execute.return_value = "Tags: #python #ai #workflow " node = TemplateTransformNode( id="test_node", diff --git a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py index c62fc4d8fe..1df75380af 100644 --- a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py +++ b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py @@ -1,14 +1,14 @@ import time import uuid -from unittest import mock from uuid import uuid4 from core.app.entities.app_invoke_entities import InvokeFrom from core.variables import ArrayStringVariable, StringVariable -from core.workflow.conversation_variable_updater import ConversationVariableUpdater from core.workflow.entities import GraphInitParams from core.workflow.graph import Graph +from core.workflow.graph_events.node import NodeRunSucceededEvent from core.workflow.nodes.node_factory import DifyNodeFactory +from core.workflow.nodes.variable_assigner.common import helpers as common_helpers from core.workflow.nodes.variable_assigner.v1 import VariableAssignerNode from core.workflow.nodes.variable_assigner.v1.node_data import WriteMode from core.workflow.runtime import GraphRuntimeState, VariablePool @@ -86,9 +86,6 @@ def test_overwrite_string_variable(): ) graph = Graph.init(graph_config=graph_config, node_factory=node_factory) - mock_conv_var_updater = mock.Mock(spec=ConversationVariableUpdater) - mock_conv_var_updater_factory = mock.Mock(return_value=mock_conv_var_updater) - node_config = { "id": "node_id", "data": { @@ -104,20 +101,14 @@ def test_overwrite_string_variable(): graph_init_params=init_params, graph_runtime_state=graph_runtime_state, config=node_config, - conv_var_updater_factory=mock_conv_var_updater_factory, ) - list(node.run()) - expected_var = StringVariable( - id=conversation_variable.id, - name=conversation_variable.name, - description=conversation_variable.description, - selector=conversation_variable.selector, - value_type=conversation_variable.value_type, - value=input_variable.value, - ) - mock_conv_var_updater.update.assert_called_once_with(conversation_id=conversation_id, variable=expected_var) - mock_conv_var_updater.flush.assert_called_once() + events = list(node.run()) + succeeded_event = next(event for event in events if isinstance(event, NodeRunSucceededEvent)) + updated_variables = common_helpers.get_updated_variables(succeeded_event.node_run_result.process_data) + assert updated_variables is not None + assert updated_variables[0].name == conversation_variable.name + assert updated_variables[0].new_value == input_variable.value got = variable_pool.get(["conversation", conversation_variable.name]) assert got is not None @@ -191,9 +182,6 @@ def test_append_variable_to_array(): ) graph = Graph.init(graph_config=graph_config, node_factory=node_factory) - mock_conv_var_updater = mock.Mock(spec=ConversationVariableUpdater) - mock_conv_var_updater_factory = mock.Mock(return_value=mock_conv_var_updater) - node_config = { "id": "node_id", "data": { @@ -209,22 +197,14 @@ def test_append_variable_to_array(): graph_init_params=init_params, graph_runtime_state=graph_runtime_state, config=node_config, - conv_var_updater_factory=mock_conv_var_updater_factory, ) - list(node.run()) - expected_value = list(conversation_variable.value) - expected_value.append(input_variable.value) - expected_var = ArrayStringVariable( - id=conversation_variable.id, - name=conversation_variable.name, - description=conversation_variable.description, - selector=conversation_variable.selector, - value_type=conversation_variable.value_type, - value=expected_value, - ) - mock_conv_var_updater.update.assert_called_once_with(conversation_id=conversation_id, variable=expected_var) - mock_conv_var_updater.flush.assert_called_once() + events = list(node.run()) + succeeded_event = next(event for event in events if isinstance(event, NodeRunSucceededEvent)) + updated_variables = common_helpers.get_updated_variables(succeeded_event.node_run_result.process_data) + assert updated_variables is not None + assert updated_variables[0].name == conversation_variable.name + assert updated_variables[0].new_value == ["the first value", "the second value"] got = variable_pool.get(["conversation", conversation_variable.name]) assert got is not None @@ -287,9 +267,6 @@ def test_clear_array(): ) graph = Graph.init(graph_config=graph_config, node_factory=node_factory) - mock_conv_var_updater = mock.Mock(spec=ConversationVariableUpdater) - mock_conv_var_updater_factory = mock.Mock(return_value=mock_conv_var_updater) - node_config = { "id": "node_id", "data": { @@ -305,20 +282,14 @@ def test_clear_array(): graph_init_params=init_params, graph_runtime_state=graph_runtime_state, config=node_config, - conv_var_updater_factory=mock_conv_var_updater_factory, ) - list(node.run()) - expected_var = ArrayStringVariable( - id=conversation_variable.id, - name=conversation_variable.name, - description=conversation_variable.description, - selector=conversation_variable.selector, - value_type=conversation_variable.value_type, - value=[], - ) - mock_conv_var_updater.update.assert_called_once_with(conversation_id=conversation_id, variable=expected_var) - mock_conv_var_updater.flush.assert_called_once() + events = list(node.run()) + succeeded_event = next(event for event in events if isinstance(event, NodeRunSucceededEvent)) + updated_variables = common_helpers.get_updated_variables(succeeded_event.node_run_result.process_data) + assert updated_variables is not None + assert updated_variables[0].name == conversation_variable.name + assert updated_variables[0].new_value == [] got = variable_pool.get(["conversation", conversation_variable.name]) assert got is not None diff --git a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py index caa36734ad..353d56fe25 100644 --- a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py +++ b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py @@ -390,3 +390,42 @@ def test_remove_last_from_empty_array(): got = variable_pool.get(["conversation", conversation_variable.name]) assert got is not None assert got.to_object() == [] + + +def test_node_factory_creates_variable_assigner_node(): + graph_config = { + "edges": [], + "nodes": [ + { + "data": {"type": "assigner", "version": "2", "title": "Variable Assigner", "items": []}, + "id": "assigner", + }, + ], + } + + init_params = GraphInitParams( + tenant_id="1", + app_id="1", + workflow_id="1", + graph_config=graph_config, + user_id="1", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + call_depth=0, + ) + variable_pool = VariablePool( + system_variables=SystemVariable(conversation_id="conversation_id"), + user_inputs={}, + environment_variables=[], + conversation_variables=[], + ) + graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()) + + node_factory = DifyNodeFactory( + graph_init_params=init_params, + graph_runtime_state=graph_runtime_state, + ) + + node = node_factory.create_node(graph_config["nodes"][0]) + + assert isinstance(node, VariableAssignerNode) diff --git a/api/tests/unit_tests/libs/test_helper.py b/api/tests/unit_tests/libs/test_helper.py index 85789bfa7e..de74eff82f 100644 --- a/api/tests/unit_tests/libs/test_helper.py +++ b/api/tests/unit_tests/libs/test_helper.py @@ -1,6 +1,6 @@ import pytest -from libs.helper import extract_tenant_id +from libs.helper import escape_like_pattern, extract_tenant_id from models.account import Account from models.model import EndUser @@ -63,3 +63,51 @@ class TestExtractTenantId: with pytest.raises(ValueError, match="Invalid user type.*Expected Account or EndUser"): extract_tenant_id(dict_user) + + +class TestEscapeLikePattern: + """Test cases for the escape_like_pattern utility function.""" + + def test_escape_percent_character(self): + """Test escaping percent character.""" + result = escape_like_pattern("50% discount") + assert result == "50\\% discount" + + def test_escape_underscore_character(self): + """Test escaping underscore character.""" + result = escape_like_pattern("test_data") + assert result == "test\\_data" + + def test_escape_backslash_character(self): + """Test escaping backslash character.""" + result = escape_like_pattern("path\\to\\file") + assert result == "path\\\\to\\\\file" + + def test_escape_combined_special_characters(self): + """Test escaping multiple special characters together.""" + result = escape_like_pattern("file_50%\\path") + assert result == "file\\_50\\%\\\\path" + + def test_escape_empty_string(self): + """Test escaping empty string returns empty string.""" + result = escape_like_pattern("") + assert result == "" + + def test_escape_none_handling(self): + """Test escaping None returns None (falsy check handles it).""" + # The function checks `if not pattern`, so None is falsy and returns as-is + result = escape_like_pattern(None) + assert result is None + + def test_escape_normal_string_no_change(self): + """Test that normal strings without special characters are unchanged.""" + result = escape_like_pattern("normal text") + assert result == "normal text" + + def test_escape_order_matters(self): + """Test that backslash is escaped first to prevent double escaping.""" + # If we escape % first, then escape \, we might get wrong results + # This test ensures the order is correct: \ first, then % and _ + result = escape_like_pattern("test\\%_value") + # Should be: test\\\%\_value + assert result == "test\\\\\\%\\_value" diff --git a/api/tests/unit_tests/models/test_app_models.py b/api/tests/unit_tests/models/test_app_models.py index e35788660d..8be2eea121 100644 --- a/api/tests/unit_tests/models/test_app_models.py +++ b/api/tests/unit_tests/models/test_app_models.py @@ -114,7 +114,7 @@ class TestAppModelValidation: def test_icon_type_validation(self): """Test icon type enum values.""" # Assert - assert {t.value for t in IconType} == {"image", "emoji"} + assert {t.value for t in IconType} == {"image", "emoji", "link"} def test_app_desc_or_prompt_with_description(self): """Test desc_or_prompt property when description exists.""" diff --git a/api/uv.lock b/api/uv.lock index fa032fa8d4..8e60fad3a7 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -2955,14 +2955,14 @@ wheels = [ [[package]] name = "intersystems-irispython" -version = "5.3.0" +version = "5.3.1" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/56/16d93576b50408d97a5cbbd055d8da024d585e96a360e2adc95b41ae6284/intersystems_irispython-5.3.0-cp38.cp39.cp310.cp311.cp312.cp313-cp38.cp39.cp310.cp311.cp312.cp313-macosx_10_9_universal2.whl", hash = "sha256:59d3176a35867a55b1ab69a6b5c75438b460291bccb254c2d2f4173be08b6e55", size = 6594480, upload-time = "2025-10-09T20:47:27.629Z" }, - { url = "https://files.pythonhosted.org/packages/99/bc/19e144ee805ea6ee0df6342a711e722c84347c05a75b3bf040c5fbe19982/intersystems_irispython-5.3.0-cp38.cp39.cp310.cp311.cp312.cp313-cp38.cp39.cp310.cp311.cp312.cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56bccefd1997c25f9f9f6c4086214c18d4fdaac0a93319d4b21dd9a6c59c9e51", size = 14779928, upload-time = "2025-10-09T20:47:30.564Z" }, - { url = "https://files.pythonhosted.org/packages/e6/fb/59ba563a80b39e9450b4627b5696019aa831dce27dacc3831b8c1e669102/intersystems_irispython-5.3.0-cp38.cp39.cp310.cp311.cp312.cp313-cp38.cp39.cp310.cp311.cp312.cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e160adc0785c55bb64e4264b8e99075691a15b0afa5d8d529f1b4bac7e57b81", size = 14422035, upload-time = "2025-10-09T20:47:32.552Z" }, - { url = "https://files.pythonhosted.org/packages/c1/68/ade8ad43f0ed1e5fba60e1710fa5ddeb01285f031e465e8c006329072e63/intersystems_irispython-5.3.0-cp38.cp39.cp310.cp311.cp312.cp313-cp38.cp39.cp310.cp311.cp312.cp313-win32.whl", hash = "sha256:820f2c5729119e5173a5bf6d6ac2a41275c4f1ffba6af6c59ea313ecd8f499cc", size = 2824316, upload-time = "2025-10-09T20:47:28.998Z" }, - { url = "https://files.pythonhosted.org/packages/f4/03/cd45cb94e42c01dc525efebf3c562543a18ee55b67fde4022665ca672351/intersystems_irispython-5.3.0-cp38.cp39.cp310.cp311.cp312.cp313-cp38.cp39.cp310.cp311.cp312.cp313-win_amd64.whl", hash = "sha256:fc07ec24bc50b6f01573221cd7d86f2937549effe31c24af8db118e0131e340c", size = 3463297, upload-time = "2025-10-09T20:47:34.636Z" }, + { url = "https://files.pythonhosted.org/packages/33/5b/8eac672a6ef26bef6ef79a7c9557096167b50c4d3577d558ae6999c195fe/intersystems_irispython-5.3.1-cp38.cp39.cp310.cp311.cp312.cp313.cp314-cp38.cp39.cp310.cp311.cp312.cp313.cp314-macosx_10_9_universal2.whl", hash = "sha256:634c9b4ec620837d830ff49543aeb2797a1ce8d8570a0e868398b85330dfcc4d", size = 6736686, upload-time = "2025-12-19T16:24:57.734Z" }, + { url = "https://files.pythonhosted.org/packages/ba/17/bab3e525ffb6711355f7feea18c1b7dced9c2484cecbcdd83f74550398c0/intersystems_irispython-5.3.1-cp38.cp39.cp310.cp311.cp312.cp313.cp314-cp38.cp39.cp310.cp311.cp312.cp313.cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf912f30f85e2a42f2c2ea77fbeb98a24154d5ea7428a50382786a684ec4f583", size = 16005259, upload-time = "2025-12-19T16:25:05.578Z" }, + { url = "https://files.pythonhosted.org/packages/39/59/9bb79d9e32e3e55fc9aed8071a797b4497924cbc6457cea9255bb09320b7/intersystems_irispython-5.3.1-cp38.cp39.cp310.cp311.cp312.cp313.cp314-cp38.cp39.cp310.cp311.cp312.cp313.cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be5659a6bb57593910f2a2417eddb9f5dc2f93a337ead6ddca778f557b8a359a", size = 15638040, upload-time = "2025-12-19T16:24:54.429Z" }, + { url = "https://files.pythonhosted.org/packages/cf/47/654ccf9c5cca4f5491f070888544165c9e2a6a485e320ea703e4e38d2358/intersystems_irispython-5.3.1-cp38.cp39.cp310.cp311.cp312.cp313.cp314-cp38.cp39.cp310.cp311.cp312.cp313.cp314-win32.whl", hash = "sha256:583e4f17088c1e0530f32efda1c0ccb02993cbc22035bc8b4c71d8693b04ee7e", size = 2879644, upload-time = "2025-12-19T16:24:59.945Z" }, + { url = "https://files.pythonhosted.org/packages/68/95/19cc13d09f1b4120bd41b1434509052e1d02afd27f2679266d7ad9cc1750/intersystems_irispython-5.3.1-cp38.cp39.cp310.cp311.cp312.cp313.cp314-cp38.cp39.cp310.cp311.cp312.cp313.cp314-win_amd64.whl", hash = "sha256:1d5d40450a0cdeec2a1f48d12d946a8a8ffc7c128576fcae7d58e66e3a127eae", size = 3522092, upload-time = "2025-12-19T16:25:01.834Z" }, ] [[package]] diff --git a/docker/.env.example b/docker/.env.example index ecb003cd70..09ee1060e2 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -58,8 +58,8 @@ FILES_URL= INTERNAL_FILES_URL= # Ensure UTF-8 encoding -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 +LANG=C.UTF-8 +LC_ALL=C.UTF-8 PYTHONIOENCODING=utf-8 # ------------------------------ @@ -1077,6 +1077,10 @@ LOGSTORE_DUAL_WRITE_ENABLED=false # Enable dual-read fallback to SQL database when LogStore returns no results (default: true) # Useful for migration scenarios where historical data exists only in SQL database LOGSTORE_DUAL_READ_ENABLED=true +# Control flag for whether to write the `graph` field to LogStore. +# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; +# otherwise write an empty {} instead. Defaults to writing the `graph` field. +LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true # HTTP request node in workflow configuration HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 3c88cddf8c..709aff23df 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -475,7 +475,8 @@ services: OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} OB_SERVER_IP: 127.0.0.1 MODE: mini - LANG: en_US.UTF-8 + LANG: C.UTF-8 + LC_ALL: C.UTF-8 ports: - "${OCEANBASE_VECTOR_PORT:-2881}:2881" healthcheck: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a67141ce05..712de84c62 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -13,8 +13,8 @@ x-shared-env: &shared-api-worker-env APP_WEB_URL: ${APP_WEB_URL:-} FILES_URL: ${FILES_URL:-} INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-} - LANG: ${LANG:-en_US.UTF-8} - LC_ALL: ${LC_ALL:-en_US.UTF-8} + LANG: ${LANG:-C.UTF-8} + LC_ALL: ${LC_ALL:-C.UTF-8} PYTHONIOENCODING: ${PYTHONIOENCODING:-utf-8} LOG_LEVEL: ${LOG_LEVEL:-INFO} LOG_OUTPUT_FORMAT: ${LOG_OUTPUT_FORMAT:-text} @@ -475,6 +475,7 @@ x-shared-env: &shared-api-worker-env ALIYUN_SLS_LOGSTORE_TTL: ${ALIYUN_SLS_LOGSTORE_TTL:-365} LOGSTORE_DUAL_WRITE_ENABLED: ${LOGSTORE_DUAL_WRITE_ENABLED:-false} LOGSTORE_DUAL_READ_ENABLED: ${LOGSTORE_DUAL_READ_ENABLED:-true} + LOGSTORE_ENABLE_PUT_GRAPH_FIELD: ${LOGSTORE_ENABLE_PUT_GRAPH_FIELD:-true} HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} @@ -1156,7 +1157,8 @@ services: OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} OB_SERVER_IP: 127.0.0.1 MODE: mini - LANG: en_US.UTF-8 + LANG: C.UTF-8 + LC_ALL: C.UTF-8 ports: - "${OCEANBASE_VECTOR_PORT:-2881}:2881" healthcheck: diff --git a/docker/middleware.env.example b/docker/middleware.env.example index f7e0252a6f..c88dbe5511 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -233,4 +233,8 @@ ALIYUN_SLS_LOGSTORE_TTL=365 LOGSTORE_DUAL_WRITE_ENABLED=true # Enable dual-read fallback to SQL database when LogStore returns no results (default: true) # Useful for migration scenarios where historical data exists only in SQL database -LOGSTORE_DUAL_READ_ENABLED=true \ No newline at end of file +LOGSTORE_DUAL_READ_ENABLED=true +# Control flag for whether to write the `graph` field to LogStore. +# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; +# otherwise write an empty {} instead. Defaults to writing the `graph` field. +LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true \ No newline at end of file diff --git a/web/.dockerignore b/web/.dockerignore index 31eb66c210..91437a2259 100644 --- a/web/.dockerignore +++ b/web/.dockerignore @@ -7,8 +7,12 @@ logs # node node_modules +dist +build +coverage .husky .next +.pnpm-store # vscode .vscode @@ -22,3 +26,7 @@ node_modules # Jetbrains .idea + +# git +.git +.gitignore \ No newline at end of file diff --git a/web/.env.example b/web/.env.example index c06a4fba87..f2f25454cb 100644 --- a/web/.env.example +++ b/web/.env.example @@ -47,6 +47,8 @@ NEXT_PUBLIC_TOP_K_MAX_VALUE=10 # The maximum number of tokens for segmentation NEXT_PUBLIC_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 +# Used by web/docker/entrypoint.sh to overwrite/export NEXT_PUBLIC_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH at container startup (Docker only) +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 # Maximum loop count in the workflow NEXT_PUBLIC_LOOP_NODE_MAX_COUNT=100 diff --git a/web/Dockerfile b/web/Dockerfile index f24e9f2fc3..8697793145 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -12,7 +12,8 @@ RUN apk add --no-cache tzdata RUN corepack enable ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" -ENV NEXT_PUBLIC_BASE_PATH="" +ARG NEXT_PUBLIC_BASE_PATH="" +ENV NEXT_PUBLIC_BASE_PATH="$NEXT_PUBLIC_BASE_PATH" # install packages diff --git a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx index 8080b565cd..1d65e4de53 100644 --- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx +++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx @@ -1,11 +1,8 @@ -/* eslint-disable dify-i18n/require-ns-option */ -import * as React from 'react' +import { useTranslation } from '#i18n' import Form from '@/app/components/datasets/settings/form' -import { getLocaleOnServer, getTranslation } from '@/i18n-config/server' -const Settings = async () => { - const locale = await getLocaleOnServer() - const { t } = await getTranslation(locale, 'dataset-settings') +const Settings = () => { + const { t } = useTranslation('datasetSettings') return (
diff --git a/web/app/components/datasets/create/step-one/components/data-source-type-selector.tsx b/web/app/components/datasets/create/step-one/components/data-source-type-selector.tsx new file mode 100644 index 0000000000..6bdc2ace56 --- /dev/null +++ b/web/app/components/datasets/create/step-one/components/data-source-type-selector.tsx @@ -0,0 +1,97 @@ +'use client' + +import { useCallback, useMemo } from 'react' +import { useTranslation } from 'react-i18next' +import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config' +import { DataSourceType } from '@/models/datasets' +import { cn } from '@/utils/classnames' +import s from '../index.module.css' + +type DataSourceTypeSelectorProps = { + currentType: DataSourceType + disabled: boolean + onChange: (type: DataSourceType) => void + onClearPreviews: (type: DataSourceType) => void +} + +type DataSourceLabelKey + = | 'stepOne.dataSourceType.file' + | 'stepOne.dataSourceType.notion' + | 'stepOne.dataSourceType.web' + +type DataSourceOption = { + type: DataSourceType + iconClass?: string + labelKey: DataSourceLabelKey +} + +const DATA_SOURCE_OPTIONS: DataSourceOption[] = [ + { + type: DataSourceType.FILE, + labelKey: 'stepOne.dataSourceType.file', + }, + { + type: DataSourceType.NOTION, + iconClass: s.notion, + labelKey: 'stepOne.dataSourceType.notion', + }, + { + type: DataSourceType.WEB, + iconClass: s.web, + labelKey: 'stepOne.dataSourceType.web', + }, +] + +/** + * Data source type selector component for choosing between file, notion, and web sources. + */ +function DataSourceTypeSelector({ + currentType, + disabled, + onChange, + onClearPreviews, +}: DataSourceTypeSelectorProps) { + const { t } = useTranslation() + + const isWebEnabled = ENABLE_WEBSITE_FIRECRAWL || ENABLE_WEBSITE_JINAREADER || ENABLE_WEBSITE_WATERCRAWL + + const handleTypeChange = useCallback((type: DataSourceType) => { + if (disabled) + return + onChange(type) + onClearPreviews(type) + }, [disabled, onChange, onClearPreviews]) + + const visibleOptions = useMemo(() => DATA_SOURCE_OPTIONS.filter((option) => { + if (option.type === DataSourceType.WEB) + return isWebEnabled + return true + }), [isWebEnabled]) + + return ( +
+ {visibleOptions.map(option => ( +
handleTypeChange(option.type)} + > + + + {t(option.labelKey, { ns: 'datasetCreation' })} + +
+ ))} +
+ ) +} + +export default DataSourceTypeSelector diff --git a/web/app/components/datasets/create/step-one/components/index.ts b/web/app/components/datasets/create/step-one/components/index.ts new file mode 100644 index 0000000000..5271835741 --- /dev/null +++ b/web/app/components/datasets/create/step-one/components/index.ts @@ -0,0 +1,3 @@ +export { default as DataSourceTypeSelector } from './data-source-type-selector' +export { default as NextStepButton } from './next-step-button' +export { default as PreviewPanel } from './preview-panel' diff --git a/web/app/components/datasets/create/step-one/components/next-step-button.tsx b/web/app/components/datasets/create/step-one/components/next-step-button.tsx new file mode 100644 index 0000000000..71e4e87fcf --- /dev/null +++ b/web/app/components/datasets/create/step-one/components/next-step-button.tsx @@ -0,0 +1,30 @@ +'use client' + +import { RiArrowRightLine } from '@remixicon/react' +import { useTranslation } from 'react-i18next' +import Button from '@/app/components/base/button' + +type NextStepButtonProps = { + disabled: boolean + onClick: () => void +} + +/** + * Reusable next step button component for dataset creation flow. + */ +function NextStepButton({ disabled, onClick }: NextStepButtonProps) { + const { t } = useTranslation() + + return ( +
+ +
+ ) +} + +export default NextStepButton diff --git a/web/app/components/datasets/create/step-one/components/preview-panel.tsx b/web/app/components/datasets/create/step-one/components/preview-panel.tsx new file mode 100644 index 0000000000..8ae0b7df55 --- /dev/null +++ b/web/app/components/datasets/create/step-one/components/preview-panel.tsx @@ -0,0 +1,62 @@ +'use client' + +import type { NotionPage } from '@/models/common' +import type { CrawlResultItem } from '@/models/datasets' +import { useTranslation } from 'react-i18next' +import PlanUpgradeModal from '@/app/components/billing/plan-upgrade-modal' +import FilePreview from '../../file-preview' +import NotionPagePreview from '../../notion-page-preview' +import WebsitePreview from '../../website/preview' + +type PreviewPanelProps = { + currentFile: File | undefined + currentNotionPage: NotionPage | undefined + currentWebsite: CrawlResultItem | undefined + notionCredentialId: string + isShowPlanUpgradeModal: boolean + hideFilePreview: () => void + hideNotionPagePreview: () => void + hideWebsitePreview: () => void + hidePlanUpgradeModal: () => void +} + +/** + * Right panel component for displaying file, notion page, or website previews. + */ +function PreviewPanel({ + currentFile, + currentNotionPage, + currentWebsite, + notionCredentialId, + isShowPlanUpgradeModal, + hideFilePreview, + hideNotionPagePreview, + hideWebsitePreview, + hidePlanUpgradeModal, +}: PreviewPanelProps) { + const { t } = useTranslation() + + return ( +
+ {currentFile && } + {currentNotionPage && ( + + )} + {currentWebsite && } + {isShowPlanUpgradeModal && ( + + )} +
+ ) +} + +export default PreviewPanel diff --git a/web/app/components/datasets/create/step-one/hooks/index.ts b/web/app/components/datasets/create/step-one/hooks/index.ts new file mode 100644 index 0000000000..bae5ce4fce --- /dev/null +++ b/web/app/components/datasets/create/step-one/hooks/index.ts @@ -0,0 +1,2 @@ +export { default as usePreviewState } from './use-preview-state' +export type { PreviewActions, PreviewState, UsePreviewStateReturn } from './use-preview-state' diff --git a/web/app/components/datasets/create/step-one/hooks/use-preview-state.ts b/web/app/components/datasets/create/step-one/hooks/use-preview-state.ts new file mode 100644 index 0000000000..3984947ab1 --- /dev/null +++ b/web/app/components/datasets/create/step-one/hooks/use-preview-state.ts @@ -0,0 +1,70 @@ +'use client' + +import type { NotionPage } from '@/models/common' +import type { CrawlResultItem } from '@/models/datasets' +import { useCallback, useState } from 'react' + +export type PreviewState = { + currentFile: File | undefined + currentNotionPage: NotionPage | undefined + currentWebsite: CrawlResultItem | undefined +} + +export type PreviewActions = { + showFilePreview: (file: File) => void + hideFilePreview: () => void + showNotionPagePreview: (page: NotionPage) => void + hideNotionPagePreview: () => void + showWebsitePreview: (website: CrawlResultItem) => void + hideWebsitePreview: () => void +} + +export type UsePreviewStateReturn = PreviewState & PreviewActions + +/** + * Custom hook for managing preview state across different data source types. + * Handles file, notion page, and website preview visibility. + */ +function usePreviewState(): UsePreviewStateReturn { + const [currentFile, setCurrentFile] = useState() + const [currentNotionPage, setCurrentNotionPage] = useState() + const [currentWebsite, setCurrentWebsite] = useState() + + const showFilePreview = useCallback((file: File) => { + setCurrentFile(file) + }, []) + + const hideFilePreview = useCallback(() => { + setCurrentFile(undefined) + }, []) + + const showNotionPagePreview = useCallback((page: NotionPage) => { + setCurrentNotionPage(page) + }, []) + + const hideNotionPagePreview = useCallback(() => { + setCurrentNotionPage(undefined) + }, []) + + const showWebsitePreview = useCallback((website: CrawlResultItem) => { + setCurrentWebsite(website) + }, []) + + const hideWebsitePreview = useCallback(() => { + setCurrentWebsite(undefined) + }, []) + + return { + currentFile, + currentNotionPage, + currentWebsite, + showFilePreview, + hideFilePreview, + showNotionPagePreview, + hideNotionPagePreview, + showWebsitePreview, + hideWebsitePreview, + } +} + +export default usePreviewState diff --git a/web/app/components/datasets/create/step-one/index.spec.tsx b/web/app/components/datasets/create/step-one/index.spec.tsx new file mode 100644 index 0000000000..1ff77dc1f6 --- /dev/null +++ b/web/app/components/datasets/create/step-one/index.spec.tsx @@ -0,0 +1,1204 @@ +import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types' +import type { NotionPage } from '@/models/common' +import type { CrawlOptions, CrawlResultItem, DataSet, FileItem } from '@/models/datasets' +import { act, fireEvent, render, renderHook, screen } from '@testing-library/react' +import { Plan } from '@/app/components/billing/type' +import { DataSourceType } from '@/models/datasets' +import { DataSourceTypeSelector, NextStepButton, PreviewPanel } from './components' +import { usePreviewState } from './hooks' +import StepOne from './index' + +// ========================================== +// Mock External Dependencies +// ========================================== + +// Mock config for website crawl features +vi.mock('@/config', () => ({ + ENABLE_WEBSITE_FIRECRAWL: true, + ENABLE_WEBSITE_JINAREADER: false, + ENABLE_WEBSITE_WATERCRAWL: false, +})) + +// Mock dataset detail context +let mockDatasetDetail: DataSet | undefined +vi.mock('@/context/dataset-detail', () => ({ + useDatasetDetailContextWithSelector: (selector: (state: { dataset: DataSet | undefined }) => DataSet | undefined) => { + return selector({ dataset: mockDatasetDetail }) + }, +})) + +// Mock provider context +let mockPlan = { + type: Plan.professional, + usage: { vectorSpace: 50, buildApps: 0, documentsUploadQuota: 0, vectorStorageQuota: 0 }, + total: { vectorSpace: 100, buildApps: 0, documentsUploadQuota: 0, vectorStorageQuota: 0 }, +} +let mockEnableBilling = false + +vi.mock('@/context/provider-context', () => ({ + useProviderContext: () => ({ + plan: mockPlan, + enableBilling: mockEnableBilling, + }), +})) + +// Mock child components +vi.mock('../file-uploader', () => ({ + default: ({ onPreview, fileList }: { onPreview: (file: File) => void, fileList: FileItem[] }) => ( +
+ {fileList.length} + +
+ ), +})) + +vi.mock('../website', () => ({ + default: ({ onPreview }: { onPreview: (item: CrawlResultItem) => void }) => ( +
+ +
+ ), +})) + +vi.mock('../empty-dataset-creation-modal', () => ({ + default: ({ show, onHide }: { show: boolean, onHide: () => void }) => ( + show + ? ( +
+ +
+ ) + : null + ), +})) + +// NotionConnector is a base component - imported directly without mock +// It only depends on i18n which is globally mocked + +vi.mock('@/app/components/base/notion-page-selector', () => ({ + NotionPageSelector: ({ onPreview }: { onPreview: (page: NotionPage) => void }) => ( +
+ +
+ ), +})) + +vi.mock('@/app/components/billing/vector-space-full', () => ({ + default: () =>
Vector Space Full
, +})) + +vi.mock('@/app/components/billing/plan-upgrade-modal', () => ({ + default: ({ show, onClose }: { show: boolean, onClose: () => void }) => ( + show + ? ( +
+ +
+ ) + : null + ), +})) + +vi.mock('../file-preview', () => ({ + default: ({ file, hidePreview }: { file: File, hidePreview: () => void }) => ( +
+ {file.name} + +
+ ), +})) + +vi.mock('../notion-page-preview', () => ({ + default: ({ currentPage, hidePreview }: { currentPage: NotionPage, hidePreview: () => void }) => ( +
+ {currentPage.page_id} + +
+ ), +})) + +// WebsitePreview is a sibling component without API dependencies - imported directly +// It only depends on i18n which is globally mocked + +vi.mock('./upgrade-card', () => ({ + default: () =>
Upgrade Card
, +})) + +// ========================================== +// Test Data Builders +// ========================================== + +const createMockCustomFile = (overrides: { id?: string, name?: string } = {}) => { + const file = new File(['test content'], overrides.name ?? 'test.txt', { type: 'text/plain' }) + return Object.assign(file, { + id: overrides.id ?? 'uploaded-id', + extension: 'txt', + mime_type: 'text/plain', + created_by: 'user-1', + created_at: Date.now(), + }) +} + +const createMockFileItem = (overrides: Partial = {}): FileItem => ({ + fileID: `file-${Date.now()}`, + file: createMockCustomFile(overrides.file as { id?: string, name?: string }), + progress: 100, + ...overrides, +}) + +const createMockNotionPage = (overrides: Partial = {}): NotionPage => ({ + page_id: `page-${Date.now()}`, + type: 'page', + ...overrides, +} as NotionPage) + +const createMockCrawlResult = (overrides: Partial = {}): CrawlResultItem => ({ + title: 'Test Page', + markdown: 'Test content', + description: 'Test description', + source_url: 'https://example.com', + ...overrides, +}) + +const createMockDataSourceAuth = (overrides: Partial = {}): DataSourceAuth => ({ + credential_id: 'cred-1', + provider: 'notion_datasource', + plugin_id: 'plugin-1', + credentials_list: [{ id: 'cred-1', name: 'Workspace 1' }], + ...overrides, +} as DataSourceAuth) + +const defaultProps = { + dataSourceType: DataSourceType.FILE, + dataSourceTypeDisable: false, + onSetting: vi.fn(), + files: [] as FileItem[], + updateFileList: vi.fn(), + updateFile: vi.fn(), + notionPages: [] as NotionPage[], + notionCredentialId: '', + updateNotionPages: vi.fn(), + updateNotionCredentialId: vi.fn(), + onStepChange: vi.fn(), + changeType: vi.fn(), + websitePages: [] as CrawlResultItem[], + updateWebsitePages: vi.fn(), + onWebsiteCrawlProviderChange: vi.fn(), + onWebsiteCrawlJobIdChange: vi.fn(), + crawlOptions: { + crawl_sub_pages: true, + only_main_content: true, + includes: '', + excludes: '', + limit: 10, + max_depth: '', + use_sitemap: true, + } as CrawlOptions, + onCrawlOptionsChange: vi.fn(), + authedDataSourceList: [] as DataSourceAuth[], +} + +// ========================================== +// usePreviewState Hook Tests +// ========================================== +describe('usePreviewState Hook', () => { + // -------------------------------------------------------------------------- + // Initial State Tests + // -------------------------------------------------------------------------- + describe('Initial State', () => { + it('should initialize with all preview states undefined', () => { + // Arrange & Act + const { result } = renderHook(() => usePreviewState()) + + // Assert + expect(result.current.currentFile).toBeUndefined() + expect(result.current.currentNotionPage).toBeUndefined() + expect(result.current.currentWebsite).toBeUndefined() + }) + }) + + // -------------------------------------------------------------------------- + // File Preview Tests + // -------------------------------------------------------------------------- + describe('File Preview', () => { + it('should show file preview when showFilePreview is called', () => { + // Arrange + const { result } = renderHook(() => usePreviewState()) + const mockFile = new File(['test'], 'test.txt') + + // Act + act(() => { + result.current.showFilePreview(mockFile) + }) + + // Assert + expect(result.current.currentFile).toBe(mockFile) + }) + + it('should hide file preview when hideFilePreview is called', () => { + // Arrange + const { result } = renderHook(() => usePreviewState()) + const mockFile = new File(['test'], 'test.txt') + + act(() => { + result.current.showFilePreview(mockFile) + }) + + // Act + act(() => { + result.current.hideFilePreview() + }) + + // Assert + expect(result.current.currentFile).toBeUndefined() + }) + }) + + // -------------------------------------------------------------------------- + // Notion Page Preview Tests + // -------------------------------------------------------------------------- + describe('Notion Page Preview', () => { + it('should show notion page preview when showNotionPagePreview is called', () => { + // Arrange + const { result } = renderHook(() => usePreviewState()) + const mockPage = createMockNotionPage() + + // Act + act(() => { + result.current.showNotionPagePreview(mockPage) + }) + + // Assert + expect(result.current.currentNotionPage).toBe(mockPage) + }) + + it('should hide notion page preview when hideNotionPagePreview is called', () => { + // Arrange + const { result } = renderHook(() => usePreviewState()) + const mockPage = createMockNotionPage() + + act(() => { + result.current.showNotionPagePreview(mockPage) + }) + + // Act + act(() => { + result.current.hideNotionPagePreview() + }) + + // Assert + expect(result.current.currentNotionPage).toBeUndefined() + }) + }) + + // -------------------------------------------------------------------------- + // Website Preview Tests + // -------------------------------------------------------------------------- + describe('Website Preview', () => { + it('should show website preview when showWebsitePreview is called', () => { + // Arrange + const { result } = renderHook(() => usePreviewState()) + const mockWebsite = createMockCrawlResult() + + // Act + act(() => { + result.current.showWebsitePreview(mockWebsite) + }) + + // Assert + expect(result.current.currentWebsite).toBe(mockWebsite) + }) + + it('should hide website preview when hideWebsitePreview is called', () => { + // Arrange + const { result } = renderHook(() => usePreviewState()) + const mockWebsite = createMockCrawlResult() + + act(() => { + result.current.showWebsitePreview(mockWebsite) + }) + + // Act + act(() => { + result.current.hideWebsitePreview() + }) + + // Assert + expect(result.current.currentWebsite).toBeUndefined() + }) + }) + + // -------------------------------------------------------------------------- + // Callback Stability Tests (Memoization) + // -------------------------------------------------------------------------- + describe('Callback Stability', () => { + it('should maintain stable showFilePreview callback reference', () => { + // Arrange + const { result, rerender } = renderHook(() => usePreviewState()) + const initialCallback = result.current.showFilePreview + + // Act + rerender() + + // Assert + expect(result.current.showFilePreview).toBe(initialCallback) + }) + + it('should maintain stable hideFilePreview callback reference', () => { + // Arrange + const { result, rerender } = renderHook(() => usePreviewState()) + const initialCallback = result.current.hideFilePreview + + // Act + rerender() + + // Assert + expect(result.current.hideFilePreview).toBe(initialCallback) + }) + + it('should maintain stable showNotionPagePreview callback reference', () => { + // Arrange + const { result, rerender } = renderHook(() => usePreviewState()) + const initialCallback = result.current.showNotionPagePreview + + // Act + rerender() + + // Assert + expect(result.current.showNotionPagePreview).toBe(initialCallback) + }) + + it('should maintain stable hideNotionPagePreview callback reference', () => { + // Arrange + const { result, rerender } = renderHook(() => usePreviewState()) + const initialCallback = result.current.hideNotionPagePreview + + // Act + rerender() + + // Assert + expect(result.current.hideNotionPagePreview).toBe(initialCallback) + }) + + it('should maintain stable showWebsitePreview callback reference', () => { + // Arrange + const { result, rerender } = renderHook(() => usePreviewState()) + const initialCallback = result.current.showWebsitePreview + + // Act + rerender() + + // Assert + expect(result.current.showWebsitePreview).toBe(initialCallback) + }) + + it('should maintain stable hideWebsitePreview callback reference', () => { + // Arrange + const { result, rerender } = renderHook(() => usePreviewState()) + const initialCallback = result.current.hideWebsitePreview + + // Act + rerender() + + // Assert + expect(result.current.hideWebsitePreview).toBe(initialCallback) + }) + }) +}) + +// ========================================== +// DataSourceTypeSelector Component Tests +// ========================================== +describe('DataSourceTypeSelector', () => { + const defaultSelectorProps = { + currentType: DataSourceType.FILE, + disabled: false, + onChange: vi.fn(), + onClearPreviews: vi.fn(), + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + // -------------------------------------------------------------------------- + // Rendering Tests + // -------------------------------------------------------------------------- + describe('Rendering', () => { + it('should render all data source options when web is enabled', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText('datasetCreation.stepOne.dataSourceType.file')).toBeInTheDocument() + expect(screen.getByText('datasetCreation.stepOne.dataSourceType.notion')).toBeInTheDocument() + expect(screen.getByText('datasetCreation.stepOne.dataSourceType.web')).toBeInTheDocument() + }) + + it('should highlight active type', () => { + // Arrange & Act + const { container } = render( + , + ) + + // Assert - The active item should have the active class + const items = container.querySelectorAll('[class*="dataSourceItem"]') + expect(items.length).toBeGreaterThan(0) + }) + }) + + // -------------------------------------------------------------------------- + // User Interactions Tests + // -------------------------------------------------------------------------- + describe('User Interactions', () => { + it('should call onChange when a type is clicked', () => { + // Arrange + const onChange = vi.fn() + render() + + // Act + fireEvent.click(screen.getByText('datasetCreation.stepOne.dataSourceType.notion')) + + // Assert + expect(onChange).toHaveBeenCalledWith(DataSourceType.NOTION) + }) + + it('should call onClearPreviews when a type is clicked', () => { + // Arrange + const onClearPreviews = vi.fn() + render() + + // Act + fireEvent.click(screen.getByText('datasetCreation.stepOne.dataSourceType.web')) + + // Assert + expect(onClearPreviews).toHaveBeenCalledWith(DataSourceType.WEB) + }) + + it('should not call onChange when disabled', () => { + // Arrange + const onChange = vi.fn() + render() + + // Act + fireEvent.click(screen.getByText('datasetCreation.stepOne.dataSourceType.notion')) + + // Assert + expect(onChange).not.toHaveBeenCalled() + }) + + it('should not call onClearPreviews when disabled', () => { + // Arrange + const onClearPreviews = vi.fn() + render() + + // Act + fireEvent.click(screen.getByText('datasetCreation.stepOne.dataSourceType.notion')) + + // Assert + expect(onClearPreviews).not.toHaveBeenCalled() + }) + }) +}) + +// ========================================== +// NextStepButton Component Tests +// ========================================== +describe('NextStepButton', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + // -------------------------------------------------------------------------- + // Rendering Tests + // -------------------------------------------------------------------------- + describe('Rendering', () => { + it('should render with correct label', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText('datasetCreation.stepOne.button')).toBeInTheDocument() + }) + + it('should render with arrow icon', () => { + // Arrange & Act + const { container } = render() + + // Assert + const svgIcon = container.querySelector('svg') + expect(svgIcon).toBeInTheDocument() + }) + }) + + // -------------------------------------------------------------------------- + // Props Tests + // -------------------------------------------------------------------------- + describe('Props', () => { + it('should be disabled when disabled prop is true', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByRole('button')).toBeDisabled() + }) + + it('should be enabled when disabled prop is false', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByRole('button')).not.toBeDisabled() + }) + + it('should call onClick when clicked and not disabled', () => { + // Arrange + const onClick = vi.fn() + render() + + // Act + fireEvent.click(screen.getByRole('button')) + + // Assert + expect(onClick).toHaveBeenCalledTimes(1) + }) + + it('should not call onClick when clicked and disabled', () => { + // Arrange + const onClick = vi.fn() + render() + + // Act + fireEvent.click(screen.getByRole('button')) + + // Assert + expect(onClick).not.toHaveBeenCalled() + }) + }) +}) + +// ========================================== +// PreviewPanel Component Tests +// ========================================== +describe('PreviewPanel', () => { + const defaultPreviewProps = { + currentFile: undefined as File | undefined, + currentNotionPage: undefined as NotionPage | undefined, + currentWebsite: undefined as CrawlResultItem | undefined, + notionCredentialId: 'cred-1', + isShowPlanUpgradeModal: false, + hideFilePreview: vi.fn(), + hideNotionPagePreview: vi.fn(), + hideWebsitePreview: vi.fn(), + hidePlanUpgradeModal: vi.fn(), + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + // -------------------------------------------------------------------------- + // Conditional Rendering Tests + // -------------------------------------------------------------------------- + describe('Conditional Rendering', () => { + it('should not render FilePreview when currentFile is undefined', () => { + // Arrange & Act + render() + + // Assert + expect(screen.queryByTestId('file-preview')).not.toBeInTheDocument() + }) + + it('should render FilePreview when currentFile is defined', () => { + // Arrange + const file = new File(['test'], 'test.txt') + + // Act + render() + + // Assert + expect(screen.getByTestId('file-preview')).toBeInTheDocument() + }) + + it('should not render NotionPagePreview when currentNotionPage is undefined', () => { + // Arrange & Act + render() + + // Assert + expect(screen.queryByTestId('notion-page-preview')).not.toBeInTheDocument() + }) + + it('should render NotionPagePreview when currentNotionPage is defined', () => { + // Arrange + const page = createMockNotionPage() + + // Act + render() + + // Assert + expect(screen.getByTestId('notion-page-preview')).toBeInTheDocument() + }) + + it('should not render WebsitePreview when currentWebsite is undefined', () => { + // Arrange & Act + render() + + // Assert - pagePreview is the title shown in WebsitePreview + expect(screen.queryByText('datasetCreation.stepOne.pagePreview')).not.toBeInTheDocument() + }) + + it('should render WebsitePreview when currentWebsite is defined', () => { + // Arrange + const website = createMockCrawlResult() + + // Act + render() + + // Assert - Check for the preview title and source URL + expect(screen.getByText('datasetCreation.stepOne.pagePreview')).toBeInTheDocument() + expect(screen.getByText(website.source_url)).toBeInTheDocument() + }) + + it('should not render PlanUpgradeModal when isShowPlanUpgradeModal is false', () => { + // Arrange & Act + render() + + // Assert + expect(screen.queryByTestId('plan-upgrade-modal')).not.toBeInTheDocument() + }) + + it('should render PlanUpgradeModal when isShowPlanUpgradeModal is true', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByTestId('plan-upgrade-modal')).toBeInTheDocument() + }) + }) + + // -------------------------------------------------------------------------- + // Event Handler Tests + // -------------------------------------------------------------------------- + describe('Event Handlers', () => { + it('should call hideFilePreview when file preview close is clicked', () => { + // Arrange + const hideFilePreview = vi.fn() + const file = new File(['test'], 'test.txt') + render() + + // Act + fireEvent.click(screen.getByTestId('hide-file-preview')) + + // Assert + expect(hideFilePreview).toHaveBeenCalledTimes(1) + }) + + it('should call hideNotionPagePreview when notion preview close is clicked', () => { + // Arrange + const hideNotionPagePreview = vi.fn() + const page = createMockNotionPage() + render() + + // Act + fireEvent.click(screen.getByTestId('hide-notion-preview')) + + // Assert + expect(hideNotionPagePreview).toHaveBeenCalledTimes(1) + }) + + it('should call hideWebsitePreview when website preview close is clicked', () => { + // Arrange + const hideWebsitePreview = vi.fn() + const website = createMockCrawlResult() + const { container } = render() + + // Act - Find the close button (div with cursor-pointer class containing the XMarkIcon) + const closeButton = container.querySelector('.cursor-pointer') + expect(closeButton).toBeInTheDocument() + fireEvent.click(closeButton!) + + // Assert + expect(hideWebsitePreview).toHaveBeenCalledTimes(1) + }) + + it('should call hidePlanUpgradeModal when modal close is clicked', () => { + // Arrange + const hidePlanUpgradeModal = vi.fn() + render() + + // Act + fireEvent.click(screen.getByTestId('close-upgrade-modal')) + + // Assert + expect(hidePlanUpgradeModal).toHaveBeenCalledTimes(1) + }) + }) +}) + +// ========================================== +// StepOne Component Tests +// ========================================== +describe('StepOne', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDatasetDetail = undefined + mockPlan = { + type: Plan.professional, + usage: { vectorSpace: 50, buildApps: 0, documentsUploadQuota: 0, vectorStorageQuota: 0 }, + total: { vectorSpace: 100, buildApps: 0, documentsUploadQuota: 0, vectorStorageQuota: 0 }, + } + mockEnableBilling = false + }) + + // -------------------------------------------------------------------------- + // Rendering Tests + // -------------------------------------------------------------------------- + describe('Rendering', () => { + it('should render without crashing', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText('datasetCreation.steps.one')).toBeInTheDocument() + }) + + it('should render DataSourceTypeSelector when not editing existing dataset', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText('datasetCreation.stepOne.dataSourceType.file')).toBeInTheDocument() + }) + + it('should render FileUploader when dataSourceType is FILE', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByTestId('file-uploader')).toBeInTheDocument() + }) + + it('should render NotionConnector when dataSourceType is NOTION and not authenticated', () => { + // Arrange & Act + render() + + // Assert - NotionConnector shows sync title and connect button + expect(screen.getByText('datasetCreation.stepOne.notionSyncTitle')).toBeInTheDocument() + expect(screen.getByRole('button', { name: /datasetCreation.stepOne.connect/i })).toBeInTheDocument() + }) + + it('should render NotionPageSelector when dataSourceType is NOTION and authenticated', () => { + // Arrange + const authedDataSourceList = [createMockDataSourceAuth()] + + // Act + render() + + // Assert + expect(screen.getByTestId('notion-page-selector')).toBeInTheDocument() + }) + + it('should render Website when dataSourceType is WEB', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByTestId('website')).toBeInTheDocument() + }) + + it('should render empty dataset creation link when no datasetId', () => { + // Arrange & Act + render() + + // Assert + expect(screen.getByText('datasetCreation.stepOne.emptyDatasetCreation')).toBeInTheDocument() + }) + + it('should not render empty dataset creation link when datasetId exists', () => { + // Arrange & Act + render() + + // Assert + expect(screen.queryByText('datasetCreation.stepOne.emptyDatasetCreation')).not.toBeInTheDocument() + }) + }) + + // -------------------------------------------------------------------------- + // Props Tests + // -------------------------------------------------------------------------- + describe('Props', () => { + it('should pass files to FileUploader', () => { + // Arrange + const files = [createMockFileItem()] + + // Act + render() + + // Assert + expect(screen.getByTestId('file-count')).toHaveTextContent('1') + }) + + it('should call onSetting when NotionConnector connect button is clicked', () => { + // Arrange + const onSetting = vi.fn() + render() + + // Act - The NotionConnector's button calls onSetting + fireEvent.click(screen.getByRole('button', { name: /datasetCreation.stepOne.connect/i })) + + // Assert + expect(onSetting).toHaveBeenCalledTimes(1) + }) + + it('should call changeType when data source type is changed', () => { + // Arrange + const changeType = vi.fn() + render() + + // Act + fireEvent.click(screen.getByText('datasetCreation.stepOne.dataSourceType.notion')) + + // Assert + expect(changeType).toHaveBeenCalledWith(DataSourceType.NOTION) + }) + }) + + // -------------------------------------------------------------------------- + // State Management Tests + // -------------------------------------------------------------------------- + describe('State Management', () => { + it('should open empty dataset modal when link is clicked', () => { + // Arrange + render() + + // Act + fireEvent.click(screen.getByText('datasetCreation.stepOne.emptyDatasetCreation')) + + // Assert + expect(screen.getByTestId('empty-dataset-modal')).toBeInTheDocument() + }) + + it('should close empty dataset modal when close is clicked', () => { + // Arrange + render() + fireEvent.click(screen.getByText('datasetCreation.stepOne.emptyDatasetCreation')) + + // Act + fireEvent.click(screen.getByTestId('close-modal')) + + // Assert + expect(screen.queryByTestId('empty-dataset-modal')).not.toBeInTheDocument() + }) + }) + + // -------------------------------------------------------------------------- + // Memoization Tests + // -------------------------------------------------------------------------- + describe('Memoization', () => { + it('should correctly compute isNotionAuthed based on authedDataSourceList', () => { + // Arrange - No auth + const { rerender } = render() + // NotionConnector shows the sync title when not authenticated + expect(screen.getByText('datasetCreation.stepOne.notionSyncTitle')).toBeInTheDocument() + + // Act - Add auth + const authedDataSourceList = [createMockDataSourceAuth()] + rerender() + + // Assert + expect(screen.getByTestId('notion-page-selector')).toBeInTheDocument() + }) + + it('should correctly compute fileNextDisabled when files are empty', () => { + // Arrange & Act + render() + + // Assert - Button should be disabled + expect(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })).toBeDisabled() + }) + + it('should correctly compute fileNextDisabled when files are loaded', () => { + // Arrange + const files = [createMockFileItem()] + + // Act + render() + + // Assert - Button should be enabled + expect(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })).not.toBeDisabled() + }) + + it('should correctly compute fileNextDisabled when some files are not uploaded', () => { + // Arrange - Create a file item without id (not yet uploaded) + const file = new File(['test'], 'test.txt', { type: 'text/plain' }) + const fileItem: FileItem = { + fileID: 'temp-id', + file: Object.assign(file, { id: undefined, extension: 'txt', mime_type: 'text/plain' }), + progress: 0, + } + + // Act + render() + + // Assert - Button should be disabled + expect(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })).toBeDisabled() + }) + }) + + // -------------------------------------------------------------------------- + // Callback Tests + // -------------------------------------------------------------------------- + describe('Callbacks', () => { + it('should call onStepChange when next button is clicked with valid files', () => { + // Arrange + const onStepChange = vi.fn() + const files = [createMockFileItem()] + render() + + // Act + fireEvent.click(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })) + + // Assert + expect(onStepChange).toHaveBeenCalledTimes(1) + }) + + it('should show plan upgrade modal when batch upload not supported and multiple files', () => { + // Arrange + mockEnableBilling = true + mockPlan.type = Plan.sandbox + const files = [createMockFileItem(), createMockFileItem()] + render() + + // Act + fireEvent.click(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })) + + // Assert + expect(screen.getByTestId('plan-upgrade-modal')).toBeInTheDocument() + }) + + it('should show upgrade card when in sandbox plan with files', () => { + // Arrange + mockEnableBilling = true + mockPlan.type = Plan.sandbox + const files = [createMockFileItem()] + + // Act + render() + + // Assert + expect(screen.getByTestId('upgrade-card')).toBeInTheDocument() + }) + }) + + // -------------------------------------------------------------------------- + // Vector Space Full Tests + // -------------------------------------------------------------------------- + describe('Vector Space Full', () => { + it('should show VectorSpaceFull when vector space is full and billing is enabled', () => { + // Arrange + mockEnableBilling = true + mockPlan.usage.vectorSpace = 100 + mockPlan.total.vectorSpace = 100 + const files = [createMockFileItem()] + + // Act + render() + + // Assert + expect(screen.getByTestId('vector-space-full')).toBeInTheDocument() + }) + + it('should disable next button when vector space is full', () => { + // Arrange + mockEnableBilling = true + mockPlan.usage.vectorSpace = 100 + mockPlan.total.vectorSpace = 100 + const files = [createMockFileItem()] + + // Act + render() + + // Assert + expect(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })).toBeDisabled() + }) + }) + + // -------------------------------------------------------------------------- + // Preview Integration Tests + // -------------------------------------------------------------------------- + describe('Preview Integration', () => { + it('should show file preview when file preview button is clicked', () => { + // Arrange + render() + + // Act + fireEvent.click(screen.getByTestId('preview-file')) + + // Assert + expect(screen.getByTestId('file-preview')).toBeInTheDocument() + }) + + it('should hide file preview when hide button is clicked', () => { + // Arrange + render() + fireEvent.click(screen.getByTestId('preview-file')) + + // Act + fireEvent.click(screen.getByTestId('hide-file-preview')) + + // Assert + expect(screen.queryByTestId('file-preview')).not.toBeInTheDocument() + }) + + it('should show notion page preview when preview button is clicked', () => { + // Arrange + const authedDataSourceList = [createMockDataSourceAuth()] + render() + + // Act + fireEvent.click(screen.getByTestId('preview-notion')) + + // Assert + expect(screen.getByTestId('notion-page-preview')).toBeInTheDocument() + }) + + it('should show website preview when preview button is clicked', () => { + // Arrange + render() + + // Act + fireEvent.click(screen.getByTestId('preview-website')) + + // Assert - Check for pagePreview title which is shown by WebsitePreview + expect(screen.getByText('datasetCreation.stepOne.pagePreview')).toBeInTheDocument() + }) + }) + + // -------------------------------------------------------------------------- + // Edge Cases + // -------------------------------------------------------------------------- + describe('Edge Cases', () => { + it('should handle empty notionPages array', () => { + // Arrange + const authedDataSourceList = [createMockDataSourceAuth()] + + // Act + render() + + // Assert - Button should be disabled when no pages selected + expect(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })).toBeDisabled() + }) + + it('should handle empty websitePages array', () => { + // Arrange & Act + render() + + // Assert - Button should be disabled when no pages crawled + expect(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })).toBeDisabled() + }) + + it('should handle empty authedDataSourceList', () => { + // Arrange & Act + render() + + // Assert - Should show NotionConnector with connect button + expect(screen.getByText('datasetCreation.stepOne.notionSyncTitle')).toBeInTheDocument() + }) + + it('should handle authedDataSourceList without notion credentials', () => { + // Arrange + const authedDataSourceList = [createMockDataSourceAuth({ credentials_list: [] })] + + // Act + render() + + // Assert - Should show NotionConnector with connect button + expect(screen.getByText('datasetCreation.stepOne.notionSyncTitle')).toBeInTheDocument() + }) + + it('should clear previews when switching data source types', () => { + // Arrange + render() + fireEvent.click(screen.getByTestId('preview-file')) + expect(screen.getByTestId('file-preview')).toBeInTheDocument() + + // Act - Change to NOTION + fireEvent.click(screen.getByText('datasetCreation.stepOne.dataSourceType.notion')) + + // Assert - File preview should be cleared + expect(screen.queryByTestId('file-preview')).not.toBeInTheDocument() + }) + }) + + // -------------------------------------------------------------------------- + // Integration Tests + // -------------------------------------------------------------------------- + describe('Integration', () => { + it('should complete file upload flow', () => { + // Arrange + const onStepChange = vi.fn() + const files = [createMockFileItem()] + + // Act + render() + fireEvent.click(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })) + + // Assert + expect(onStepChange).toHaveBeenCalled() + }) + + it('should complete notion page selection flow', () => { + // Arrange + const onStepChange = vi.fn() + const authedDataSourceList = [createMockDataSourceAuth()] + const notionPages = [createMockNotionPage()] + + // Act + render( + , + ) + fireEvent.click(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })) + + // Assert + expect(onStepChange).toHaveBeenCalled() + }) + + it('should complete website crawl flow', () => { + // Arrange + const onStepChange = vi.fn() + const websitePages = [createMockCrawlResult()] + + // Act + render( + , + ) + fireEvent.click(screen.getByRole('button', { name: /datasetCreation.stepOne.button/i })) + + // Assert + expect(onStepChange).toHaveBeenCalled() + }) + }) +}) diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index 5c74e69e7e..a86c9d86c2 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -1,29 +1,25 @@ 'use client' + import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types' import type { DataSourceProvider, NotionPage } from '@/models/common' import type { CrawlOptions, CrawlResultItem, FileItem } from '@/models/datasets' -import { RiArrowRightLine, RiFolder6Line } from '@remixicon/react' +import { RiFolder6Line } from '@remixicon/react' import { useBoolean } from 'ahooks' -import * as React from 'react' -import { useCallback, useMemo, useState } from 'react' +import { useCallback, useMemo } from 'react' import { useTranslation } from 'react-i18next' -import Button from '@/app/components/base/button' import NotionConnector from '@/app/components/base/notion-connector' import { NotionPageSelector } from '@/app/components/base/notion-page-selector' -import PlanUpgradeModal from '@/app/components/billing/plan-upgrade-modal' import { Plan } from '@/app/components/billing/type' import VectorSpaceFull from '@/app/components/billing/vector-space-full' -import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' import { useProviderContext } from '@/context/provider-context' import { DataSourceType } from '@/models/datasets' import { cn } from '@/utils/classnames' import EmptyDatasetCreationModal from '../empty-dataset-creation-modal' -import FilePreview from '../file-preview' import FileUploader from '../file-uploader' -import NotionPagePreview from '../notion-page-preview' import Website from '../website' -import WebsitePreview from '../website/preview' +import { DataSourceTypeSelector, NextStepButton, PreviewPanel } from './components' +import { usePreviewState } from './hooks' import s from './index.module.css' import UpgradeCard from './upgrade-card' @@ -50,6 +46,24 @@ type IStepOneProps = { authedDataSourceList: DataSourceAuth[] } +// Helper function to check if notion is authenticated +function checkNotionAuth(authedDataSourceList: DataSourceAuth[]): boolean { + const notionSource = authedDataSourceList.find(item => item.provider === 'notion_datasource') + return Boolean(notionSource && notionSource.credentials_list.length > 0) +} + +// Helper function to get notion credential list +function getNotionCredentialList(authedDataSourceList: DataSourceAuth[]) { + return authedDataSourceList.find(item => item.provider === 'notion_datasource')?.credentials_list || [] +} + +// Lookup table for checking multiple items by data source type +const MULTIPLE_ITEMS_CHECK: Record boolean> = { + [DataSourceType.FILE]: ({ files }) => files.length > 1, + [DataSourceType.NOTION]: ({ notionPages }) => notionPages.length > 1, + [DataSourceType.WEB]: ({ websitePages }) => websitePages.length > 1, +} + const StepOne = ({ datasetId, dataSourceType: inCreatePageDataSourceType, @@ -72,76 +86,47 @@ const StepOne = ({ onCrawlOptionsChange, authedDataSourceList, }: IStepOneProps) => { - const dataset = useDatasetDetailContextWithSelector(state => state.dataset) - const [showModal, setShowModal] = useState(false) - const [currentFile, setCurrentFile] = useState() - const [currentNotionPage, setCurrentNotionPage] = useState() - const [currentWebsite, setCurrentWebsite] = useState() const { t } = useTranslation() + const dataset = useDatasetDetailContextWithSelector(state => state.dataset) + const { plan, enableBilling } = useProviderContext() - const modalShowHandle = () => setShowModal(true) - const modalCloseHandle = () => setShowModal(false) + // Preview state management + const { + currentFile, + currentNotionPage, + currentWebsite, + showFilePreview, + hideFilePreview, + showNotionPagePreview, + hideNotionPagePreview, + showWebsitePreview, + hideWebsitePreview, + } = usePreviewState() - const updateCurrentFile = useCallback((file: File) => { - setCurrentFile(file) - }, []) + // Empty dataset modal state + const [showModal, { setTrue: openModal, setFalse: closeModal }] = useBoolean(false) - const hideFilePreview = useCallback(() => { - setCurrentFile(undefined) - }, []) - - const updateCurrentPage = useCallback((page: NotionPage) => { - setCurrentNotionPage(page) - }, []) - - const hideNotionPagePreview = useCallback(() => { - setCurrentNotionPage(undefined) - }, []) - - const updateWebsite = useCallback((website: CrawlResultItem) => { - setCurrentWebsite(website) - }, []) - - const hideWebsitePreview = useCallback(() => { - setCurrentWebsite(undefined) - }, []) + // Plan upgrade modal state + const [isShowPlanUpgradeModal, { setTrue: showPlanUpgradeModal, setFalse: hidePlanUpgradeModal }] = useBoolean(false) + // Computed values const shouldShowDataSourceTypeList = !datasetId || (datasetId && !dataset?.data_source_type) const isInCreatePage = shouldShowDataSourceTypeList - const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : dataset?.data_source_type - const { plan, enableBilling } = useProviderContext() - const allFileLoaded = (files.length > 0 && files.every(file => file.file.id)) - const hasNotin = notionPages.length > 0 + // Default to FILE type when no type is provided from either source + const dataSourceType = isInCreatePage + ? (inCreatePageDataSourceType ?? DataSourceType.FILE) + : (dataset?.data_source_type ?? DataSourceType.FILE) + + const allFileLoaded = files.length > 0 && files.every(file => file.file.id) + const hasNotion = notionPages.length > 0 const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace - const isShowVectorSpaceFull = (allFileLoaded || hasNotin) && isVectorSpaceFull && enableBilling + const isShowVectorSpaceFull = (allFileLoaded || hasNotion) && isVectorSpaceFull && enableBilling const supportBatchUpload = !enableBilling || plan.type !== Plan.sandbox - const notSupportBatchUpload = !supportBatchUpload - const [isShowPlanUpgradeModal, { - setTrue: showPlanUpgradeModal, - setFalse: hidePlanUpgradeModal, - }] = useBoolean(false) - const onStepChange = useCallback(() => { - if (notSupportBatchUpload) { - let isMultiple = false - if (dataSourceType === DataSourceType.FILE && files.length > 1) - isMultiple = true + const isNotionAuthed = useMemo(() => checkNotionAuth(authedDataSourceList), [authedDataSourceList]) + const notionCredentialList = useMemo(() => getNotionCredentialList(authedDataSourceList), [authedDataSourceList]) - if (dataSourceType === DataSourceType.NOTION && notionPages.length > 1) - isMultiple = true - - if (dataSourceType === DataSourceType.WEB && websitePages.length > 1) - isMultiple = true - - if (isMultiple) { - showPlanUpgradeModal() - return - } - } - doOnStepChange() - }, [dataSourceType, doOnStepChange, files.length, notSupportBatchUpload, notionPages.length, showPlanUpgradeModal, websitePages.length]) - - const nextDisabled = useMemo(() => { + const fileNextDisabled = useMemo(() => { if (!files.length) return true if (files.some(file => !file.file.id)) @@ -149,109 +134,50 @@ const StepOne = ({ return isShowVectorSpaceFull }, [files, isShowVectorSpaceFull]) - const isNotionAuthed = useMemo(() => { - if (!authedDataSourceList) - return false - const notionSource = authedDataSourceList.find(item => item.provider === 'notion_datasource') - if (!notionSource) - return false - return notionSource.credentials_list.length > 0 - }, [authedDataSourceList]) + // Clear previews when switching data source type + const handleClearPreviews = useCallback((newType: DataSourceType) => { + if (newType !== DataSourceType.FILE) + hideFilePreview() + if (newType !== DataSourceType.NOTION) + hideNotionPagePreview() + if (newType !== DataSourceType.WEB) + hideWebsitePreview() + }, [hideFilePreview, hideNotionPagePreview, hideWebsitePreview]) - const notionCredentialList = useMemo(() => { - return authedDataSourceList.find(item => item.provider === 'notion_datasource')?.credentials_list || [] - }, [authedDataSourceList]) + // Handle step change with batch upload check + const onStepChange = useCallback(() => { + if (!supportBatchUpload && dataSourceType) { + const checkFn = MULTIPLE_ITEMS_CHECK[dataSourceType] + if (checkFn?.({ files, notionPages, websitePages })) { + showPlanUpgradeModal() + return + } + } + doOnStepChange() + }, [dataSourceType, doOnStepChange, files, supportBatchUpload, notionPages, showPlanUpgradeModal, websitePages]) return (
+ {/* Left Panel - Form */}
- { - shouldShowDataSourceTypeList && ( + {shouldShowDataSourceTypeList && ( + <>
{t('steps.one', { ns: 'datasetCreation' })}
- ) - } - { - shouldShowDataSourceTypeList && ( -
-
{ - if (dataSourceTypeDisable) - return - changeType(DataSourceType.FILE) - hideNotionPagePreview() - hideWebsitePreview() - }} - > - - - {t('stepOne.dataSourceType.file', { ns: 'datasetCreation' })} - -
-
{ - if (dataSourceTypeDisable) - return - changeType(DataSourceType.NOTION) - hideFilePreview() - hideWebsitePreview() - }} - > - - - {t('stepOne.dataSourceType.notion', { ns: 'datasetCreation' })} - -
- {(ENABLE_WEBSITE_FIRECRAWL || ENABLE_WEBSITE_JINAREADER || ENABLE_WEBSITE_WATERCRAWL) && ( -
{ - if (dataSourceTypeDisable) - return - changeType(DataSourceType.WEB) - hideFilePreview() - hideNotionPagePreview() - }} - > - - - {t('stepOne.dataSourceType.web', { ns: 'datasetCreation' })} - -
- )} -
- ) - } + + + )} + + {/* File Data Source */} {dataSourceType === DataSourceType.FILE && ( <> {isShowVectorSpaceFull && ( @@ -268,24 +194,17 @@ const StepOne = ({
)} -
- -
- { - enableBilling && plan.type === Plan.sandbox && files.length > 0 && ( -
-
- -
- ) - } + + {enableBilling && plan.type === Plan.sandbox && files.length > 0 && ( +
+
+ +
+ )} )} + + {/* Notion Data Source */} {dataSourceType === DataSourceType.NOTION && ( <> {!isNotionAuthed && } @@ -295,7 +214,7 @@ const StepOne = ({ page.page_id)} onSelect={updateNotionPages} - onPreview={updateCurrentPage} + onPreview={showNotionPagePreview} credentialList={notionCredentialList} onSelectCredential={updateNotionCredentialId} datasetId={datasetId} @@ -306,23 +225,21 @@ const StepOne = ({
)} -
- -
+ )} )} + + {/* Web Data Source */} {dataSourceType === DataSourceType.WEB && ( <>
)} -
- -
+ )} + + {/* Empty Dataset Creation Link */} {!datasetId && ( <>
- + {t('stepOne.emptyDatasetCreation', { ns: 'datasetCreation' })} )}
- +
-
- {currentFile && } - {currentNotionPage && ( - - )} - {currentWebsite && } - {isShowPlanUpgradeModal && ( - - )} -
+ + {/* Right Panel - Preview */} +
) diff --git a/web/app/components/header/account-setting/model-provider-page/index.tsx b/web/app/components/header/account-setting/model-provider-page/index.tsx index f456bcaaa6..d3daaee859 100644 --- a/web/app/components/header/account-setting/model-provider-page/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/index.tsx @@ -31,14 +31,19 @@ const FixedModelProvider = ['langgenius/openai/openai', 'langgenius/anthropic/an const ModelProviderPage = ({ searchText }: Props) => { const debouncedSearchText = useDebounce(searchText, { wait: 500 }) const { t } = useTranslation() - const { data: textGenerationDefaultModel } = useDefaultModel(ModelTypeEnum.textGeneration) - const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding) - const { data: rerankDefaultModel } = useDefaultModel(ModelTypeEnum.rerank) - const { data: speech2textDefaultModel } = useDefaultModel(ModelTypeEnum.speech2text) - const { data: ttsDefaultModel } = useDefaultModel(ModelTypeEnum.tts) + const { data: textGenerationDefaultModel, isLoading: isTextGenerationDefaultModelLoading } = useDefaultModel(ModelTypeEnum.textGeneration) + const { data: embeddingsDefaultModel, isLoading: isEmbeddingsDefaultModelLoading } = useDefaultModel(ModelTypeEnum.textEmbedding) + const { data: rerankDefaultModel, isLoading: isRerankDefaultModelLoading } = useDefaultModel(ModelTypeEnum.rerank) + const { data: speech2textDefaultModel, isLoading: isSpeech2textDefaultModelLoading } = useDefaultModel(ModelTypeEnum.speech2text) + const { data: ttsDefaultModel, isLoading: isTTSDefaultModelLoading } = useDefaultModel(ModelTypeEnum.tts) const { modelProviders: providers } = useProviderContext() const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures) - const defaultModelNotConfigured = !textGenerationDefaultModel && !embeddingsDefaultModel && !speech2textDefaultModel && !rerankDefaultModel && !ttsDefaultModel + const isDefaultModelLoading = isTextGenerationDefaultModelLoading + || isEmbeddingsDefaultModelLoading + || isRerankDefaultModelLoading + || isSpeech2textDefaultModelLoading + || isTTSDefaultModelLoading + const defaultModelNotConfigured = !isDefaultModelLoading && !textGenerationDefaultModel && !embeddingsDefaultModel && !speech2textDefaultModel && !rerankDefaultModel && !ttsDefaultModel const [configuredProviders, notConfiguredProviders] = useMemo(() => { const configuredProviders: ModelProvider[] = [] const notConfiguredProviders: ModelProvider[] = [] @@ -106,6 +111,7 @@ const ModelProviderPage = ({ searchText }: Props) => { rerankDefaultModel={rerankDefaultModel} speech2textDefaultModel={speech2textDefaultModel} ttsDefaultModel={ttsDefaultModel} + isLoading={isDefaultModelLoading} />
diff --git a/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx b/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx index 74222ed56d..29c71e04fc 100644 --- a/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx @@ -3,7 +3,7 @@ import type { DefaultModel, DefaultModelResponse, } from '../declarations' -import { RiEqualizer2Line } from '@remixicon/react' +import { RiEqualizer2Line, RiLoader2Line } from '@remixicon/react' import { useState } from 'react' import { useTranslation } from 'react-i18next' import Button from '@/app/components/base/button' @@ -32,6 +32,7 @@ type SystemModelSelectorProps = { speech2textDefaultModel: DefaultModelResponse | undefined ttsDefaultModel: DefaultModelResponse | undefined notConfigured: boolean + isLoading?: boolean } const SystemModel: FC = ({ textGenerationDefaultModel, @@ -40,6 +41,7 @@ const SystemModel: FC = ({ speech2textDefaultModel, ttsDefaultModel, notConfigured, + isLoading, }) => { const { t } = useTranslation() const { notify } = useToastContext() @@ -129,13 +131,16 @@ const SystemModel: FC = ({ crossAxis: 8, }} > - setOpen(v => !v)}> + setOpen(v => !v)}> diff --git a/web/app/components/plugins/marketplace/description/index.spec.tsx b/web/app/components/plugins/marketplace/description/index.spec.tsx index b5c8cb716b..054949ee1f 100644 --- a/web/app/components/plugins/marketplace/description/index.spec.tsx +++ b/web/app/components/plugins/marketplace/description/index.spec.tsx @@ -1,7 +1,5 @@ import { render, screen } from '@testing-library/react' import { beforeEach, describe, expect, it, vi } from 'vitest' - -// Import component after mocks are set up import Description from './index' // ================================ @@ -30,20 +28,18 @@ const commonTranslations: Record = { 'operation.in': 'in', } -// Mock getLocaleOnServer and translate -vi.mock('@/i18n-config/server', () => ({ - getLocaleOnServer: vi.fn(() => Promise.resolve(mockDefaultLocale)), - getTranslation: vi.fn((locale: string, ns: string) => { - return Promise.resolve({ - t: (key: string) => { - if (ns === 'plugin') - return pluginTranslations[key] || key - if (ns === 'common') - return commonTranslations[key] || key - return key - }, - }) - }), +// Mock i18n hooks +vi.mock('#i18n', () => ({ + useLocale: vi.fn(() => mockDefaultLocale), + useTranslation: vi.fn((ns: string) => ({ + t: (key: string) => { + if (ns === 'plugin') + return pluginTranslations[key] || key + if (ns === 'common') + return commonTranslations[key] || key + return key + }, + })), })) // ================================ @@ -59,29 +55,29 @@ describe('Description', () => { // Rendering Tests // ================================ describe('Rendering', () => { - it('should render without crashing', async () => { - const { container } = render(await Description({})) + it('should render without crashing', () => { + const { container } = render() expect(container.firstChild).toBeInTheDocument() }) - it('should render h1 heading with empower text', async () => { - render(await Description({})) + it('should render h1 heading with empower text', () => { + render() const heading = screen.getByRole('heading', { level: 1 }) expect(heading).toBeInTheDocument() expect(heading).toHaveTextContent('Empower your AI development') }) - it('should render h2 subheading', async () => { - render(await Description({})) + it('should render h2 subheading', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading).toBeInTheDocument() }) - it('should apply correct CSS classes to h1', async () => { - render(await Description({})) + it('should apply correct CSS classes to h1', () => { + render() const heading = screen.getByRole('heading', { level: 1 }) expect(heading).toHaveClass('title-4xl-semi-bold') @@ -90,8 +86,8 @@ describe('Description', () => { expect(heading).toHaveClass('text-text-primary') }) - it('should apply correct CSS classes to h2', async () => { - render(await Description({})) + it('should apply correct CSS classes to h2', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading).toHaveClass('body-md-regular') @@ -104,14 +100,18 @@ describe('Description', () => { // Non-Chinese Locale Rendering Tests // ================================ describe('Non-Chinese Locale Rendering', () => { - it('should render discover text for en-US locale', async () => { - render(await Description({ locale: 'en-US' })) + beforeEach(() => { + mockDefaultLocale = 'en-US' + }) + + it('should render discover text for en-US locale', () => { + render() expect(screen.getByText(/Discover/)).toBeInTheDocument() }) - it('should render all category names', async () => { - render(await Description({ locale: 'en-US' })) + it('should render all category names', () => { + render() expect(screen.getByText('Models')).toBeInTheDocument() expect(screen.getByText('Tools')).toBeInTheDocument() @@ -122,36 +122,36 @@ describe('Description', () => { expect(screen.getByText('Bundles')).toBeInTheDocument() }) - it('should render "and" conjunction text', async () => { - render(await Description({ locale: 'en-US' })) + it('should render "and" conjunction text', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading.textContent).toContain('and') }) - it('should render "in" preposition at the end for non-Chinese locales', async () => { - render(await Description({ locale: 'en-US' })) + it('should render "in" preposition at the end for non-Chinese locales', () => { + render() expect(screen.getByText('in')).toBeInTheDocument() }) - it('should render Dify Marketplace text at the end for non-Chinese locales', async () => { - render(await Description({ locale: 'en-US' })) + it('should render Dify Marketplace text at the end for non-Chinese locales', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading.textContent).toContain('Dify Marketplace') }) - it('should render category spans with styled underline effect', async () => { - const { container } = render(await Description({ locale: 'en-US' })) + it('should render category spans with styled underline effect', () => { + const { container } = render() const styledSpans = container.querySelectorAll('.body-md-medium.relative.z-\\[1\\]') // 7 category spans (models, tools, datasources, triggers, agents, extensions, bundles) expect(styledSpans.length).toBe(7) }) - it('should apply text-text-secondary class to category spans', async () => { - const { container } = render(await Description({ locale: 'en-US' })) + it('should apply text-text-secondary class to category spans', () => { + const { container } = render() const styledSpans = container.querySelectorAll('.text-text-secondary') expect(styledSpans.length).toBeGreaterThanOrEqual(7) @@ -162,29 +162,33 @@ describe('Description', () => { // Chinese (zh-Hans) Locale Rendering Tests // ================================ describe('Chinese (zh-Hans) Locale Rendering', () => { - it('should render "in" text at the beginning for zh-Hans locale', async () => { - render(await Description({ locale: 'zh-Hans' })) + beforeEach(() => { + mockDefaultLocale = 'zh-Hans' + }) + + it('should render "in" text at the beginning for zh-Hans locale', () => { + render() // In zh-Hans mode, "in" appears at the beginning const inElements = screen.getAllByText('in') expect(inElements.length).toBeGreaterThanOrEqual(1) }) - it('should render Dify Marketplace text for zh-Hans locale', async () => { - render(await Description({ locale: 'zh-Hans' })) + it('should render Dify Marketplace text for zh-Hans locale', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading.textContent).toContain('Dify Marketplace') }) - it('should render discover text for zh-Hans locale', async () => { - render(await Description({ locale: 'zh-Hans' })) + it('should render discover text for zh-Hans locale', () => { + render() expect(screen.getByText(/Discover/)).toBeInTheDocument() }) - it('should render all categories for zh-Hans locale', async () => { - render(await Description({ locale: 'zh-Hans' })) + it('should render all categories for zh-Hans locale', () => { + render() expect(screen.getByText('Models')).toBeInTheDocument() expect(screen.getByText('Tools')).toBeInTheDocument() @@ -195,8 +199,8 @@ describe('Description', () => { expect(screen.getByText('Bundles')).toBeInTheDocument() }) - it('should render both zh-Hans specific elements and shared elements', async () => { - render(await Description({ locale: 'zh-Hans' })) + it('should render both zh-Hans specific elements and shared elements', () => { + render() // zh-Hans has specific element order: "in" -> Dify Marketplace -> Discover // then the same category list with "and" -> Bundles @@ -206,61 +210,57 @@ describe('Description', () => { }) // ================================ - // Locale Prop Variations Tests + // Locale Variations Tests // ================================ - describe('Locale Prop Variations', () => { - it('should use default locale when locale prop is undefined', async () => { + describe('Locale Variations', () => { + it('should use en-US locale by default', () => { mockDefaultLocale = 'en-US' - render(await Description({})) + render() - // Should use the default locale from getLocaleOnServer expect(screen.getByText('Empower your AI development')).toBeInTheDocument() }) - it('should use provided locale prop instead of default', async () => { + it('should handle ja-JP locale as non-Chinese', () => { mockDefaultLocale = 'ja-JP' - render(await Description({ locale: 'en-US' })) - - // The locale prop should be used, triggering non-Chinese rendering - const subheading = screen.getByRole('heading', { level: 2 }) - expect(subheading).toBeInTheDocument() - }) - - it('should handle ja-JP locale as non-Chinese', async () => { - render(await Description({ locale: 'ja-JP' })) + render() // Should render in non-Chinese format (discover first, then "in Dify Marketplace" at end) const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading.textContent).toContain('Dify Marketplace') }) - it('should handle ko-KR locale as non-Chinese', async () => { - render(await Description({ locale: 'ko-KR' })) + it('should handle ko-KR locale as non-Chinese', () => { + mockDefaultLocale = 'ko-KR' + render() // Should render in non-Chinese format expect(screen.getByText('Empower your AI development')).toBeInTheDocument() }) - it('should handle de-DE locale as non-Chinese', async () => { - render(await Description({ locale: 'de-DE' })) + it('should handle de-DE locale as non-Chinese', () => { + mockDefaultLocale = 'de-DE' + render() expect(screen.getByText('Empower your AI development')).toBeInTheDocument() }) - it('should handle fr-FR locale as non-Chinese', async () => { - render(await Description({ locale: 'fr-FR' })) + it('should handle fr-FR locale as non-Chinese', () => { + mockDefaultLocale = 'fr-FR' + render() expect(screen.getByText('Empower your AI development')).toBeInTheDocument() }) - it('should handle pt-BR locale as non-Chinese', async () => { - render(await Description({ locale: 'pt-BR' })) + it('should handle pt-BR locale as non-Chinese', () => { + mockDefaultLocale = 'pt-BR' + render() expect(screen.getByText('Empower your AI development')).toBeInTheDocument() }) - it('should handle es-ES locale as non-Chinese', async () => { - render(await Description({ locale: 'es-ES' })) + it('should handle es-ES locale as non-Chinese', () => { + mockDefaultLocale = 'es-ES' + render() expect(screen.getByText('Empower your AI development')).toBeInTheDocument() }) @@ -270,24 +270,27 @@ describe('Description', () => { // Conditional Rendering Tests // ================================ describe('Conditional Rendering', () => { - it('should render zh-Hans specific content when locale is zh-Hans', async () => { - const { container } = render(await Description({ locale: 'zh-Hans' })) + it('should render zh-Hans specific content when locale is zh-Hans', () => { + mockDefaultLocale = 'zh-Hans' + const { container } = render() // zh-Hans has additional span with mr-1 before "in" text at the start const mrSpan = container.querySelector('span.mr-1') expect(mrSpan).toBeInTheDocument() }) - it('should render non-Chinese specific content when locale is not zh-Hans', async () => { - render(await Description({ locale: 'en-US' })) + it('should render non-Chinese specific content when locale is not zh-Hans', () => { + mockDefaultLocale = 'en-US' + render() // Non-Chinese has "in" and "Dify Marketplace" at the end const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading.textContent).toContain('Dify Marketplace') }) - it('should not render zh-Hans intro content for non-Chinese locales', async () => { - render(await Description({ locale: 'en-US' })) + it('should not render zh-Hans intro content for non-Chinese locales', () => { + mockDefaultLocale = 'en-US' + render() // For en-US, the order should be Discover ... in Dify Marketplace // The "in" text should only appear once at the end @@ -303,8 +306,9 @@ describe('Description', () => { expect(inIndex).toBeLessThan(marketplaceIndex) }) - it('should render zh-Hans with proper word order', async () => { - render(await Description({ locale: 'zh-Hans' })) + it('should render zh-Hans with proper word order', () => { + mockDefaultLocale = 'zh-Hans' + render() const subheading = screen.getByRole('heading', { level: 2 }) const content = subheading.textContent || '' @@ -323,58 +327,58 @@ describe('Description', () => { // Category Styling Tests // ================================ describe('Category Styling', () => { - it('should apply underline effect with after pseudo-element styling', async () => { - const { container } = render(await Description({})) + it('should apply underline effect with after pseudo-element styling', () => { + const { container } = render() const categorySpan = container.querySelector('.after\\:absolute') expect(categorySpan).toBeInTheDocument() }) - it('should apply correct after pseudo-element classes', async () => { - const { container } = render(await Description({})) + it('should apply correct after pseudo-element classes', () => { + const { container } = render() // Check for the specific after pseudo-element classes const categorySpans = container.querySelectorAll('.after\\:bottom-\\[1\\.5px\\]') expect(categorySpans.length).toBe(7) }) - it('should apply full width to after element', async () => { - const { container } = render(await Description({})) + it('should apply full width to after element', () => { + const { container } = render() const categorySpans = container.querySelectorAll('.after\\:w-full') expect(categorySpans.length).toBe(7) }) - it('should apply correct height to after element', async () => { - const { container } = render(await Description({})) + it('should apply correct height to after element', () => { + const { container } = render() const categorySpans = container.querySelectorAll('.after\\:h-2') expect(categorySpans.length).toBe(7) }) - it('should apply bg-text-text-selected to after element', async () => { - const { container } = render(await Description({})) + it('should apply bg-text-text-selected to after element', () => { + const { container } = render() const categorySpans = container.querySelectorAll('.after\\:bg-text-text-selected') expect(categorySpans.length).toBe(7) }) - it('should have z-index 1 on category spans', async () => { - const { container } = render(await Description({})) + it('should have z-index 1 on category spans', () => { + const { container } = render() const categorySpans = container.querySelectorAll('.z-\\[1\\]') expect(categorySpans.length).toBe(7) }) - it('should apply left margin to category spans', async () => { - const { container } = render(await Description({})) + it('should apply left margin to category spans', () => { + const { container } = render() const categorySpans = container.querySelectorAll('.ml-1') expect(categorySpans.length).toBeGreaterThanOrEqual(7) }) - it('should apply both left and right margin to specific spans', async () => { - const { container } = render(await Description({})) + it('should apply both left and right margin to specific spans', () => { + const { container } = render() // Extensions and Bundles spans have both ml-1 and mr-1 const extensionsBundlesSpans = container.querySelectorAll('.ml-1.mr-1') @@ -386,28 +390,17 @@ describe('Description', () => { // Edge Cases Tests // ================================ describe('Edge Cases', () => { - it('should handle empty props object', async () => { - const { container } = render(await Description({})) - - expect(container.firstChild).toBeInTheDocument() - }) - - it('should render fragment as root element', async () => { - const { container } = render(await Description({})) + it('should render fragment as root element', () => { + const { container } = render() // Fragment renders h1 and h2 as direct children expect(container.querySelector('h1')).toBeInTheDocument() expect(container.querySelector('h2')).toBeInTheDocument() }) - it('should handle locale prop with undefined value', async () => { - render(await Description({ locale: undefined })) - - expect(screen.getByRole('heading', { level: 1 })).toBeInTheDocument() - }) - - it('should handle zh-Hant as non-Chinese simplified', async () => { - render(await Description({ locale: 'zh-Hant' })) + it('should handle zh-Hant as non-Chinese simplified', () => { + mockDefaultLocale = 'zh-Hant' + render() // zh-Hant is different from zh-Hans, should use non-Chinese format const subheading = screen.getByRole('heading', { level: 2 }) @@ -426,8 +419,8 @@ describe('Description', () => { // Content Structure Tests // ================================ describe('Content Structure', () => { - it('should have comma separators between categories', async () => { - render(await Description({})) + it('should have comma separators between categories', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) const content = subheading.textContent || '' @@ -436,8 +429,8 @@ describe('Description', () => { expect(content).toMatch(/Models[^\n\r,\u2028\u2029]*,.*Tools[^\n\r,\u2028\u2029]*,.*Data Sources[^\n\r,\u2028\u2029]*,.*Triggers[^\n\r,\u2028\u2029]*,.*Agent Strategies[^\n\r,\u2028\u2029]*,.*Extensions/) }) - it('should have "and" before last category (Bundles)', async () => { - render(await Description({})) + it('should have "and" before last category (Bundles)', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) const content = subheading.textContent || '' @@ -449,8 +442,9 @@ describe('Description', () => { expect(andIndex).toBeLessThan(bundlesIndex) }) - it('should render all text elements in correct order for en-US', async () => { - render(await Description({ locale: 'en-US' })) + it('should render all text elements in correct order for en-US', () => { + mockDefaultLocale = 'en-US' + render() const subheading = screen.getByRole('heading', { level: 2 }) const content = subheading.textContent || '' @@ -477,8 +471,9 @@ describe('Description', () => { } }) - it('should render all text elements in correct order for zh-Hans', async () => { - render(await Description({ locale: 'zh-Hans' })) + it('should render all text elements in correct order for zh-Hans', () => { + mockDefaultLocale = 'zh-Hans' + render() const subheading = screen.getByRole('heading', { level: 2 }) const content = subheading.textContent || '' @@ -499,82 +494,48 @@ describe('Description', () => { // Layout Tests // ================================ describe('Layout', () => { - it('should have shrink-0 on h1 heading', async () => { - render(await Description({})) + it('should have shrink-0 on h1 heading', () => { + render() const heading = screen.getByRole('heading', { level: 1 }) expect(heading).toHaveClass('shrink-0') }) - it('should have shrink-0 on h2 subheading', async () => { - render(await Description({})) + it('should have shrink-0 on h2 subheading', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading).toHaveClass('shrink-0') }) - it('should have flex layout on h2', async () => { - render(await Description({})) + it('should have flex layout on h2', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading).toHaveClass('flex') }) - it('should have items-center on h2', async () => { - render(await Description({})) + it('should have items-center on h2', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading).toHaveClass('items-center') }) - it('should have justify-center on h2', async () => { - render(await Description({})) + it('should have justify-center on h2', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading).toHaveClass('justify-center') }) }) - // ================================ - // Translation Function Tests - // ================================ - describe('Translation Functions', () => { - it('should call getTranslation for plugin namespace', async () => { - const { getTranslation } = await import('@/i18n-config/server') - render(await Description({ locale: 'en-US' })) - - expect(getTranslation).toHaveBeenCalledWith('en-US', 'plugin') - }) - - it('should call getTranslation for common namespace', async () => { - const { getTranslation } = await import('@/i18n-config/server') - render(await Description({ locale: 'en-US' })) - - expect(getTranslation).toHaveBeenCalledWith('en-US', 'common') - }) - - it('should call getLocaleOnServer when locale prop is undefined', async () => { - const { getLocaleOnServer } = await import('@/i18n-config/server') - render(await Description({})) - - expect(getLocaleOnServer).toHaveBeenCalled() - }) - - it('should use locale prop when provided', async () => { - const { getTranslation } = await import('@/i18n-config/server') - render(await Description({ locale: 'ja-JP' })) - - expect(getTranslation).toHaveBeenCalledWith('ja-JP', 'plugin') - expect(getTranslation).toHaveBeenCalledWith('ja-JP', 'common') - }) - }) - // ================================ // Accessibility Tests // ================================ describe('Accessibility', () => { - it('should have proper heading hierarchy', async () => { - render(await Description({})) + it('should have proper heading hierarchy', () => { + render() const h1 = screen.getByRole('heading', { level: 1 }) const h2 = screen.getByRole('heading', { level: 2 }) @@ -583,22 +544,22 @@ describe('Description', () => { expect(h2).toBeInTheDocument() }) - it('should have readable text content', async () => { - render(await Description({})) + it('should have readable text content', () => { + render() const h1 = screen.getByRole('heading', { level: 1 }) expect(h1.textContent).not.toBe('') }) - it('should have visible h1 heading', async () => { - render(await Description({})) + it('should have visible h1 heading', () => { + render() const heading = screen.getByRole('heading', { level: 1 }) expect(heading).toBeVisible() }) - it('should have visible h2 heading', async () => { - render(await Description({})) + it('should have visible h2 heading', () => { + render() const subheading = screen.getByRole('heading', { level: 2 }) expect(subheading).toBeVisible() @@ -615,8 +576,8 @@ describe('Description Integration', () => { mockDefaultLocale = 'en-US' }) - it('should render complete component structure', async () => { - const { container } = render(await Description({ locale: 'en-US' })) + it('should render complete component structure', () => { + const { container } = render() // Main headings expect(container.querySelector('h1')).toBeInTheDocument() @@ -627,8 +588,9 @@ describe('Description Integration', () => { expect(categorySpans.length).toBe(7) }) - it('should render complete zh-Hans structure', async () => { - const { container } = render(await Description({ locale: 'zh-Hans' })) + it('should render complete zh-Hans structure', () => { + mockDefaultLocale = 'zh-Hans' + const { container } = render() // Main headings expect(container.querySelector('h1')).toBeInTheDocument() @@ -639,14 +601,16 @@ describe('Description Integration', () => { expect(categorySpans.length).toBe(7) }) - it('should correctly switch between zh-Hans and en-US layouts', async () => { + it('should correctly differentiate between zh-Hans and en-US layouts', () => { // Render en-US - const { container: enContainer, unmount: unmountEn } = render(await Description({ locale: 'en-US' })) + mockDefaultLocale = 'en-US' + const { container: enContainer, unmount: unmountEn } = render() const enContent = enContainer.querySelector('h2')?.textContent || '' unmountEn() // Render zh-Hans - const { container: zhContainer } = render(await Description({ locale: 'zh-Hans' })) + mockDefaultLocale = 'zh-Hans' + const { container: zhContainer } = render() const zhContent = zhContainer.querySelector('h2')?.textContent || '' // Both should have all categories @@ -666,14 +630,16 @@ describe('Description Integration', () => { expect(zhMarketplaceIndex).toBeLessThan(zhDiscoverIndex) }) - it('should maintain consistent styling across locales', async () => { + it('should maintain consistent styling across locales', () => { // Render en-US - const { container: enContainer, unmount: unmountEn } = render(await Description({ locale: 'en-US' })) + mockDefaultLocale = 'en-US' + const { container: enContainer, unmount: unmountEn } = render() const enCategoryCount = enContainer.querySelectorAll('.body-md-medium').length unmountEn() // Render zh-Hans - const { container: zhContainer } = render(await Description({ locale: 'zh-Hans' })) + mockDefaultLocale = 'zh-Hans' + const { container: zhContainer } = render() const zhCategoryCount = zhContainer.querySelectorAll('.body-md-medium').length // Both should have same number of styled category spans diff --git a/web/app/components/plugins/marketplace/description/index.tsx b/web/app/components/plugins/marketplace/description/index.tsx index d3ca964538..30ccbdb76e 100644 --- a/web/app/components/plugins/marketplace/description/index.tsx +++ b/web/app/components/plugins/marketplace/description/index.tsx @@ -1,17 +1,11 @@ -/* eslint-disable dify-i18n/require-ns-option */ -import type { Locale } from '@/i18n-config' -import { getLocaleOnServer, getTranslation } from '@/i18n-config/server' +import { useLocale, useTranslation } from '#i18n' -type DescriptionProps = { - locale?: Locale -} -const Description = async ({ - locale: localeFromProps, -}: DescriptionProps) => { - const localeDefault = await getLocaleOnServer() - const { t } = await getTranslation(localeFromProps || localeDefault, 'plugin') - const { t: tCommon } = await getTranslation(localeFromProps || localeDefault, 'common') - const isZhHans = localeFromProps === 'zh-Hans' +const Description = () => { + const { t } = useTranslation('plugin') + const { t: tCommon } = useTranslation('common') + const locale = useLocale() + + const isZhHans = locale === 'zh-Hans' return ( <> diff --git a/web/app/components/plugins/marketplace/index.tsx b/web/app/components/plugins/marketplace/index.tsx index ff9a4d60bc..f9f7e86b9a 100644 --- a/web/app/components/plugins/marketplace/index.tsx +++ b/web/app/components/plugins/marketplace/index.tsx @@ -42,7 +42,7 @@ const Marketplace = async ({ scrollContainerId={scrollContainerId} showSearchParams={showSearchParams} > - + { return false if (node.parentId === loopId) + return true return false diff --git a/web/app/components/workflow/utils/workflow-init.ts b/web/app/components/workflow/utils/workflow-init.ts index 4ee8390fc1..9eab630cfe 100644 --- a/web/app/components/workflow/utils/workflow-init.ts +++ b/web/app/components/workflow/utils/workflow-init.ts @@ -5,14 +5,9 @@ import type { IterationNodeType } from '../nodes/iteration/types' import type { LoopNodeType } from '../nodes/loop/types' import type { QuestionClassifierNodeType } from '../nodes/question-classifier/types' import type { ToolNodeType } from '../nodes/tool/types' -import type { - Edge, - Node, -} from '../types' +import type { Edge, Node } from '../types' import { cloneDeep } from 'es-toolkit/object' -import { - getConnectedEdges, -} from 'reactflow' +import { getConnectedEdges } from 'reactflow' import { getIterationStartNode, getLoopStartNode } from '@/app/components/workflow/utils/node' import { correctModelProvider } from '@/utils' import { @@ -24,22 +19,22 @@ import { NODE_WIDTH_X_OFFSET, START_INITIAL_POSITION, } from '../constants' -import { - CUSTOM_GROUP_NODE, - GROUP_CHILDREN_Z_INDEX, -} from '../custom-group-node' +import { CUSTOM_GROUP_NODE, GROUP_CHILDREN_Z_INDEX } from '../custom-group-node' import { branchNameCorrect } from '../nodes/if-else/utils' import { CUSTOM_ITERATION_START_NODE } from '../nodes/iteration-start/constants' import { CUSTOM_LOOP_START_NODE } from '../nodes/loop-start/constants' -import { - BlockEnum, - ErrorHandleMode, -} from '../types' +import { BlockEnum, ErrorHandleMode } from '../types' const WHITE = 'WHITE' const GRAY = 'GRAY' const BLACK = 'BLACK' -const isCyclicUtil = (nodeId: string, color: Record, adjList: Record, stack: string[]) => { + +const isCyclicUtil = ( + nodeId: string, + color: Record, + adjList: Record, + stack: string[], +) => { color[nodeId] = GRAY stack.push(nodeId) @@ -50,8 +45,12 @@ const isCyclicUtil = (nodeId: string, color: Record, adjList: Re stack.push(childId) return true } - if (color[childId] === WHITE && isCyclicUtil(childId, color, adjList, stack)) + if ( + color[childId] === WHITE + && isCyclicUtil(childId, color, adjList, stack) + ) { return true + } } color[nodeId] = BLACK if (stack.length > 0 && stack[stack.length - 1] === nodeId) @@ -69,8 +68,7 @@ const getCycleEdges = (nodes: Node[], edges: Edge[]) => { adjList[node.id] = [] } - for (const edge of edges) - adjList[edge.source]?.push(edge.target) + for (const edge of edges) adjList[edge.source]?.push(edge.target) for (let i = 0; i < nodes.length; i++) { if (color[nodes[i].id] === WHITE) @@ -90,22 +88,34 @@ const getCycleEdges = (nodes: Node[], edges: Edge[]) => { } export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { - const hasIterationNode = nodes.some(node => node.data.type === BlockEnum.Iteration) + const hasIterationNode = nodes.some( + node => node.data.type === BlockEnum.Iteration, + ) const hasLoopNode = nodes.some(node => node.data.type === BlockEnum.Loop) const hasGroupNode = nodes.some(node => node.type === CUSTOM_GROUP_NODE) - const hasBusinessGroupNode = nodes.some(node => node.data.type === BlockEnum.Group) + const hasBusinessGroupNode = nodes.some( + node => node.data.type === BlockEnum.Group, + ) - if (!hasIterationNode && !hasLoopNode && !hasGroupNode && !hasBusinessGroupNode) { + if ( + !hasIterationNode + && !hasLoopNode + && !hasGroupNode + && !hasBusinessGroupNode + ) { return { nodes, edges, } } - const nodesMap = nodes.reduce((prev, next) => { - prev[next.id] = next - return prev - }, {} as Record) + const nodesMap = nodes.reduce( + (prev, next) => { + prev[next.id] = next + return prev + }, + {} as Record, + ) const iterationNodesWithStartNode = [] const iterationNodesWithoutStartNode = [] @@ -117,8 +127,12 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { if (currentNode.data.type === BlockEnum.Iteration) { if (currentNode.data.start_node_id) { - if (nodesMap[currentNode.data.start_node_id]?.type !== CUSTOM_ITERATION_START_NODE) + if ( + nodesMap[currentNode.data.start_node_id]?.type + !== CUSTOM_ITERATION_START_NODE + ) { iterationNodesWithStartNode.push(currentNode) + } } else { iterationNodesWithoutStartNode.push(currentNode) @@ -127,8 +141,12 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { if (currentNode.data.type === BlockEnum.Loop) { if (currentNode.data.start_node_id) { - if (nodesMap[currentNode.data.start_node_id]?.type !== CUSTOM_LOOP_START_NODE) + if ( + nodesMap[currentNode.data.start_node_id]?.type + !== CUSTOM_LOOP_START_NODE + ) { loopNodesWithStartNode.push(currentNode) + } } else { loopNodesWithoutStartNode.push(currentNode) @@ -137,7 +155,10 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { } const newIterationStartNodesMap = {} as Record - const newIterationStartNodes = [...iterationNodesWithStartNode, ...iterationNodesWithoutStartNode].map((iterationNode, index) => { + const newIterationStartNodes = [ + ...iterationNodesWithStartNode, + ...iterationNodesWithoutStartNode, + ].map((iterationNode, index) => { const newNode = getIterationStartNode(iterationNode.id) newNode.id = newNode.id + index newIterationStartNodesMap[iterationNode.id] = newNode @@ -145,24 +166,34 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { }) const newLoopStartNodesMap = {} as Record - const newLoopStartNodes = [...loopNodesWithStartNode, ...loopNodesWithoutStartNode].map((loopNode, index) => { + const newLoopStartNodes = [ + ...loopNodesWithStartNode, + ...loopNodesWithoutStartNode, + ].map((loopNode, index) => { const newNode = getLoopStartNode(loopNode.id) newNode.id = newNode.id + index newLoopStartNodesMap[loopNode.id] = newNode return newNode }) - const newEdges = [...iterationNodesWithStartNode, ...loopNodesWithStartNode].map((nodeItem) => { + const newEdges = [ + ...iterationNodesWithStartNode, + ...loopNodesWithStartNode, + ].map((nodeItem) => { const isIteration = nodeItem.data.type === BlockEnum.Iteration - const newNode = (isIteration ? newIterationStartNodesMap : newLoopStartNodesMap)[nodeItem.id] + const newNode = ( + isIteration ? newIterationStartNodesMap : newLoopStartNodesMap + )[nodeItem.id] const startNode = nodesMap[nodeItem.data.start_node_id] const source = newNode.id const sourceHandle = 'source' const target = startNode.id const targetHandle = 'target' - const parentNode = nodes.find(node => node.id === startNode.parentId) || null - const isInIteration = !!parentNode && parentNode.data.type === BlockEnum.Iteration + const parentNode + = nodes.find(node => node.id === startNode.parentId) || null + const isInIteration + = !!parentNode && parentNode.data.type === BlockEnum.Iteration const isInLoop = !!parentNode && parentNode.data.type === BlockEnum.Loop return { @@ -185,11 +216,18 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { } }) nodes.forEach((node) => { - if (node.data.type === BlockEnum.Iteration && newIterationStartNodesMap[node.id]) - (node.data as IterationNodeType).start_node_id = newIterationStartNodesMap[node.id].id + if ( + node.data.type === BlockEnum.Iteration + && newIterationStartNodesMap[node.id] + ) { + (node.data as IterationNodeType).start_node_id + = newIterationStartNodesMap[node.id].id + } - if (node.data.type === BlockEnum.Loop && newLoopStartNodesMap[node.id]) - (node.data as LoopNodeType).start_node_id = newLoopStartNodesMap[node.id].id + if (node.data.type === BlockEnum.Loop && newLoopStartNodesMap[node.id]) { + (node.data as LoopNodeType).start_node_id + = newLoopStartNodesMap[node.id].id + } }) // Derive Group internal edges (input → entries, leaves → exits) @@ -240,7 +278,7 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { targetHandle: 'target', data: { sourceType: leafNode.data.type, - targetType: '' as any, // Exit port has empty type + targetType: '' as string, // Exit port has empty type _isGroupInternal: true, _groupId: groupNode.id, }, @@ -260,7 +298,12 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { return const groupData = groupNode.data as GroupNodeData - const { members = [], headNodeIds = [], leafNodeIds = [], handlers = [] } = groupData + const { + members = [], + headNodeIds = [], + leafNodeIds = [], + handlers = [], + } = groupData const memberSet = new Set(members.map(m => m.id)) const headSet = new Set(headNodeIds) const leafSet = new Set(leafNodeIds) @@ -293,7 +336,8 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { if (leafSet.has(edge.source) && !memberSet.has(edge.target)) { const edgeSourceHandle = edge.sourceHandle || 'source' const handler = handlers.find( - h => h.nodeId === edge.source && h.sourceHandle === edgeSourceHandle, + h => + h.nodeId === edge.source && h.sourceHandle === edgeSourceHandle, ) if (handler) { groupTempEdges.push({ @@ -321,7 +365,10 @@ export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { } export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { - const { nodes, edges } = preprocessNodesAndEdges(cloneDeep(originNodes), cloneDeep(originEdges)) + const { nodes, edges } = preprocessNodesAndEdges( + cloneDeep(originNodes), + cloneDeep(originEdges), + ) const firstNode = nodes[0] if (!firstNode?.position) { @@ -333,23 +380,35 @@ export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { }) } - const iterationOrLoopNodeMap = nodes.reduce((acc, node) => { - if (node.parentId) { - if (acc[node.parentId]) - acc[node.parentId].push({ nodeId: node.id, nodeType: node.data.type }) - else - acc[node.parentId] = [{ nodeId: node.id, nodeType: node.data.type }] - } - return acc - }, {} as Record) + const iterationOrLoopNodeMap = nodes.reduce( + (acc, node) => { + if (node.parentId) { + if (acc[node.parentId]) { + acc[node.parentId].push({ + nodeId: node.id, + nodeType: node.data.type, + }) + } + else { + acc[node.parentId] = [{ nodeId: node.id, nodeType: node.data.type }] + } + } + return acc + }, + {} as Record, + ) return nodes.map((node) => { if (!node.type) node.type = CUSTOM_NODE const connectedEdges = getConnectedEdges([node], edges) - node.data._connectedSourceHandleIds = connectedEdges.filter(edge => edge.source === node.id).map(edge => edge.sourceHandle || 'source') - node.data._connectedTargetHandleIds = connectedEdges.filter(edge => edge.target === node.id).map(edge => edge.targetHandle || 'target') + node.data._connectedSourceHandleIds = connectedEdges + .filter(edge => edge.source === node.id) + .map(edge => edge.sourceHandle || 'source') + node.data._connectedTargetHandleIds = connectedEdges + .filter(edge => edge.target === node.id) + .map(edge => edge.targetHandle || 'target') if (node.data.type === BlockEnum.IfElse) { const nodeData = node.data as IfElseNodeType @@ -364,18 +423,27 @@ export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { ] } node.data._targetBranches = branchNameCorrect([ - ...(node.data as IfElseNodeType).cases.map(item => ({ id: item.case_id, name: '' })), + ...(node.data as IfElseNodeType).cases.map(item => ({ + id: item.case_id, + name: '', + })), { id: 'false', name: '' }, ]) // delete conditions and logical_operator if cases is not empty - if (nodeData.cases.length > 0 && nodeData.conditions && nodeData.logical_operator) { + if ( + nodeData.cases.length > 0 + && nodeData.conditions + && nodeData.logical_operator + ) { delete nodeData.conditions delete nodeData.logical_operator } } if (node.data.type === BlockEnum.QuestionClassifier) { - node.data._targetBranches = (node.data as QuestionClassifierNodeType).classes.map((topic) => { + node.data._targetBranches = ( + node.data as QuestionClassifierNodeType + ).classes.map((topic) => { return topic }) } @@ -395,28 +463,46 @@ export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { iterationNodeData._children = iterationOrLoopNodeMap[node.id] || [] iterationNodeData.is_parallel = iterationNodeData.is_parallel || false iterationNodeData.parallel_nums = iterationNodeData.parallel_nums || 10 - iterationNodeData.error_handle_mode = iterationNodeData.error_handle_mode || ErrorHandleMode.Terminated + iterationNodeData.error_handle_mode + = iterationNodeData.error_handle_mode || ErrorHandleMode.Terminated } // TODO: loop error handle mode if (node.data.type === BlockEnum.Loop) { const loopNodeData = node.data as LoopNodeType loopNodeData._children = iterationOrLoopNodeMap[node.id] || [] - loopNodeData.error_handle_mode = loopNodeData.error_handle_mode || ErrorHandleMode.Terminated + loopNodeData.error_handle_mode + = loopNodeData.error_handle_mode || ErrorHandleMode.Terminated } // legacy provider handle - if (node.data.type === BlockEnum.LLM) - (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) + if (node.data.type === BlockEnum.LLM) { + (node as any).data.model.provider = correctModelProvider( + (node as any).data.model.provider, + ) + } - if (node.data.type === BlockEnum.KnowledgeRetrieval && (node as any).data.multiple_retrieval_config?.reranking_model) - (node as any).data.multiple_retrieval_config.reranking_model.provider = correctModelProvider((node as any).data.multiple_retrieval_config?.reranking_model.provider) + if ( + node.data.type === BlockEnum.KnowledgeRetrieval + && (node as any).data.multiple_retrieval_config?.reranking_model + ) { + (node as any).data.multiple_retrieval_config.reranking_model.provider + = correctModelProvider( + (node as any).data.multiple_retrieval_config?.reranking_model.provider, + ) + } - if (node.data.type === BlockEnum.QuestionClassifier) - (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) + if (node.data.type === BlockEnum.QuestionClassifier) { + (node as any).data.model.provider = correctModelProvider( + (node as any).data.model.provider, + ) + } - if (node.data.type === BlockEnum.ParameterExtractor) - (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) + if (node.data.type === BlockEnum.ParameterExtractor) { + (node as any).data.model.provider = correctModelProvider( + (node as any).data.model.provider, + ) + } if (node.data.type === BlockEnum.HttpRequest && !node.data.retry_config) { node.data.retry_config = { @@ -426,14 +512,21 @@ export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { } } - if (node.data.type === BlockEnum.Tool && !(node as Node).data.version && !(node as Node).data.tool_node_version) { + if ( + node.data.type === BlockEnum.Tool + && !(node as Node).data.version + && !(node as Node).data.tool_node_version + ) { (node as Node).data.tool_node_version = '2' const toolConfigurations = (node as Node).data.tool_configurations if (toolConfigurations && Object.keys(toolConfigurations).length > 0) { const newValues = { ...toolConfigurations } Object.keys(toolConfigurations).forEach((key) => { - if (typeof toolConfigurations[key] !== 'object' || toolConfigurations[key] === null) { + if ( + typeof toolConfigurations[key] !== 'object' + || toolConfigurations[key] === null + ) { newValues[key] = { type: 'constant', value: toolConfigurations[key], @@ -449,50 +542,62 @@ export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { } export const initialEdges = (originEdges: Edge[], originNodes: Node[]) => { - const { nodes, edges } = preprocessNodesAndEdges(cloneDeep(originNodes), cloneDeep(originEdges)) + const { nodes, edges } = preprocessNodesAndEdges( + cloneDeep(originNodes), + cloneDeep(originEdges), + ) let selectedNode: Node | null = null - const nodesMap = nodes.reduce((acc, node) => { - acc[node.id] = node + const nodesMap = nodes.reduce( + (acc, node) => { + acc[node.id] = node - if (node.data?.selected) - selectedNode = node + if (node.data?.selected) + selectedNode = node - return acc - }, {} as Record) + return acc + }, + {} as Record, + ) const cycleEdges = getCycleEdges(nodes, edges) - return edges.filter((edge) => { - return !cycleEdges.find(cycEdge => cycEdge.source === edge.source && cycEdge.target === edge.target) - }).map((edge) => { - edge.type = 'custom' + return edges + .filter((edge) => { + return !cycleEdges.find( + cycEdge => + cycEdge.source === edge.source && cycEdge.target === edge.target, + ) + }) + .map((edge) => { + edge.type = 'custom' - if (!edge.sourceHandle) - edge.sourceHandle = 'source' + if (!edge.sourceHandle) + edge.sourceHandle = 'source' - if (!edge.targetHandle) - edge.targetHandle = 'target' + if (!edge.targetHandle) + edge.targetHandle = 'target' - if (!edge.data?.sourceType && edge.source && nodesMap[edge.source]) { - edge.data = { - ...edge.data, - sourceType: nodesMap[edge.source].data.type!, - } as any - } + if (!edge.data?.sourceType && edge.source && nodesMap[edge.source]) { + edge.data = { + ...edge.data, + sourceType: nodesMap[edge.source].data.type!, + } as any + } - if (!edge.data?.targetType && edge.target && nodesMap[edge.target]) { - edge.data = { - ...edge.data, - targetType: nodesMap[edge.target].data.type!, - } as any - } + if (!edge.data?.targetType && edge.target && nodesMap[edge.target]) { + edge.data = { + ...edge.data, + targetType: nodesMap[edge.target].data.type!, + } as any + } - if (selectedNode) { - edge.data = { - ...edge.data, - _connectedNodeIsSelected: edge.source === selectedNode.id || edge.target === selectedNode.id, - } as any - } + if (selectedNode) { + edge.data = { + ...edge.data, + _connectedNodeIsSelected: + edge.source === selectedNode.id || edge.target === selectedNode.id, + } as any + } - return edge - }) + return edge + }) } diff --git a/web/eslint-rules/index.js b/web/eslint-rules/index.js index edb6b96ba4..66c2034625 100644 --- a/web/eslint-rules/index.js +++ b/web/eslint-rules/index.js @@ -1,6 +1,8 @@ import noAsAnyInT from './rules/no-as-any-in-t.js' +import noExtraKeys from './rules/no-extra-keys.js' import noLegacyNamespacePrefix from './rules/no-legacy-namespace-prefix.js' import requireNsOption from './rules/require-ns-option.js' +import validI18nKeys from './rules/valid-i18n-keys.js' /** @type {import('eslint').ESLint.Plugin} */ const plugin = { @@ -10,8 +12,10 @@ const plugin = { }, rules: { 'no-as-any-in-t': noAsAnyInT, + 'no-extra-keys': noExtraKeys, 'no-legacy-namespace-prefix': noLegacyNamespacePrefix, 'require-ns-option': requireNsOption, + 'valid-i18n-keys': validI18nKeys, }, } diff --git a/web/eslint-rules/rules/no-extra-keys.js b/web/eslint-rules/rules/no-extra-keys.js new file mode 100644 index 0000000000..eb47f60934 --- /dev/null +++ b/web/eslint-rules/rules/no-extra-keys.js @@ -0,0 +1,70 @@ +import fs from 'node:fs' +import path, { normalize, sep } from 'node:path' + +/** @type {import('eslint').Rule.RuleModule} */ +export default { + meta: { + type: 'problem', + docs: { + description: 'Ensure non-English JSON files don\'t have extra keys not present in en-US', + }, + fixable: 'code', + }, + create(context) { + return { + Program(node) { + const { filename, sourceCode } = context + + if (!filename.endsWith('.json')) + return + + const parts = normalize(filename).split(sep) + // e.g., i18n/ar-TN/common.json -> jsonFile = common.json, lang = ar-TN + const jsonFile = parts.at(-1) + const lang = parts.at(-2) + + // Skip English files + if (lang === 'en-US') + return + + let currentJson = {} + let englishJson = {} + + try { + currentJson = JSON.parse(sourceCode.text) + // Look for the same filename in en-US folder + // e.g., i18n/ar-TN/common.json -> i18n/en-US/common.json + const englishFilePath = path.join(path.dirname(filename), '..', 'en-US', jsonFile ?? '') + englishJson = JSON.parse(fs.readFileSync(englishFilePath, 'utf8')) + } + catch (error) { + context.report({ + node, + message: `Error parsing JSON: ${error instanceof Error ? error.message : String(error)}`, + }) + return + } + + const extraKeys = Object.keys(currentJson).filter( + key => !Object.prototype.hasOwnProperty.call(englishJson, key), + ) + + for (const key of extraKeys) { + context.report({ + node, + message: `Key "${key}" is present in ${lang}/${jsonFile} but not in en-US/${jsonFile}`, + fix(fixer) { + const newJson = Object.fromEntries( + Object.entries(currentJson).filter(([k]) => !extraKeys.includes(k)), + ) + + const newText = `${JSON.stringify(newJson, null, 2)}\n` + + return fixer.replaceText(node, newText) + }, + }) + } + }, + } + }, +} diff --git a/web/eslint-rules/rules/valid-i18n-keys.js b/web/eslint-rules/rules/valid-i18n-keys.js new file mode 100644 index 0000000000..08d863a19a --- /dev/null +++ b/web/eslint-rules/rules/valid-i18n-keys.js @@ -0,0 +1,61 @@ +import { cleanJsonText } from '../utils.js' + +/** @type {import('eslint').Rule.RuleModule} */ +export default { + meta: { + type: 'problem', + docs: { + description: 'Ensure i18n JSON keys are flat and valid as object paths', + }, + }, + create(context) { + return { + Program(node) { + const { filename, sourceCode } = context + + if (!filename.endsWith('.json')) + return + + let json + try { + json = JSON.parse(cleanJsonText(sourceCode.text)) + } + catch { + context.report({ + node, + message: 'Invalid JSON format', + }) + return + } + + const keys = Object.keys(json) + const keyPrefixes = new Set() + + for (const key of keys) { + if (key.includes('.')) { + const parts = key.split('.') + for (let i = 1; i < parts.length; i++) { + const prefix = parts.slice(0, i).join('.') + if (keys.includes(prefix)) { + context.report({ + node, + message: `Invalid key structure: '${key}' conflicts with '${prefix}'`, + }) + } + keyPrefixes.add(prefix) + } + } + } + + for (const key of keys) { + if (keyPrefixes.has(key)) { + context.report({ + node, + message: `Invalid key structure: '${key}' is a prefix of another key`, + }) + } + } + }, + } + }, +} diff --git a/web/eslint-rules/utils.js b/web/eslint-rules/utils.js new file mode 100644 index 0000000000..2030c96b5a --- /dev/null +++ b/web/eslint-rules/utils.js @@ -0,0 +1,10 @@ +export const cleanJsonText = (text) => { + const cleaned = text.replaceAll(/,\s*\}/g, '}') + try { + JSON.parse(cleaned) + return cleaned + } + catch { + return text + } +} diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs index 3cdd3efedb..b8191a5eea 100644 --- a/web/eslint.config.mjs +++ b/web/eslint.config.mjs @@ -130,15 +130,6 @@ export default antfu( sonarjs: sonar, }, }, - // allow generated i18n files (like i18n/*/workflow.ts) to exceed max-lines - { - files: ['i18n/**'], - rules: { - 'sonarjs/max-lines': 'off', - 'max-lines': 'off', - 'jsonc/sort-keys': 'error', - }, - }, tailwind.configs['flat/recommended'], { settings: { @@ -188,7 +179,22 @@ export default antfu( // 'dify-i18n/no-as-any-in-t': ['error', { mode: 'all' }], 'dify-i18n/no-as-any-in-t': 'error', // 'dify-i18n/no-legacy-namespace-prefix': 'error', - 'dify-i18n/require-ns-option': 'error', + // 'dify-i18n/require-ns-option': 'error', + }, + }, + // i18n JSON validation rules + { + files: ['i18n/**/*.json'], + plugins: { + 'dify-i18n': difyI18n, + }, + rules: { + 'sonarjs/max-lines': 'off', + 'max-lines': 'off', + 'jsonc/sort-keys': 'error', + + 'dify-i18n/valid-i18n-keys': 'error', + 'dify-i18n/no-extra-keys': 'error', }, }, ) diff --git a/web/i18n-config/lib.client.ts b/web/i18n-config/lib.client.ts new file mode 100644 index 0000000000..fc40384458 --- /dev/null +++ b/web/i18n-config/lib.client.ts @@ -0,0 +1,10 @@ +'use client' + +import type { NamespaceCamelCase } from './i18next-config' +import { useTranslation as useTranslationOriginal } from 'react-i18next' + +export function useTranslation(ns?: NamespaceCamelCase) { + return useTranslationOriginal(ns) +} + +export { useLocale } from '@/context/i18n' diff --git a/web/i18n-config/lib.server.ts b/web/i18n-config/lib.server.ts new file mode 100644 index 0000000000..4969492cd4 --- /dev/null +++ b/web/i18n-config/lib.server.ts @@ -0,0 +1,16 @@ +import type { NamespaceCamelCase } from './i18next-config' +import { use } from 'react' +import { getLocaleOnServer, getTranslation } from './server' + +async function getI18nConfig(ns?: NamespaceCamelCase) { + const lang = await getLocaleOnServer() + return getTranslation(lang, ns) +} + +export function useTranslation(ns?: NamespaceCamelCase) { + return use(getI18nConfig(ns)) +} + +export function useLocale() { + return use(getLocaleOnServer()) +} diff --git a/web/i18n-config/server.ts b/web/i18n-config/server.ts index 4912e86323..f5966149fd 100644 --- a/web/i18n-config/server.ts +++ b/web/i18n-config/server.ts @@ -2,13 +2,13 @@ import type { i18n as I18nInstance } from 'i18next' import type { Locale } from '.' import type { NamespaceCamelCase, NamespaceKebabCase } from './i18next-config' import { match } from '@formatjs/intl-localematcher' -import { camelCase, kebabCase } from 'es-toolkit/compat' +import { kebabCase } from 'es-toolkit/compat' import { createInstance } from 'i18next' import resourcesToBackend from 'i18next-resources-to-backend' import Negotiator from 'negotiator' import { cookies, headers } from 'next/headers' import { initReactI18next } from 'react-i18next/initReactI18next' -import serverOnlyContext from '@/utils/server-only-context' +import { serverOnlyContext } from '@/utils/server-only-context' import { i18n } from '.' const [getLocaleCache, setLocaleCache] = serverOnlyContext(null) @@ -35,15 +35,14 @@ const getOrCreateI18next = async (lng: Locale) => { return instance } -export async function getTranslation(lng: Locale, ns: NamespaceKebabCase) { - const camelNs = camelCase(ns) as NamespaceCamelCase +export async function getTranslation(lng: Locale, ns?: NamespaceCamelCase) { const i18nextInstance = await getOrCreateI18next(lng) - if (!i18nextInstance.hasLoadedNamespace(camelNs)) - await i18nextInstance.loadNamespaces(camelNs) + if (ns && !i18nextInstance.hasLoadedNamespace(ns)) + await i18nextInstance.loadNamespaces(ns) return { - t: i18nextInstance.getFixedT(lng, camelNs), + t: i18nextInstance.getFixedT(lng, ns), i18n: i18nextInstance, } } diff --git a/web/i18n/ar-TN/common.json b/web/i18n/ar-TN/common.json index beda6bb4c7..d015f1ae0b 100644 --- a/web/i18n/ar-TN/common.json +++ b/web/i18n/ar-TN/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "أوقات الاتصال", "modelProvider.card.buyQuota": "شراء حصة", "modelProvider.card.callTimes": "أوقات الاتصال", - "modelProvider.card.modelAPI": "النماذج {{modelName}} تستخدم مفتاح واجهة برمجة التطبيقات.", - "modelProvider.card.modelNotSupported": "النماذج {{modelName}} غير مثبتة.", - "modelProvider.card.modelSupported": "النماذج {{modelName}} تستخدم هذا الحصة.", "modelProvider.card.onTrial": "في التجربة", "modelProvider.card.paid": "مدفوع", "modelProvider.card.priorityUse": "أولوية الاستخدام", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "الرموز المجانية المتاحة المتبقية", "modelProvider.rerankModel.key": "نموذج إعادة الترتيب", "modelProvider.rerankModel.tip": "سيعيد نموذج إعادة الترتيب ترتيب قائمة المستندات المرشحة بناءً على المطابقة الدلالية مع استعلام المستخدم، مما يحسن نتائج الترتيب الدلالي", - "modelProvider.resetDate": "إعادة الضبط على {{date}}", "modelProvider.searchModel": "نموذج البحث", "modelProvider.selectModel": "اختر نموذجك", "modelProvider.selector.emptySetting": "يرجى الانتقال إلى الإعدادات للتكوين", diff --git a/web/i18n/de-DE/common.json b/web/i18n/de-DE/common.json index 1792c9b7ca..f54f6a939f 100644 --- a/web/i18n/de-DE/common.json +++ b/web/i18n/de-DE/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Anrufzeiten", "modelProvider.card.buyQuota": "Kontingent kaufen", "modelProvider.card.callTimes": "Anrufzeiten", - "modelProvider.card.modelAPI": "{{modelName}}-Modelle verwenden den API-Schlüssel.", - "modelProvider.card.modelNotSupported": "{{modelName}}-Modelle sind nicht installiert.", - "modelProvider.card.modelSupported": "{{modelName}}-Modelle verwenden dieses Kontingent.", "modelProvider.card.onTrial": "In Probe", "modelProvider.card.paid": "Bezahlt", "modelProvider.card.priorityUse": "Priorisierte Nutzung", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Verbleibende verfügbare kostenlose Token", "modelProvider.rerankModel.key": "Rerank-Modell", "modelProvider.rerankModel.tip": "Rerank-Modell wird die Kandidatendokumentenliste basierend auf der semantischen Übereinstimmung mit der Benutzeranfrage neu ordnen und die Ergebnisse der semantischen Rangordnung verbessern", - "modelProvider.resetDate": "Zurücksetzen bei {{date}}", "modelProvider.searchModel": "Suchmodell", "modelProvider.selectModel": "Wählen Sie Ihr Modell", "modelProvider.selector.emptySetting": "Bitte gehen Sie zu den Einstellungen, um zu konfigurieren", diff --git a/web/i18n/es-ES/common.json b/web/i18n/es-ES/common.json index d99c36d9dd..ec08f11ed7 100644 --- a/web/i18n/es-ES/common.json +++ b/web/i18n/es-ES/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Tiempos de llamada", "modelProvider.card.buyQuota": "Comprar Cuota", "modelProvider.card.callTimes": "Tiempos de llamada", - "modelProvider.card.modelAPI": "Los modelos {{modelName}} están usando la clave de API.", - "modelProvider.card.modelNotSupported": "Los modelos {{modelName}} no están instalados.", - "modelProvider.card.modelSupported": "Los modelos {{modelName}} están utilizando esta cuota.", "modelProvider.card.onTrial": "En prueba", "modelProvider.card.paid": "Pagado", "modelProvider.card.priorityUse": "Uso prioritario", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Tokens gratuitos restantes disponibles", "modelProvider.rerankModel.key": "Modelo de Reordenar", "modelProvider.rerankModel.tip": "El modelo de reordenar reordenará la lista de documentos candidatos basada en la coincidencia semántica con la consulta del usuario, mejorando los resultados de clasificación semántica", - "modelProvider.resetDate": "Reiniciar en {{date}}", "modelProvider.searchModel": "Modelo de búsqueda", "modelProvider.selectModel": "Selecciona tu modelo", "modelProvider.selector.emptySetting": "Por favor ve a configuraciones para configurar", diff --git a/web/i18n/fa-IR/common.json b/web/i18n/fa-IR/common.json index 588b37ee43..78f9b9e388 100644 --- a/web/i18n/fa-IR/common.json +++ b/web/i18n/fa-IR/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "تعداد فراخوانی", "modelProvider.card.buyQuota": "خرید سهمیه", "modelProvider.card.callTimes": "تعداد فراخوانی", - "modelProvider.card.modelAPI": "مدل‌های {{modelName}} در حال استفاده از کلید API هستند.", - "modelProvider.card.modelNotSupported": "مدل‌های {{modelName}} نصب نشده‌اند.", - "modelProvider.card.modelSupported": "مدل‌های {{modelName}} از این سهمیه استفاده می‌کنند.", "modelProvider.card.onTrial": "در حال آزمایش", "modelProvider.card.paid": "پرداخت شده", "modelProvider.card.priorityUse": "استفاده با اولویت", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "توکن‌های رایگان باقی‌مانده در دسترس", "modelProvider.rerankModel.key": "مدل رتبه‌بندی مجدد", "modelProvider.rerankModel.tip": "مدل رتبه‌بندی مجدد، لیست اسناد کاندید را بر اساس تطابق معنایی با پرسش کاربر مرتب می‌کند و نتایج رتبه‌بندی معنایی را بهبود می‌بخشد", - "modelProvider.resetDate": "بازنشانی در {{date}}", "modelProvider.searchModel": "جستجوی مدل", "modelProvider.selectModel": "مدل خود را انتخاب کنید", "modelProvider.selector.emptySetting": "لطفاً به تنظیمات بروید تا پیکربندی کنید", diff --git a/web/i18n/fr-FR/common.json b/web/i18n/fr-FR/common.json index 7df7d86272..7cc1af2d80 100644 --- a/web/i18n/fr-FR/common.json +++ b/web/i18n/fr-FR/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Temps d'appel", "modelProvider.card.buyQuota": "Acheter Quota", "modelProvider.card.callTimes": "Temps d'appel", - "modelProvider.card.modelAPI": "Les modèles {{modelName}} utilisent la clé API.", - "modelProvider.card.modelNotSupported": "Les modèles {{modelName}} ne sont pas installés.", - "modelProvider.card.modelSupported": "Les modèles {{modelName}} utilisent ce quota.", "modelProvider.card.onTrial": "En Essai", "modelProvider.card.paid": "Payé", "modelProvider.card.priorityUse": "Utilisation prioritaire", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Tokens gratuits restants disponibles", "modelProvider.rerankModel.key": "Modèle de Réorganisation", "modelProvider.rerankModel.tip": "Le modèle de réorganisation réorganisera la liste des documents candidats en fonction de la correspondance sémantique avec la requête de l'utilisateur, améliorant ainsi les résultats du classement sémantique.", - "modelProvider.resetDate": "Réinitialiser sur {{date}}", "modelProvider.searchModel": "Modèle de recherche", "modelProvider.selectModel": "Sélectionnez votre modèle", "modelProvider.selector.emptySetting": "Veuillez aller dans les paramètres pour configurer", diff --git a/web/i18n/hi-IN/common.json b/web/i18n/hi-IN/common.json index 996880d51c..4670d5a545 100644 --- a/web/i18n/hi-IN/common.json +++ b/web/i18n/hi-IN/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "कॉल समय", "modelProvider.card.buyQuota": "कोटा खरीदें", "modelProvider.card.callTimes": "कॉल समय", - "modelProvider.card.modelAPI": "{{modelName}} मॉडल एपीआई कुंजी का उपयोग कर रहे हैं।", - "modelProvider.card.modelNotSupported": "{{modelName}} मॉडल इंस्टॉल नहीं हैं।", - "modelProvider.card.modelSupported": "{{modelName}} मॉडल इस कोटा का उपयोग कर रहे हैं।", "modelProvider.card.onTrial": "परीक्षण पर", "modelProvider.card.paid": "भुगतान किया हुआ", "modelProvider.card.priorityUse": "प्राथमिकता उपयोग", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "बचे हुए उपलब्ध मुफ्त टोकन", "modelProvider.rerankModel.key": "रीरैंक मॉडल", "modelProvider.rerankModel.tip": "रीरैंक मॉडल उपयोगकर्ता प्रश्न के साथ सांविधिक मेल के आधार पर उम्मीदवार दस्तावेज़ सूची को पुनः क्रमित करेगा, सांविधिक रैंकिंग के परिणामों में सुधार करेगा।", - "modelProvider.resetDate": "{{date}} पर रीसेट करें", "modelProvider.searchModel": "खोज मॉडल", "modelProvider.selectModel": "अपने मॉडल का चयन करें", "modelProvider.selector.emptySetting": "कॉन्फ़िगर करने के लिए कृपया सेटिंग्स पर जाएं", diff --git a/web/i18n/id-ID/common.json b/web/i18n/id-ID/common.json index 710136c7e2..ede4d3ae44 100644 --- a/web/i18n/id-ID/common.json +++ b/web/i18n/id-ID/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Waktu panggilan", "modelProvider.card.buyQuota": "Beli Kuota", "modelProvider.card.callTimes": "Waktu panggilan", - "modelProvider.card.modelAPI": "Model {{modelName}} sedang menggunakan API Key.", - "modelProvider.card.modelNotSupported": "Model {{modelName}} tidak terpasang.", - "modelProvider.card.modelSupported": "Model {{modelName}} sedang menggunakan kuota ini.", "modelProvider.card.onTrial": "Sedang Diadili", "modelProvider.card.paid": "Dibayar", "modelProvider.card.priorityUse": "Penggunaan prioritas", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Token gratis yang masih tersedia", "modelProvider.rerankModel.key": "Peringkat ulang Model", "modelProvider.rerankModel.tip": "Model rerank akan menyusun ulang daftar dokumen kandidat berdasarkan kecocokan semantik dengan kueri pengguna, meningkatkan hasil peringkat semantik", - "modelProvider.resetDate": "Atur ulang pada {{date}}", "modelProvider.searchModel": "Model pencarian", "modelProvider.selectModel": "Pilih model Anda", "modelProvider.selector.emptySetting": "Silakan buka pengaturan untuk mengonfigurasi", diff --git a/web/i18n/it-IT/common.json b/web/i18n/it-IT/common.json index 64bf2e3d1d..737ef923b1 100644 --- a/web/i18n/it-IT/common.json +++ b/web/i18n/it-IT/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Numero di chiamate", "modelProvider.card.buyQuota": "Acquista Quota", "modelProvider.card.callTimes": "Numero di chiamate", - "modelProvider.card.modelAPI": "I modelli {{modelName}} stanno utilizzando la chiave API.", - "modelProvider.card.modelNotSupported": "I modelli {{modelName}} non sono installati.", - "modelProvider.card.modelSupported": "I modelli {{modelName}} stanno utilizzando questa quota.", "modelProvider.card.onTrial": "In Prova", "modelProvider.card.paid": "Pagato", "modelProvider.card.priorityUse": "Uso prioritario", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Token gratuiti rimanenti disponibili", "modelProvider.rerankModel.key": "Modello di Rerank", "modelProvider.rerankModel.tip": "Il modello di rerank riordinerà la lista dei documenti candidati basandosi sulla corrispondenza semantica con la query dell'utente, migliorando i risultati del ranking semantico", - "modelProvider.resetDate": "Reimposta su {{date}}", "modelProvider.searchModel": "Modello di ricerca", "modelProvider.selectModel": "Seleziona il tuo modello", "modelProvider.selector.emptySetting": "Per favore vai alle impostazioni per configurare", diff --git a/web/i18n/ko-KR/common.json b/web/i18n/ko-KR/common.json index fa3736e561..5640cb353d 100644 --- a/web/i18n/ko-KR/common.json +++ b/web/i18n/ko-KR/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "호출 횟수", "modelProvider.card.buyQuota": "Buy Quota", "modelProvider.card.callTimes": "호출 횟수", - "modelProvider.card.modelAPI": "{{modelName}} 모델이 API 키를 사용하고 있습니다.", - "modelProvider.card.modelNotSupported": "{{modelName}} 모델이 설치되지 않았습니다.", - "modelProvider.card.modelSupported": "{{modelName}} 모델이 이 할당량을 사용하고 있습니다.", "modelProvider.card.onTrial": "트라이얼 중", "modelProvider.card.paid": "유료", "modelProvider.card.priorityUse": "우선 사용", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "남은 무료 토큰 사용 가능", "modelProvider.rerankModel.key": "재랭크 모델", "modelProvider.rerankModel.tip": "재랭크 모델은 사용자 쿼리와의 의미적 일치를 기반으로 후보 문서 목록을 재배열하여 의미적 순위를 향상시킵니다.", - "modelProvider.resetDate": "{{date}}에서 재설정", "modelProvider.searchModel": "검색 모델", "modelProvider.selectModel": "모델 선택", "modelProvider.selector.emptySetting": "설정으로 이동하여 구성하세요", diff --git a/web/i18n/pl-PL/common.json b/web/i18n/pl-PL/common.json index 1a83dc517e..ae654e04ac 100644 --- a/web/i18n/pl-PL/common.json +++ b/web/i18n/pl-PL/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Czasy wywołań", "modelProvider.card.buyQuota": "Kup limit", "modelProvider.card.callTimes": "Czasy wywołań", - "modelProvider.card.modelAPI": "Modele {{modelName}} używają klucza API.", - "modelProvider.card.modelNotSupported": "Modele {{modelName}} nie są zainstalowane.", - "modelProvider.card.modelSupported": "{{modelName}} modeli korzysta z tej kwoty.", "modelProvider.card.onTrial": "Na próbę", "modelProvider.card.paid": "Płatny", "modelProvider.card.priorityUse": "Używanie z priorytetem", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Pozostałe dostępne darmowe tokeny", "modelProvider.rerankModel.key": "Model ponownego rankingu", "modelProvider.rerankModel.tip": "Model ponownego rankingu zmieni kolejność listy dokumentów kandydatów na podstawie semantycznego dopasowania z zapytaniem użytkownika, poprawiając wyniki rankingu semantycznego", - "modelProvider.resetDate": "Reset na {{date}}", "modelProvider.searchModel": "Model wyszukiwania", "modelProvider.selectModel": "Wybierz swój model", "modelProvider.selector.emptySetting": "Przejdź do ustawień, aby skonfigurować", diff --git a/web/i18n/pt-BR/common.json b/web/i18n/pt-BR/common.json index e97e4364ad..2e7f49de7e 100644 --- a/web/i18n/pt-BR/common.json +++ b/web/i18n/pt-BR/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Chamadas", "modelProvider.card.buyQuota": "Comprar Quota", "modelProvider.card.callTimes": "Chamadas", - "modelProvider.card.modelAPI": "Os modelos {{modelName}} estão usando a Chave de API.", - "modelProvider.card.modelNotSupported": "Modelos {{modelName}} não estão instalados.", - "modelProvider.card.modelSupported": "Modelos {{modelName}} estão usando esta cota.", "modelProvider.card.onTrial": "Em Teste", "modelProvider.card.paid": "Pago", "modelProvider.card.priorityUse": "Uso prioritário", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Tokens gratuitos disponíveis restantes", "modelProvider.rerankModel.key": "Modelo de Reordenação", "modelProvider.rerankModel.tip": "O modelo de reordenaenação reorganizará a lista de documentos candidatos com base na correspondência semântica com a consulta do usuário, melhorando os resultados da classificação semântica", - "modelProvider.resetDate": "Redefinir em {{date}}", "modelProvider.searchModel": "Modelo de pesquisa", "modelProvider.selectModel": "Selecione seu modelo", "modelProvider.selector.emptySetting": "Por favor, vá para configurações para configurar", diff --git a/web/i18n/ro-RO/common.json b/web/i18n/ro-RO/common.json index 785050d8ec..c21e755b3c 100644 --- a/web/i18n/ro-RO/common.json +++ b/web/i18n/ro-RO/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Apeluri", "modelProvider.card.buyQuota": "Cumpără cotă", "modelProvider.card.callTimes": "Apeluri", - "modelProvider.card.modelAPI": "Modelele {{modelName}} folosesc cheia API.", - "modelProvider.card.modelNotSupported": "Modelele {{modelName}} nu sunt instalate.", - "modelProvider.card.modelSupported": "{{modelName}} modele utilizează această cotă.", "modelProvider.card.onTrial": "În probă", "modelProvider.card.paid": "Plătit", "modelProvider.card.priorityUse": "Utilizare prioritară", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Jetoane gratuite disponibile rămase", "modelProvider.rerankModel.key": "Model de reordonare", "modelProvider.rerankModel.tip": "Modelul de reordonare va reordona lista de documente candidate pe baza potrivirii semantice cu interogarea utilizatorului, îmbunătățind rezultatele clasificării semantice", - "modelProvider.resetDate": "Resetați la {{date}}", "modelProvider.searchModel": "Model de căutare", "modelProvider.selectModel": "Selectați modelul dvs.", "modelProvider.selector.emptySetting": "Vă rugăm să mergeți la setări pentru a configura", diff --git a/web/i18n/ru-RU/common.json b/web/i18n/ru-RU/common.json index 63f3758185..e763a7ec2a 100644 --- a/web/i18n/ru-RU/common.json +++ b/web/i18n/ru-RU/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Количество вызовов", "modelProvider.card.buyQuota": "Купить квоту", "modelProvider.card.callTimes": "Количество вызовов", - "modelProvider.card.modelAPI": "{{modelName}} модели используют ключ API.", - "modelProvider.card.modelNotSupported": "Модели {{modelName}} не установлены.", - "modelProvider.card.modelSupported": "Эту квоту используют модели {{modelName}}.", "modelProvider.card.onTrial": "Пробная версия", "modelProvider.card.paid": "Платный", "modelProvider.card.priorityUse": "Приоритетное использование", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Оставшиеся доступные бесплатные токены", "modelProvider.rerankModel.key": "Модель повторного ранжирования", "modelProvider.rerankModel.tip": "Модель повторного ранжирования изменит порядок списка документов-кандидатов на основе семантического соответствия запросу пользователя, улучшая результаты семантического ранжирования", - "modelProvider.resetDate": "Сброс на {{date}}", "modelProvider.searchModel": "Поиск модели", "modelProvider.selectModel": "Выберите свою модель", "modelProvider.selector.emptySetting": "Пожалуйста, перейдите в настройки для настройки", diff --git a/web/i18n/sl-SI/common.json b/web/i18n/sl-SI/common.json index be5a4e5320..d092fe10c8 100644 --- a/web/i18n/sl-SI/common.json +++ b/web/i18n/sl-SI/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Število klicev", "modelProvider.card.buyQuota": "Kupi kvoto", "modelProvider.card.callTimes": "Časi klicev", - "modelProvider.card.modelAPI": "{{modelName}} modeli uporabljajo API ključ.", - "modelProvider.card.modelNotSupported": "{{modelName}} modeli niso nameščeni.", - "modelProvider.card.modelSupported": "{{modelName}} modeli uporabljajo to kvoto.", "modelProvider.card.onTrial": "Na preizkusu", "modelProvider.card.paid": "Plačano", "modelProvider.card.priorityUse": "Prednostna uporaba", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Preostali razpoložljivi brezplačni žetoni", "modelProvider.rerankModel.key": "Model za prerazvrstitev", "modelProvider.rerankModel.tip": "Model za prerazvrstitev bo prerazporedil seznam kandidatskih dokumentov na podlagi semantične ujemanja z uporabniško poizvedbo, s čimer se izboljšajo rezultati semantičnega razvrščanja.", - "modelProvider.resetDate": "Ponastavi na {{date}}", "modelProvider.searchModel": "Model iskanja", "modelProvider.selectModel": "Izberite svoj model", "modelProvider.selector.emptySetting": "Prosimo, pojdite v nastavitve za konfiguracijo", diff --git a/web/i18n/th-TH/common.json b/web/i18n/th-TH/common.json index c6dd9c9259..9a38f7f683 100644 --- a/web/i18n/th-TH/common.json +++ b/web/i18n/th-TH/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "เวลาโทร", "modelProvider.card.buyQuota": "ซื้อโควต้า", "modelProvider.card.callTimes": "เวลาโทร", - "modelProvider.card.modelAPI": "{{modelName}} โมเดลกำลังใช้คีย์ API", - "modelProvider.card.modelNotSupported": "โมเดล {{modelName}} ยังไม่ได้ติดตั้ง", - "modelProvider.card.modelSupported": "โมเดล {{modelName}} กำลังใช้โควต้านี้อยู่", "modelProvider.card.onTrial": "ทดลองใช้", "modelProvider.card.paid": "จ่าย", "modelProvider.card.priorityUse": "ลําดับความสําคัญในการใช้งาน", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "โทเค็นฟรีที่เหลืออยู่", "modelProvider.rerankModel.key": "จัดอันดับโมเดลใหม่", "modelProvider.rerankModel.tip": "โมเดล Rerank จะจัดลําดับรายการเอกสารผู้สมัครใหม่ตามการจับคู่ความหมายกับการสืบค้นของผู้ใช้ ซึ่งช่วยปรับปรุงผลลัพธ์ของการจัดอันดับความหมาย", - "modelProvider.resetDate": "รีเซ็ตเมื่อ {{date}}", "modelProvider.searchModel": "ค้นหารุ่น", "modelProvider.selectModel": "เลือกรุ่นของคุณ", "modelProvider.selector.emptySetting": "โปรดไปที่การตั้งค่าเพื่อกําหนดค่า", diff --git a/web/i18n/tr-TR/common.json b/web/i18n/tr-TR/common.json index 68d4358281..0ee51e161c 100644 --- a/web/i18n/tr-TR/common.json +++ b/web/i18n/tr-TR/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Çağrı Süreleri", "modelProvider.card.buyQuota": "Kota Satın Al", "modelProvider.card.callTimes": "Çağrı Süreleri", - "modelProvider.card.modelAPI": "{{modelName}} modelleri API Anahtarını kullanıyor.", - "modelProvider.card.modelNotSupported": "{{modelName}} modelleri yüklü değil.", - "modelProvider.card.modelSupported": "{{modelName}} modelleri bu kotayı kullanıyor.", "modelProvider.card.onTrial": "Deneme Sürümünde", "modelProvider.card.paid": "Ücretli", "modelProvider.card.priorityUse": "Öncelikli Kullan", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Kalan kullanılabilir ücretsiz tokenler", "modelProvider.rerankModel.key": "Yeniden Sıralama Modeli", "modelProvider.rerankModel.tip": "Yeniden sıralama modeli, kullanıcı sorgusuyla anlam eşleştirmesine dayalı olarak aday belge listesini yeniden sıralayacak ve anlam sıralama sonuçlarını iyileştirecektir.", - "modelProvider.resetDate": "{{date}} üzerine sıfırlama", "modelProvider.searchModel": "Model ara", "modelProvider.selectModel": "Modelinizi seçin", "modelProvider.selector.emptySetting": "Lütfen ayarlara gidip yapılandırın", diff --git a/web/i18n/uk-UA/common.json b/web/i18n/uk-UA/common.json index 7e3b7fe05f..ddec8637e1 100644 --- a/web/i18n/uk-UA/common.json +++ b/web/i18n/uk-UA/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Кількість викликів", "modelProvider.card.buyQuota": "Придбати квоту", "modelProvider.card.callTimes": "Кількість викликів", - "modelProvider.card.modelAPI": "Моделі {{modelName}} використовують API-ключ.", - "modelProvider.card.modelNotSupported": "Моделі {{modelName}} не встановлені.", - "modelProvider.card.modelSupported": "Моделі {{modelName}} використовують цю квоту.", "modelProvider.card.onTrial": "У пробному періоді", "modelProvider.card.paid": "Оплачено", "modelProvider.card.priorityUse": "Пріоритетне використання", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Залишилося доступних безкоштовних токенів", "modelProvider.rerankModel.key": "Модель повторного ранжування", "modelProvider.rerankModel.tip": "Модель повторного ранжування змінить порядок списку документів-кандидатів на основі семантичної відповідності запиту користувача, покращуючи результати семантичного ранжування.", - "modelProvider.resetDate": "Скинути на {{date}}", "modelProvider.searchModel": "Пошукова модель", "modelProvider.selectModel": "Виберіть свою модель", "modelProvider.selector.emptySetting": "Перейдіть до налаштувань, щоб налаштувати", diff --git a/web/i18n/vi-VN/common.json b/web/i18n/vi-VN/common.json index 20364c3ee9..f8fa9c07d5 100644 --- a/web/i18n/vi-VN/common.json +++ b/web/i18n/vi-VN/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "Số lần gọi", "modelProvider.card.buyQuota": "Mua Quota", "modelProvider.card.callTimes": "Số lần gọi", - "modelProvider.card.modelAPI": "Các mô hình {{modelName}} đang sử dụng Khóa API.", - "modelProvider.card.modelNotSupported": "Các mô hình {{modelName}} chưa được cài đặt.", - "modelProvider.card.modelSupported": "{{modelName}} mô hình đang sử dụng hạn mức này.", "modelProvider.card.onTrial": "Thử nghiệm", "modelProvider.card.paid": "Đã thanh toán", "modelProvider.card.priorityUse": "Ưu tiên sử dụng", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "Số lượng mã thông báo miễn phí còn lại", "modelProvider.rerankModel.key": "Mô hình Sắp xếp lại", "modelProvider.rerankModel.tip": "Mô hình sắp xếp lại sẽ sắp xếp lại danh sách tài liệu ứng cử viên dựa trên sự phù hợp ngữ nghĩa với truy vấn của người dùng, cải thiện kết quả của việc xếp hạng ngữ nghĩa", - "modelProvider.resetDate": "Đặt lại vào {{date}}", "modelProvider.searchModel": "Mô hình tìm kiếm", "modelProvider.selectModel": "Chọn mô hình của bạn", "modelProvider.selector.emptySetting": "Vui lòng vào cài đặt để cấu hình", diff --git a/web/i18n/zh-Hant/common.json b/web/i18n/zh-Hant/common.json index 11846c9772..8fe3e5bd07 100644 --- a/web/i18n/zh-Hant/common.json +++ b/web/i18n/zh-Hant/common.json @@ -339,9 +339,6 @@ "modelProvider.callTimes": "呼叫次數", "modelProvider.card.buyQuota": "購買額度", "modelProvider.card.callTimes": "呼叫次數", - "modelProvider.card.modelAPI": "{{modelName}} 模型正在使用 API 金鑰。", - "modelProvider.card.modelNotSupported": "{{modelName}} 模型未安裝。", - "modelProvider.card.modelSupported": "{{modelName}} 模型正在使用這個配額。", "modelProvider.card.onTrial": "試用中", "modelProvider.card.paid": "已購買", "modelProvider.card.priorityUse": "優先使用", @@ -397,7 +394,6 @@ "modelProvider.quotaTip": "剩餘免費額度", "modelProvider.rerankModel.key": "Rerank 模型", "modelProvider.rerankModel.tip": "重排序模型將根據候選文件列表與使用者問題語義匹配度進行重新排序，從而改進語義排序的結果", - "modelProvider.resetDate": "在 {{date}} 重置", "modelProvider.searchModel": "搜尋模型", "modelProvider.selectModel": "選擇您的模型", "modelProvider.selector.emptySetting": "請前往設定進行配置", diff --git a/web/package.json b/web/package.json index 3f05f90e09..62a484b483 100644 --- a/web/package.json +++ b/web/package.json @@ -4,6 +4,12 @@ "version": "1.11.2", "private": true, "packageManager": "pnpm@10.27.0+sha512.72d699da16b1179c14ba9e64dc71c9a40988cbdc65c264cb0e489db7de917f20dcf4d64d8723625f2969ba52d4b7e2a1170682d9ac2a5dcaeaab732b7e16f04a", + "imports": { + "#i18n": { + "react-server": "./i18n-config/lib.server.ts", + "default": "./i18n-config/lib.client.ts" + } + }, "engines": { "node": ">=v22.11.0" }, @@ -47,7 +53,7 @@ "knip": "knip" }, "dependencies": { - "@amplitude/analytics-browser": "^2.31.3", + "@amplitude/analytics-browser": "^2.33.1", "@amplitude/plugin-session-replay-browser": "^1.23.6", "@emoji-mart/data": "^1.2.1", "@floating-ui/react": "^0.26.28", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index 9ffb092c6e..d9f90b62f3 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -58,8 +58,8 @@ importers: .: dependencies: '@amplitude/analytics-browser': - specifier: ^2.31.3 - version: 2.31.4 + specifier: ^2.33.1 + version: 2.33.1 '@amplitude/plugin-session-replay-browser': specifier: ^1.23.6 version: 1.24.1(@amplitude/rrweb@2.0.0-alpha.33)(rollup@2.79.2) @@ -578,8 +578,8 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} - '@amplitude/analytics-browser@2.31.4': - resolution: {integrity: sha512-9O8a0SK55tQOgJJ0z9eE+q/C2xWo6a65wN4iSglxYwm1vvGJKG6Z/QV4XKQ6X0syGscRuG1XoMc0mt3xdVPtDg==} + '@amplitude/analytics-browser@2.33.1': + resolution: {integrity: sha512-93wZjuAFJ7QdyptF82i1pezm5jKuBWITHI++XshDgpks1RstJvJ9n11Ak8MnE4L2BGQ93XDN2aVEHfmQkt0/Pw==} '@amplitude/analytics-client-common@2.4.16': resolution: {integrity: sha512-qF7NAl6Qr6QXcWKnldGJfO0Kp1TYoy1xsmzEDnOYzOS96qngtvsZ8MuKya1lWdVACoofwQo82V0VhNZJKk/2YA==} @@ -590,29 +590,32 @@ packages: '@amplitude/analytics-core@2.33.0': resolution: {integrity: sha512-56m0R12TjZ41D2YIghb/XNHSdL4CurAVyRT3L2FD+9DCFfbgjfT8xhDBnsZtA+aBkb6Yak1EGUojGBunfAm2/A==} + '@amplitude/analytics-core@2.35.0': + resolution: {integrity: sha512-7RmHYELXCGu8yuO9D6lEXiqkMtiC5sePNhCWmwuP30dneDYHtH06gaYvAFH/YqOFuE6enwEEJfFYtcaPhyiqtA==} + '@amplitude/analytics-types@2.11.0': resolution: {integrity: sha512-L1niBXYSWmbyHUE/GNuf6YBljbafaxWI3X5jjEIZDFCjQvdWO3DKalY1VPFUbhgYQgWw7+bC6I/AlUaporyfig==} '@amplitude/experiment-core@0.7.2': resolution: {integrity: sha512-Wc2NWvgQ+bLJLeF0A9wBSPIaw0XuqqgkPKsoNFQrmS7r5Djd56um75In05tqmVntPJZRvGKU46pAp8o5tdf4mA==} - '@amplitude/plugin-autocapture-browser@1.18.0': - resolution: {integrity: sha512-hBBZpghTEnl+XF8UZaGxe1xCbSjawdmOkJC0/tQF2k1FwlJS/rdWBGmPd8wH7iU4hd55pnSw28Kd2NL7q0zTcA==} + '@amplitude/plugin-autocapture-browser@1.18.3': + resolution: {integrity: sha512-njYque5t1QCEEe5V8Ls4yVVklTM6V7OXxBk6pqznN/hj/Pc4X8Wjy898pZ2VtbnvpagBKKzGb5B6Syl8OXiicw==} - '@amplitude/plugin-network-capture-browser@1.7.0': - resolution: {integrity: sha512-tlwkBL0tlc1OUTT2XYTjWx4mm6O0DSggKzkkDq+8DhW+ZFl9OfHMFIh/hDLJzxs1LTtX7CvFUfAVSDifJOs+NA==} + '@amplitude/plugin-network-capture-browser@1.7.3': + resolution: {integrity: sha512-zfWgAN7g6AigJAsgrGmlgVwydOHH6XvweBoxhU+qEvRydboiIVCDLSxuXczUsBG7kYVLWRdBK1DYoE5J7lqTGA==} - '@amplitude/plugin-page-url-enrichment-browser@0.5.6': - resolution: {integrity: sha512-H6+tf0zYhvM+8oJsdC/kAbIzuxOY/0p+3HBmX4K+G4doo5nCGAB0DYTr6dqMp1GcPOZ09pKT41+DJ6vwSy4ypQ==} + '@amplitude/plugin-page-url-enrichment-browser@0.5.9': + resolution: {integrity: sha512-TqdELx4WrdRutCjHUFUzum/f/UjhbdTZw0UKkYFAj5gwAKDjaPEjL4waRvINOTaVLsne1A6ck4KEMfC8AKByFw==} - '@amplitude/plugin-page-view-tracking-browser@2.6.3': - resolution: {integrity: sha512-lLU4W2r5jXtfn/14cZKM9c9CQDxT7PVVlgm0susHJ3Kfsua9jJQuMHs4Zlg6rwByAtZi5nF4nYE5z0GF09gx0A==} + '@amplitude/plugin-page-view-tracking-browser@2.6.6': + resolution: {integrity: sha512-dBcJlrdKgPzSgS3exDRRrMLqhIaOjwlIy7o8sEMn1PpMawERlbumSSdtfII6L4L67HYUPo4PY4Kp4acqSzaLvQ==} '@amplitude/plugin-session-replay-browser@1.24.1': resolution: {integrity: sha512-NHePIu2Yv9ba+fOt5N33b8FFQPzyKvjs1BnWBgBCM5RECos3w6n/+zUWTnTJ4at2ipO2lz111abKDteUwbuptg==} - '@amplitude/plugin-web-vitals-browser@1.1.0': - resolution: {integrity: sha512-TA0X4Np4Wt5hkQ4+Ouhg6nm2xjDd9l03OV9N8Kbe1cqpr/sxvRwSpd+kp2eREbp6D7tHFFkKJA2iNtxbE5Y0cA==} + '@amplitude/plugin-web-vitals-browser@1.1.4': + resolution: {integrity: sha512-XQXI9OjTNSz2yi0lXw2VYMensDzzSkMCfvXNniTb1LgnHwBcQ1JWPcTqHLPFrvvNckeIdOT78vjs7yA+c1FyzA==} '@amplitude/rrdom@2.0.0-alpha.33': resolution: {integrity: sha512-uu+1w1RGEJ7QcGPwCC898YBR47DpNYOZTnQMY9/IgMzTXQ0+Hh1/JLsQfMnBBtAePhvCS0BlHd/qGD5w0taIcg==} @@ -8734,8 +8737,8 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} - web-vitals@5.0.1: - resolution: {integrity: sha512-BsULPWaCKAAtNntUz0aJq1cu1wyuWmDzf4N6vYNMbYA6zzQAf2pzCYbyClf+Ui2MI54bt225AwugXIfL1W+Syg==} + web-vitals@5.1.0: + resolution: {integrity: sha512-ArI3kx5jI0atlTtmV0fWU3fjpLmq/nD3Zr1iFFlJLaqa5wLBkUSzINwBPySCX/8jRyjlmy1Volw1kz1g9XE4Jg==} webidl-conversions@4.0.2: resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==} @@ -9016,14 +9019,14 @@ snapshots: '@alloc/quick-lru@5.2.0': {} - '@amplitude/analytics-browser@2.31.4': + '@amplitude/analytics-browser@2.33.1': dependencies: - '@amplitude/analytics-core': 2.33.0 - '@amplitude/plugin-autocapture-browser': 1.18.0 - '@amplitude/plugin-network-capture-browser': 1.7.0 - '@amplitude/plugin-page-url-enrichment-browser': 0.5.6 - '@amplitude/plugin-page-view-tracking-browser': 2.6.3 - '@amplitude/plugin-web-vitals-browser': 1.1.0 + '@amplitude/analytics-core': 2.35.0 + '@amplitude/plugin-autocapture-browser': 1.18.3 + '@amplitude/plugin-network-capture-browser': 1.7.3 + '@amplitude/plugin-page-url-enrichment-browser': 0.5.9 + '@amplitude/plugin-page-view-tracking-browser': 2.6.6 + '@amplitude/plugin-web-vitals-browser': 1.1.4 tslib: 2.8.1 '@amplitude/analytics-client-common@2.4.16': @@ -9041,31 +9044,37 @@ snapshots: tslib: 2.8.1 zen-observable-ts: 1.1.0 + '@amplitude/analytics-core@2.35.0': + dependencies: + '@amplitude/analytics-connector': 1.6.4 + tslib: 2.8.1 + zen-observable-ts: 1.1.0 + '@amplitude/analytics-types@2.11.0': {} '@amplitude/experiment-core@0.7.2': dependencies: js-base64: 3.7.8 - '@amplitude/plugin-autocapture-browser@1.18.0': + '@amplitude/plugin-autocapture-browser@1.18.3': dependencies: - '@amplitude/analytics-core': 2.33.0 + '@amplitude/analytics-core': 2.35.0 rxjs: 7.8.2 tslib: 2.8.1 - '@amplitude/plugin-network-capture-browser@1.7.0': + '@amplitude/plugin-network-capture-browser@1.7.3': dependencies: - '@amplitude/analytics-core': 2.33.0 + '@amplitude/analytics-core': 2.35.0 tslib: 2.8.1 - '@amplitude/plugin-page-url-enrichment-browser@0.5.6': + '@amplitude/plugin-page-url-enrichment-browser@0.5.9': dependencies: - '@amplitude/analytics-core': 2.33.0 + '@amplitude/analytics-core': 2.35.0 tslib: 2.8.1 - '@amplitude/plugin-page-view-tracking-browser@2.6.3': + '@amplitude/plugin-page-view-tracking-browser@2.6.6': dependencies: - '@amplitude/analytics-core': 2.33.0 + '@amplitude/analytics-core': 2.35.0 tslib: 2.8.1 '@amplitude/plugin-session-replay-browser@1.24.1(@amplitude/rrweb@2.0.0-alpha.33)(rollup@2.79.2)': @@ -9080,11 +9089,11 @@ snapshots: - '@amplitude/rrweb' - rollup - '@amplitude/plugin-web-vitals-browser@1.1.0': + '@amplitude/plugin-web-vitals-browser@1.1.4': dependencies: - '@amplitude/analytics-core': 2.33.0 + '@amplitude/analytics-core': 2.35.0 tslib: 2.8.1 - web-vitals: 5.0.1 + web-vitals: 5.1.0 '@amplitude/rrdom@2.0.0-alpha.33': dependencies: @@ -9148,7 +9157,7 @@ snapshots: '@amplitude/targeting@0.2.0': dependencies: '@amplitude/analytics-client-common': 2.4.16 - '@amplitude/analytics-core': 2.33.0 + '@amplitude/analytics-core': 2.35.0 '@amplitude/analytics-types': 2.11.0 '@amplitude/experiment-core': 0.7.2 idb: 8.0.0 @@ -18336,7 +18345,7 @@ snapshots: web-namespaces@2.0.1: {} - web-vitals@5.0.1: {} + web-vitals@5.1.0: {} webidl-conversions@4.0.2: {} diff --git a/web/types/app.ts b/web/types/app.ts index eb1b29bb60..c8c409a22c 100644 --- a/web/types/app.ts +++ b/web/types/app.ts @@ -316,7 +316,7 @@ export type SiteConfig = { use_icon_as_answer_icon: boolean } -export type AppIconType = 'image' | 'emoji' +export type AppIconType = 'image' | 'emoji' | 'link' /** * App diff --git a/web/utils/server-only-context.ts b/web/utils/server-only-context.ts index e58dbfe98b..43fe743811 100644 --- a/web/utils/server-only-context.ts +++ b/web/utils/server-only-context.ts @@ -2,7 +2,7 @@ import { cache } from 'react' -export default (defaultValue: T): [() => T, (v: T) => void] => { +export function serverOnlyContext(defaultValue: T): [() => T, (v: T) => void] { const getRef = cache(() => ({ current: defaultValue })) const getValue = (): T => getRef().current