Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion api/core/agent/base_agent_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from extensions.ext_database import db
from factories import file_factory
from graphon.file import file_manager
from graphon.file.enums import FileType
from graphon.model_runtime.entities import (
AssistantPromptMessage,
LLMUsage,
Expand All @@ -53,6 +54,16 @@
logger = logging.getLogger(__name__)
_file_access_controller = DatabaseFileAccessController()

# Maps each file type to the model feature required to pass it to the LLM.
# Mirrors _REQUIRED_MODEL_FEATURE_BY_CONTENT_TYPE in graphon so that non-image
# files (e.g. documents) are not dropped just because the model lacks vision.
_REQUIRED_MODEL_FEATURE_BY_FILE_TYPE: dict[FileType, ModelFeature] = {
FileType.IMAGE: ModelFeature.VISION,
FileType.DOCUMENT: ModelFeature.DOCUMENT,
FileType.AUDIO: ModelFeature.AUDIO,
FileType.VIDEO: ModelFeature.VIDEO,
}


class BaseAgentRunner(AppRunner):
def __init__(
Expand Down Expand Up @@ -131,7 +142,15 @@ def __init__(
features = model_schema.features if model_schema and model_schema.features else []
self.stream_tool_call = ModelFeature.STREAM_TOOL_CALL in features
self.vision_enabled = ModelFeature.VISION in features
self.files = application_generate_entity.files if self.vision_enabled else []
# Filter files by the model feature each file type requires, instead of
# gating all files on vision alone. Without this, document/audio/video
# files are silently dropped when the model does not support vision.
file_type_feature = _REQUIRED_MODEL_FEATURE_BY_FILE_TYPE.get
self.files = [
f
for f in (application_generate_entity.files or [])
if not hasattr(f, "type") or file_type_feature(f.type, ModelFeature.VISION) in features
]
self.query: str = ""
self._current_thoughts: list[PromptMessage] = []

Expand Down
64 changes: 64 additions & 0 deletions api/tests/unit_tests/core/agent/test_base_agent_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from core.app.entities.app_invoke_entities import (
AgentChatAppGenerateEntity,
InvokeFrom,
ModelConfigWithCredentialsEntity,
)
from core.model_manager import ModelInstance
from core.tools.__base.tool import Tool
Expand Down Expand Up @@ -767,3 +768,66 @@ def test_init_uses_real_session_for_count_and_dependencies(
assert initialized.agent_thought_count == 2
assert initialized.history_prompt_messages == []
assert get_dataset_tools.call_args.kwargs["session"] is sqlite_session


def test_init_filters_files_by_type_not_vision(sqlite_session: Session, mocker: MockerFixture) -> None:
"""Regression test for https://github.com/langgenius/dify/issues/39431

Documents should be kept when the model supports DOCUMENT, even if it
does not support VISION. The old code gated all files on VISION alone.
"""
from graphon.file.enums import FileTransferMethod, FileType
from graphon.file.models import File

caller_session = mocker.MagicMock()
caller_session.scalar.return_value = 0
global_session = mocker.MagicMock()
mocker.patch.object(module.db, "session", global_session)
mocker.patch.object(BaseAgentRunner, "organize_agent_history", return_value=[])
mocker.patch.object(module.DatasetRetrieverTool, "get_dataset_tools", return_value=[])

llm = mocker.MagicMock()
# Model supports DOCUMENT but NOT VISION
llm.get_model_schema.return_value = mocker.MagicMock(features=[module.ModelFeature.DOCUMENT])
model_instance = mocker.MagicMock(model_type_instance=llm, model="m", credentials="c")

app_config = mocker.MagicMock()
app_config.app_id = "app1"
app_config.agent = None
app_config.dataset = mocker.MagicMock(dataset_ids=[], retrieve_config={"k": "v"})
app_config.additional_features = mocker.MagicMock(show_retrieve_source=False)

doc_file = File(
file_type=FileType.DOCUMENT,
transfer_method=FileTransferMethod.LOCAL_FILE,
upload_file_id="doc1",
url="http://x",
)
image_file = File(
file_type=FileType.IMAGE,
transfer_method=FileTransferMethod.LOCAL_FILE,
upload_file_id="img1",
url="http://y",
)

app_generate = mocker.MagicMock(invoke_from="test", inputs={}, files=[doc_file, image_file])

runner = BaseAgentRunner(
session=sqlite_session,
tenant_id="tenant",
application_generate_entity=app_generate,
conversation=mocker.MagicMock(),
app_config=app_config,
model_config=ModelConfigWithCredentialsEntity.model_construct(),
config=_agent(),
queue_manager=mocker.Mock(spec=module.AppQueueManager),
message=_message(message_id="msg1"),
user_id="user",
model_instance=model_instance,
execution_driver=WorkflowRunAgg.run,
)

# Document file should be kept; image file should be filtered out
assert doc_file in runner.files
assert image_file not in runner.files
assert len(runner.files) == 1
13 changes: 11 additions & 2 deletions api/tests/unit_tests/core/agent/test_fc_agent_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,9 @@ def test_with_none_query(self, runner: FunctionCallAgentRunner):
assert len(result) == 1

def test_with_files_uses_image_detail_config(self, runner: FunctionCallAgentRunner, mocker: MockerFixture):
from graphon.file.enums import FileTransferMethod, FileType
from graphon.file.models import File

file_content = TextPromptMessageContent(data="file-content")
mock_to_prompt = mocker.patch(
"core.agent.fc_agent_runner.file_manager.to_prompt_message_content",
Expand All @@ -348,13 +351,19 @@ def test_with_files_uses_image_detail_config(self, runner: FunctionCallAgentRunn

image_config = MagicMock(detail=ImagePromptMessageContent.DETAIL.HIGH)
runner.application_generate_entity.file_upload_config = MagicMock(image_config=image_config)
runner.files = ["file1"]
test_file = File(
file_type=FileType.IMAGE,
transfer_method=FileTransferMethod.REMOTE_URL,
remote_url="https://example.com/file1.png",
filename="file1.png",
)
runner.files = [test_file]

result = runner._organize_user_query("query", [])

assert len(result) == 1
assert isinstance(result[0].content, list)
mock_to_prompt.assert_called_once_with("file1", image_detail_config=ImagePromptMessageContent.DETAIL.HIGH)
mock_to_prompt.assert_called_once_with(test_file, image_detail_config=ImagePromptMessageContent.DETAIL.HIGH)


# ==============================
Expand Down
Loading