From 60c3732db69de063069542d15f4b7a6d675cc234 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Fri, 12 Dec 2025 20:31:48 +0800 Subject: [PATCH 1/4] feat(chat): add streaming support with real-time tool status display - Switch from generateText to streamText with saveStreamDeltas for live updates - Add getThreadMessagesStreaming query using useUIMessages hook - Display dynamic tool invocation status in ThinkingAnimation component - Show contextual messages like "Searching X" or "Reading example.com" - Support multiple concurrent tool calls with smart text grouping - Improve web_read search tool guidance - Add next_action_required field to search results - Update agent prompt to emphasize fetch_url after search - Add fallback URLs for common queries (weather via wttr.in) --- .../[id]/chat/components/chat-interface.tsx | 212 +++++++++++++++--- services/platform/convex/_generated/api.d.ts | 4 + .../crawler/helpers/search_web.ts | 8 + .../convex_tools/crawler/helpers/types.ts | 5 + .../convex_tools/crawler/web_read_tool.ts | 19 +- .../platform/convex/lib/create_chat_agent.ts | 21 +- .../chat_agent/generate_agent_response.ts | 59 +++-- .../model/threads/get_latest_tool_message.ts | 124 ++++++++++ .../threads/get_thread_messages_streaming.ts | 52 +++++ .../platform/convex/model/threads/index.ts | 4 + services/platform/convex/threads.ts | 41 ++++ 11 files changed, 486 insertions(+), 63 deletions(-) create mode 100644 services/platform/convex/model/threads/get_latest_tool_message.ts create mode 100644 services/platform/convex/model/threads/get_thread_messages_streaming.ts diff --git a/services/platform/app/(app)/dashboard/[id]/chat/components/chat-interface.tsx b/services/platform/app/(app)/dashboard/[id]/chat/components/chat-interface.tsx index dfd1354728..51ccce1666 100644 --- a/services/platform/app/(app)/dashboard/[id]/chat/components/chat-interface.tsx +++ b/services/platform/app/(app)/dashboard/[id]/chat/components/chat-interface.tsx @@ -12,6 +12,7 @@ import { cn } from '@/lib/utils/cn'; import { uuidv7 } from 'uuidv7'; import { useThrottledScroll } from '@/hooks/use-throttled-scroll'; import { useQuery, useMutation } from 'convex/react'; +import { useUIMessages, type UIMessage } from '@convex-dev/agent/react'; import { api } from '@/convex/_generated/api'; import type { Id } from '@/convex/_generated/dataModel'; import { Button } from '@/components/ui/button'; @@ -38,33 +39,169 @@ interface ChatMessage { attachments?: FileAttachment[]; } -function ThinkingAnimation() { - const [currentStep, setCurrentStep] = useState(0); +/** + * Represents a tool invocation with its details extracted from streaming parts. + */ +interface ToolDetail { + toolName: string; + displayText: string; +} - const thinkingSteps = [ - 'Thinking', - 'Searching for related topics', - 'Compiling an answer', - ]; +/** + * Extracts a hostname from a URL for display purposes. + */ +function extractHostname(url: string): string { + try { + const parsed = new URL(url); + return parsed.hostname.replace(/^www\./, ''); + } catch { + return url; + } +} - useEffect(() => { - let interval: NodeJS.Timeout; +/** + * Truncates a string to a maximum length, adding ellipsis if needed. + */ +function truncate(str: string, maxLength: number): string { + if (str.length <= maxLength) return str; + return str.slice(0, maxLength - 1) + '…'; +} - if (currentStep < thinkingSteps.length - 1) { - interval = setInterval(() => { - setCurrentStep((prev) => prev + 1); - }, 2500); +/** + * Formats a tool invocation into a human-readable display text with context. + * Extracts relevant details from the tool's input arguments. + */ +function formatToolDetail( + toolName: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + input?: Record, +): ToolDetail { + // Handle web_read tool with operation-specific display + if (toolName === 'web_read' && input) { + if (input.operation === 'search' && input.query) { + return { + toolName, + displayText: `Searching "${truncate(input.query, 30)}"`, + }; + } + if (input.operation === 'fetch_url' && input.url) { + return { + toolName, + displayText: `Reading ${extractHostname(input.url)}`, + }; } + } - return () => { - if (interval) clearInterval(interval); + // Handle rag_search with query + if (toolName === 'rag_search' && input?.query) { + return { + toolName, + displayText: `Searching knowledge base for "${truncate(input.query, 25)}"`, }; - }, [currentStep, thinkingSteps.length]); + } + + // Default fallback display names for tools without detailed input + const defaultDisplayNames: Record = { + customer_read: 'Reading customer data', + product_read: 'Reading product catalog', + rag_search: 'Searching knowledge base', + rag_write: 'Updating knowledge base', + web_read: 'Fetching web content', + pdf: 'Processing PDF', + image: 'Analyzing image', + pptx: 'Processing presentation', + docx: 'Processing document', + resource_check: 'Checking resources', + workflow_read: 'Reading workflow', + update_workflow_step: 'Updating workflow step', + generate_workflow_from_description: 'Generating workflow', + save_workflow_definition: 'Saving workflow', + validate_workflow_definition: 'Validating workflow', + generate_excel: 'Generating Excel file', + context_search: 'Searching for related topics', + }; + + const displayText = + defaultDisplayNames[toolName] || + toolName + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + return { toolName, displayText }; +} + +interface ThinkingAnimationProps { + threadId?: string; + streamingMessage?: UIMessage; +} + +function ThinkingAnimation({ + threadId, + streamingMessage, +}: ThinkingAnimationProps) { + // Extract tool details from streaming message parts + // Parts with tool info have type format 'tool-{toolName}' (e.g., 'tool-web_read') + // and may contain 'input' with the tool's arguments + const toolDetails: ToolDetail[] = []; + + if (streamingMessage?.parts) { + for (const part of streamingMessage.parts) { + // The type format is 'tool-{toolName}', e.g., 'tool-web_read', 'tool-rag_search' + if (part.type.startsWith('tool-')) { + // Extract tool name from type (remove 'tool-' prefix) + const toolName = part.type.slice(5); // 'tool-'.length === 5 + if (toolName && toolName !== 'invocation') { + // Extract input if available (cast part to access input property) + const toolPart = part as { input?: Record }; + const detail = formatToolDetail(toolName, toolPart.input); + toolDetails.push(detail); + } + } + } + } + + // Determine what text to display - show tool details or default "Thinking" + let displayText = 'Thinking'; + + if (toolDetails.length === 1) { + // Single tool - show its detailed display text + displayText = toolDetails[0].displayText; + } else if (toolDetails.length > 1) { + // Multiple tools - deduplicate by display text and join them + const uniqueDisplayTexts = [...new Set(toolDetails.map((d) => d.displayText))]; + + // Check if all display texts start with the same verb (e.g., "Searching", "Reading") + // to create a more natural grouped message + const searchPrefix = 'Searching "'; + const allSearches = uniqueDisplayTexts.every((t) => t.startsWith(searchPrefix)); + + if (allSearches && uniqueDisplayTexts.length > 1) { + // Extract just the query parts (remove "Searching " prefix and closing quote) + const queries = uniqueDisplayTexts.map((t) => + t.slice(searchPrefix.length - 1, t.endsWith('"') ? t.length : t.length) + ); + if (queries.length <= 2) { + displayText = `Searching ${queries.join(' and ')}`; + } else { + displayText = `Searching ${queries[0]}, ${queries[1]} and ${queries.length - 2} more`; + } + } else if (uniqueDisplayTexts.length <= 2) { + displayText = uniqueDisplayTexts.join(' and '); + } else { + // For 3+ different tool calls, show first two and count + displayText = `${uniqueDisplayTexts[0]}, ${uniqueDisplayTexts[1]} and ${uniqueDisplayTexts.length - 2} more`; + } + } + + // Use tool details as key for animation + const animationKey = + toolDetails.length > 0 ? toolDetails.map((d) => d.displayText).join('-') : 'thinking'; return (
- {thinkingSteps[currentStep]} + {displayText}
@@ -123,18 +260,26 @@ export default function ChatInterface({ // Optimistic user message content const userDraftMessage = optimisticMessage?.content || ''; - // Fetch thread messages - const rawThreadMessages = useQuery( - api.threads.getThreadMessages, + // Fetch thread messages with streaming support + const { results: uiMessages } = useUIMessages( + api.threads.getThreadMessagesStreaming, threadId ? { threadId } : 'skip', + { initialNumItems: 50, stream: true }, ); - const threadMessages: ChatMessage[] = (rawThreadMessages?.messages || []).map( - (m) => ({ - id: m._id, - content: m.content, - role: m.role, + + // Convert UIMessage to ChatMessage format for compatibility + const threadMessages: ChatMessage[] = (uiMessages || []) + .filter((m) => m.role === 'user' || m.role === 'assistant') + .map((m) => ({ + id: m.key, + content: m.text, + role: m.role as 'user' | 'assistant', timestamp: m._creationTime, - }), + })); + + // Find if there's currently a streaming assistant message + const streamingMessage = uiMessages?.find( + (m) => m.role === 'assistant' && m.status === 'streaming', ); // Query for active runId from thread (for recovery on page refresh) @@ -201,7 +346,7 @@ export default function ChatInterface({ useEffect(() => { if ( optimisticMessage?.content && - rawThreadMessages !== undefined && + uiMessages !== undefined && threadMessages?.some((m) => { if (m.role !== 'user') return false; // Check for exact match OR if the message starts with the optimistic content @@ -214,7 +359,7 @@ export default function ChatInterface({ ) { setOptimisticMessage(null); } - }, [rawThreadMessages, threadMessages, optimisticMessage?.content, setOptimisticMessage]); + }, [uiMessages, threadMessages, optimisticMessage?.content, setOptimisticMessage]); // Scroll handling const containerRef = useRef(null); @@ -375,7 +520,12 @@ export default function ChatInterface({ }} /> )} - {isLoading && } + {isLoading && !streamingMessage?.text && ( + + )}
)}
diff --git a/services/platform/convex/_generated/api.d.ts b/services/platform/convex/_generated/api.d.ts index df264ff264..3cd262627e 100644 --- a/services/platform/convex/_generated/api.d.ts +++ b/services/platform/convex/_generated/api.d.ts @@ -288,7 +288,9 @@ import type * as model_threads_create_chat_thread from "../model/threads/create_ import type * as model_threads_delete_chat_thread from "../model/threads/delete_chat_thread.js"; import type * as model_threads_get_active_run_id from "../model/threads/get_active_run_id.js"; import type * as model_threads_get_latest_thread_with_message_count from "../model/threads/get_latest_thread_with_message_count.js"; +import type * as model_threads_get_latest_tool_message from "../model/threads/get_latest_tool_message.js"; import type * as model_threads_get_thread_messages from "../model/threads/get_thread_messages.js"; +import type * as model_threads_get_thread_messages_streaming from "../model/threads/get_thread_messages_streaming.js"; import type * as model_threads_index from "../model/threads/index.js"; import type * as model_threads_list_threads from "../model/threads/list_threads.js"; import type * as model_threads_update_chat_thread from "../model/threads/update_chat_thread.js"; @@ -888,7 +890,9 @@ declare const fullApi: ApiFromModules<{ "model/threads/delete_chat_thread": typeof model_threads_delete_chat_thread; "model/threads/get_active_run_id": typeof model_threads_get_active_run_id; "model/threads/get_latest_thread_with_message_count": typeof model_threads_get_latest_thread_with_message_count; + "model/threads/get_latest_tool_message": typeof model_threads_get_latest_tool_message; "model/threads/get_thread_messages": typeof model_threads_get_thread_messages; + "model/threads/get_thread_messages_streaming": typeof model_threads_get_thread_messages_streaming; "model/threads/index": typeof model_threads_index; "model/threads/list_threads": typeof model_threads_list_threads; "model/threads/update_chat_thread": typeof model_threads_update_chat_thread; diff --git a/services/platform/convex/agent_tools/convex_tools/crawler/helpers/search_web.ts b/services/platform/convex/agent_tools/convex_tools/crawler/helpers/search_web.ts index 652912f3bd..1fdda60744 100644 --- a/services/platform/convex/agent_tools/convex_tools/crawler/helpers/search_web.ts +++ b/services/platform/convex/agent_tools/convex_tools/crawler/helpers/search_web.ts @@ -69,6 +69,13 @@ export async function searchWeb( has_more: hasMore, }); + // Build the next action instruction based on results + const hasResults = pageData.items.length > 0; + const firstUrl = hasResults ? pageData.items[0].link : null; + const nextActionRequired = hasResults + ? `IMPORTANT: These are only search result snippets, NOT the actual page content. To get real data (weather, prices, news, etc.), you MUST now call web_read with { operation: "fetch_url", url: "${firstUrl}" } or another relevant URL from the results above.` + : 'No results found. Try a different search query.'; + return { operation: 'search', success: true, @@ -79,6 +86,7 @@ export async function searchWeb( has_more: hasMore, next_start_index: hasMore ? pageNumber + 1 : null, suggestions: pageData.suggestions, + next_action_required: nextActionRequired, }; } catch (error) { console.error('[tool:web_read:search] error', { diff --git a/services/platform/convex/agent_tools/convex_tools/crawler/helpers/types.ts b/services/platform/convex/agent_tools/convex_tools/crawler/helpers/types.ts index cb53c156e6..ff780db83f 100644 --- a/services/platform/convex/agent_tools/convex_tools/crawler/helpers/types.ts +++ b/services/platform/convex/agent_tools/convex_tools/crawler/helpers/types.ts @@ -58,6 +58,11 @@ export type WebReadSearchResult = { has_more: boolean; next_start_index: number | null; suggestions?: string[]; + /** + * Explicit instruction for the AI on what to do next. + * This helps ensure the AI calls fetch_url after search. + */ + next_action_required: string; }; // ============================================================================= diff --git a/services/platform/convex/agent_tools/convex_tools/crawler/web_read_tool.ts b/services/platform/convex/agent_tools/convex_tools/crawler/web_read_tool.ts index 2b487568bd..635c118c03 100644 --- a/services/platform/convex/agent_tools/convex_tools/crawler/web_read_tool.ts +++ b/services/platform/convex/agent_tools/convex_tools/crawler/web_read_tool.ts @@ -91,18 +91,27 @@ OPERATIONS: 1. fetch_url - Fetch and extract content from a web URL (HTML pages only) Use when a user provides a URL and wants to know what's on the page. - Returns: page title, main content text, word count. + Returns: page title, main content text, word count, structured_data. LIMITATIONS: - CANNOT read PDF, Excel, Word, or other binary files - For documents in conversation thread, use "context_search" tool - For knowledge base documents, use "rag_search" tool 2. search - Search the web using SearXNG meta search engine - Aggregates results from multiple search engines (Brave, Google, Bing, DuckDuckGo). - Returns URLs with titles and snippets. - Recommended workflow: use "search" to find candidate links, then call "fetch_url" on the most relevant URLs before answering. + Aggregates results from multiple search engines (Brave, Google, Bing, DuckDuckGo). + Returns URLs with titles and SHORT SNIPPETS ONLY - NOT the actual page content! -EXAMPLE USAGE: +CRITICAL: For real-world data (weather, prices, news, etc.), you MUST: +1. First call "search" to find relevant URLs +2. Then call "fetch_url" on the best URL(s) to get the ACTUAL content +Without step 2, you only have brief snippets and cannot answer accurately! + +EXAMPLE WORKFLOW for weather/prices/news: +1. { operation: "search", query: "weather in Zurich today" } +2. { operation: "fetch_url", url: "https://weather.com/..." } (from search results) +3. Read the fetched content and answer the user + +OTHER EXAMPLES: • Fetch a URL: { operation: "fetch_url", url: "https://example.com/article" } • Simple search: { operation: "search", query: "next.js app router" } • Site-specific: { operation: "search", query: "hooks tutorial", site: "react.dev" }`, diff --git a/services/platform/convex/lib/create_chat_agent.ts b/services/platform/convex/lib/create_chat_agent.ts index f021742af5..907702be11 100644 --- a/services/platform/convex/lib/create_chat_agent.ts +++ b/services/platform/convex/lib/create_chat_agent.ts @@ -109,18 +109,27 @@ DO NOT ask the user "where is this data stored?" or "which field contains this?" - It is OK to call rag_search multiple times with different queries for complex tasks. - If rag_search returns no useful results, say so clearly and use other tools or ask for clarification. -3) PUBLIC / REAL-WORLD INFORMATION → web_read.search -If the question is about public, real‑world, or time‑sensitive information, you MUST call the web_read tool with operation = "search". Examples: +3) PUBLIC / REAL-WORLD INFORMATION → web_read.search + fetch_url +If the question is about public, real‑world, or time‑sensitive information, use web_read. Examples: - Weather today or in the future - Currency exchange rates or interest rates - Stock prices, market data, macro‑economics - Current product features for third‑party tools or frameworks - News, public documentation, or other internet content -Usage pattern: -- First: rag_search(query = full user question) -- Then: web_read with { operation: "search", query: "..." } for external/public info -- Optionally: call web_read again with { operation: "fetch_url", url: "..." } on the most relevant link(s) when you need to read page content in detail. +IMPORTANT: The "search" operation only returns URLs with brief snippets - it does NOT return the actual page content! +To get real data (weather, prices, etc.), you MUST follow up with "fetch_url" on a relevant URL. + +Usage pattern (REQUIRED for real-world data): +1. web_read with { operation: "search", query: "..." } to find relevant URLs +2. web_read with { operation: "fetch_url", url: "..." } on the most relevant URL(s) to get actual content +3. Extract and present the data from the fetched content + +FALLBACK FOR COMMON QUERIES: If search returns no relevant results or only irrelevant sites after 1-2 attempts, you MAY directly fetch well-known authoritative URLs for common data types: +- WEATHER: Use https://wttr.in/{city}?format=4 (e.g., https://wttr.in/Zurich?format=4) - this is a simple text-based weather service +- You can also try: https://www.timeanddate.com/weather/switzerland/zurich + +Do NOT keep searching endlessly if results are poor. After 1-2 failed searches, use the fallback URLs above. 4) WEB LINKS → web_read.fetch_url When the user provides a direct http/https URL and asks what is on the page or to summarize/analyze it: diff --git a/services/platform/convex/model/chat_agent/generate_agent_response.ts b/services/platform/convex/model/chat_agent/generate_agent_response.ts index ba5fbda57f..fa1e02447e 100644 --- a/services/platform/convex/model/chat_agent/generate_agent_response.ts +++ b/services/platform/convex/model/chat_agent/generate_agent_response.ts @@ -300,28 +300,45 @@ export async function generateAgentResponse( // Determine if we need special handling for attachments const hasAttachmentContent = promptContent !== undefined; - const result: { text?: string; steps?: unknown[]; usage?: Usage } = - await agent.generateText( - contextWithOrg, - { threadId }, - { - promptMessageId, - abortSignal: abortController.signal, - messages: contextMessages, - // If we have attachments, use prompt to override the stored message content - // This allows multi-modal content without storing the large base64 data - ...(promptContent ? { prompt: promptContent } : {}), - }, - { - contextOptions: { - recentMessages: 20, - excludeToolMessages: true, - searchOtherThreads: false, - }, - // User message was already saved in the mutation with promptMessageId. - // The library only saves the assistant response when promptMessageId is provided. + // Use streamText with saveStreamDeltas for real-time UI updates + // This allows the UI to show tool calls as they happen + const streamResult = await agent.streamText( + contextWithOrg, + { threadId }, + { + promptMessageId, + abortSignal: abortController.signal, + messages: contextMessages, + // If we have attachments, use prompt to override the stored message content + // This allows multi-modal content without storing the large base64 data + ...(promptContent ? { prompt: promptContent } : {}), + }, + { + contextOptions: { + recentMessages: 20, + excludeToolMessages: true, + searchOtherThreads: false, }, - ); + // Save stream deltas so UI can show real-time progress + saveStreamDeltas: true, + // User message was already saved in the mutation with promptMessageId. + // The library only saves the assistant response when promptMessageId is provided. + }, + ); + + // Consume the stream to completion and get the final result + // We need to iterate through the stream for it to complete + let finalText = ''; + for await (const textPart of streamResult.textStream) { + finalText += textPart; + } + + // Get the final result after stream completes + const result: { text?: string; steps?: unknown[]; usage?: Usage } = { + text: finalText, + steps: await streamResult.steps, + usage: await streamResult.usage, + }; clearTimeout(timeoutId); const elapsedMs = Date.now() - startTime; diff --git a/services/platform/convex/model/threads/get_latest_tool_message.ts b/services/platform/convex/model/threads/get_latest_tool_message.ts new file mode 100644 index 0000000000..5bef309bf0 --- /dev/null +++ b/services/platform/convex/model/threads/get_latest_tool_message.ts @@ -0,0 +1,124 @@ +/** + * Get the latest tool message for a thread. + * + * Used to display dynamic loading status in the UI when the agent is + * running tools. Returns the most recent tool-call or tool-result message. + * Supports multiple tool calls in a single message. + */ + +import { QueryCtx } from '../../_generated/server'; +import { components } from '../../_generated/api'; +import { listMessages, type MessageDoc } from '@convex-dev/agent'; + +export interface LatestToolMessage { + toolNames: string[]; + status: 'calling' | 'completed' | null; + timestamp: number | null; +} + +/** + * Extracts all tool names from a message's content. + * Content can be a string or an array of content parts. + * Returns unique tool names to avoid duplicates. + */ +function extractToolNames(message: MessageDoc): string[] { + const msg = message.message; + if (!msg) return []; + + const content = msg.content; + const toolNames = new Set(); + + // If content is an array, look for tool-call or tool-result parts + if (Array.isArray(content)) { + for (const part of content) { + if (typeof part === 'object' && part !== null) { + const p = part as Record; + if ( + (p.type === 'tool-call' || p.type === 'tool-result') && + typeof p.toolName === 'string' + ) { + toolNames.add(p.toolName); + } + } + } + } + + return Array.from(toolNames); +} + +/** + * Determines if the message represents a completed tool call. + */ +function isToolCompleted(message: MessageDoc): boolean { + const msg = message.message; + if (!msg) return false; + + // Tool role messages are results + if (msg.role === 'tool') return true; + + const content = msg.content; + if (Array.isArray(content)) { + for (const part of content) { + if (typeof part === 'object' && part !== null) { + const p = part as Record; + if (p.type === 'tool-result') { + return true; + } + } + } + } + + return false; +} + +export async function getLatestToolMessage( + ctx: QueryCtx, + threadId: string, +): Promise { + // Get the most recent messages (not excluding tool messages) + const result = await listMessages(ctx, components.agent, { + threadId, + paginationOpts: { cursor: null, numItems: 10 }, + // Do NOT exclude tool messages - we want them + }); + + // Find the most recent message that is tool-related + // Messages are returned in descending order (newest first) + for (const doc of result.page) { + const msg = doc.message; + if (!msg) continue; + + // Check if it's a tool role message + if (msg.role === 'tool') { + const toolNames = extractToolNames(doc); + return { + toolNames, + status: 'completed', + timestamp: doc._creationTime, + }; + } + + // Check if it's an assistant message with tool-call content + if (msg.role === 'assistant') { + const toolNames = extractToolNames(doc); + if (toolNames.length > 0) { + // Check if there's a corresponding tool result + const hasResult = result.page.some( + (d) => d._creationTime > doc._creationTime && isToolCompleted(d), + ); + return { + toolNames, + status: hasResult ? 'completed' : 'calling', + timestamp: doc._creationTime, + }; + } + } + } + + return { + toolNames: [], + status: null, + timestamp: null, + }; +} + diff --git a/services/platform/convex/model/threads/get_thread_messages_streaming.ts b/services/platform/convex/model/threads/get_thread_messages_streaming.ts new file mode 100644 index 0000000000..282da0ba95 --- /dev/null +++ b/services/platform/convex/model/threads/get_thread_messages_streaming.ts @@ -0,0 +1,52 @@ +/** + * Get messages for a thread with streaming support. + * + * Uses listUIMessages and syncStreams to support real-time streaming updates. + * This enables the UI to show tool calls and text as they happen. + */ + +import { QueryCtx } from '../../_generated/server'; +import { components } from '../../_generated/api'; +import { + listUIMessages, + syncStreams, + type StreamArgs, + type UIMessage, +} from '@convex-dev/agent'; +import type { PaginationOptions } from 'convex/server'; + +export interface StreamingMessagesResult { + page: UIMessage[]; + isDone: boolean; + continueCursor: string; + streams: Awaited>; +} + +export async function getThreadMessagesStreaming( + ctx: QueryCtx, + args: { + threadId: string; + paginationOpts: PaginationOptions; + streamArgs: StreamArgs | undefined; + }, +): Promise { + // Fetch paginated UI messages + const paginated = await listUIMessages(ctx, components.agent, { + threadId: args.threadId, + paginationOpts: args.paginationOpts, + }); + + // Fetch streaming deltas for real-time updates + // Pass args directly as syncStreams expects { threadId, streamArgs, ... } + const streams = await syncStreams(ctx, components.agent, { + ...args, + // Include all statuses to avoid UI flashes when messages transition + includeStatuses: ['streaming', 'aborted', 'finished'], + }); + + return { + ...paginated, + streams, + }; +} + diff --git a/services/platform/convex/model/threads/index.ts b/services/platform/convex/model/threads/index.ts index 2ec20f4fdb..72fc3103e1 100644 --- a/services/platform/convex/model/threads/index.ts +++ b/services/platform/convex/model/threads/index.ts @@ -9,6 +9,10 @@ export { getThreadMessages } from './get_thread_messages'; export { listThreads } from './list_threads'; export { updateChatThread } from './update_chat_thread'; export { getLatestThreadWithMessageCount } from './get_latest_thread_with_message_count'; +export { getLatestToolMessage } from './get_latest_tool_message'; +export { getThreadMessagesStreaming } from './get_thread_messages_streaming'; export type { ThreadMessage } from './get_thread_messages'; export type { Thread } from './list_threads'; +export type { LatestToolMessage } from './get_latest_tool_message'; +export type { StreamingMessagesResult } from './get_thread_messages_streaming'; diff --git a/services/platform/convex/threads.ts b/services/platform/convex/threads.ts index 32eefe8578..bd817e62d6 100644 --- a/services/platform/convex/threads.ts +++ b/services/platform/convex/threads.ts @@ -5,8 +5,10 @@ * using the Convex Agent Component. */ +import { paginationOptsValidator } from 'convex/server'; import { query, mutation } from './_generated/server'; import { v } from 'convex/values'; +import { vStreamArgs } from '@convex-dev/agent'; import { validateOrganizationAccess, getAuthenticatedUser } from './lib/rls'; import * as ThreadsModel from './model/threads'; @@ -157,3 +159,42 @@ export const clearActiveRunId = mutation({ return null; }, }); + +/** + * Get the latest tool message for a thread. + * Used to display dynamic loading status in the UI when the agent is running tools. + * Supports multiple tool calls in a single message. + */ +export const getLatestToolMessage = query({ + args: { + threadId: v.string(), + }, + returns: v.object({ + toolNames: v.array(v.string()), + status: v.union(v.literal('calling'), v.literal('completed'), v.null()), + timestamp: v.union(v.number(), v.null()), + }), + handler: async (ctx, args) => { + return await ThreadsModel.getLatestToolMessage(ctx, args.threadId); + }, +}); + +/** + * Get thread messages with streaming support. + * Used by useUIMessages hook with stream: true for real-time updates. + * Returns paginated messages plus streaming deltas for active streams. + */ +export const getThreadMessagesStreaming = query({ + args: { + threadId: v.string(), + paginationOpts: paginationOptsValidator, + streamArgs: vStreamArgs, + }, + handler: async (ctx, args) => { + return await ThreadsModel.getThreadMessagesStreaming(ctx, { + threadId: args.threadId, + paginationOpts: args.paginationOpts, + streamArgs: args.streamArgs, + }); + }, +}); From 75b6d86630bd0001e81910cda68a40558a2fab25 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Fri, 12 Dec 2025 21:33:10 +0800 Subject: [PATCH 2/4] feat: improve image generation quality with high-DPI support - Add scale parameter for device scale factor (default 2.0 for Retina) - Increase default quality from 90 to 100 for JPEG images - Increase default viewport width from 800 to 1200px - Update image tool description to warn against setting explicit dimensions - Propagate scale option through crawler service, API endpoints, and Convex types --- services/crawler/app/converter_service.py | 18 ++++++++++++++---- services/crawler/app/main.py | 3 +++ services/crawler/app/models.py | 5 +++-- .../convex_tools/files/image_tool.ts | 5 +++-- services/platform/convex/documents.ts | 1 + .../documents/generate_document_helpers.ts | 5 +++-- .../platform/convex/model/documents/types.ts | 1 + 7 files changed, 28 insertions(+), 10 deletions(-) diff --git a/services/crawler/app/converter_service.py b/services/crawler/app/converter_service.py index 0ea63d3540..0d6ebf657c 100644 --- a/services/crawler/app/converter_service.py +++ b/services/crawler/app/converter_service.py @@ -233,15 +233,16 @@ async def html_to_image( html: str, wrap_in_template: bool = True, image_type: Literal["png", "jpeg"] = "png", - quality: int = 90, + quality: int = 100, full_page: bool = True, - width: int = 800, + width: int = 1200, extra_css: Optional[str] = None, + scale: float = 2.0, ) -> bytes: """Convert HTML to image (PNG or JPEG).""" page = await self._get_page() try: - # Set viewport width + # Set viewport with device scale factor for high-quality rendering await page.set_viewport_size({"width": width, "height": 600}) # Wrap in template if requested @@ -264,10 +265,14 @@ async def html_to_image( screenshot_options = { "type": image_type, "full_page": full_page, + "scale": "device" if scale > 1.0 else "css", } if image_type == "jpeg": screenshot_options["quality"] = quality + # Set device scale factor for high-quality output + await page.evaluate(f"() => {{ window.devicePixelRatio = {scale}; }}") + image_bytes = await page.screenshot(**screenshot_options) return image_bytes finally: @@ -325,10 +330,11 @@ async def url_to_image( url: str, wait_until: WaitUntilType = "networkidle", image_type: Literal["png", "jpeg"] = "png", - quality: int = 90, + quality: int = 100, full_page: bool = True, width: int = 1280, height: int = 800, + scale: float = 2.0, ) -> bytes: """Capture a URL as image (screenshot).""" page = await self._get_page() @@ -339,10 +345,14 @@ async def url_to_image( screenshot_options = { "type": image_type, "full_page": full_page, + "scale": "device" if scale > 1.0 else "css", } if image_type == "jpeg": screenshot_options["quality"] = quality + # Set device scale factor for high-quality output + await page.evaluate(f"() => {{ window.devicePixelRatio = {scale}; }}") + image_bytes = await page.screenshot(**screenshot_options) return image_bytes finally: diff --git a/services/crawler/app/main.py b/services/crawler/app/main.py index 14fe861a22..a2b7fccb3a 100644 --- a/services/crawler/app/main.py +++ b/services/crawler/app/main.py @@ -407,6 +407,7 @@ async def convert_markdown_to_image(request: MarkdownToImageRequest): full_page=request.options.full_page, width=request.options.width, extra_css=request.extra_css, + scale=request.options.scale, ) media_type = "image/png" if request.options.image_type == "png" else "image/jpeg" @@ -493,6 +494,7 @@ async def convert_html_to_image(request: HtmlToImageRequest): full_page=request.options.full_page, width=request.options.width, extra_css=request.extra_css, + scale=request.options.scale, ) media_type = "image/png" if request.options.image_type == "png" else "image/jpeg" @@ -578,6 +580,7 @@ async def convert_url_to_image(request: UrlToImageRequest): full_page=request.options.full_page, width=request.options.width, height=request.height, + scale=request.options.scale, ) media_type = "image/png" if request.options.image_type == "png" else "image/jpeg" diff --git a/services/crawler/app/models.py b/services/crawler/app/models.py index 91377ac2d3..96ce124c00 100644 --- a/services/crawler/app/models.py +++ b/services/crawler/app/models.py @@ -115,9 +115,10 @@ class ImageOptions(BaseModel): """Options for image generation.""" image_type: str = Field("png", description="Image type (png or jpeg)") - quality: int = Field(90, description="JPEG quality (1-100)", ge=1, le=100) + quality: int = Field(100, description="JPEG quality (1-100)", ge=1, le=100) full_page: bool = Field(True, description="Capture full page or viewport only") - width: int = Field(800, description="Viewport width", ge=100, le=4096) + width: int = Field(1200, description="Viewport width", ge=100, le=4096) + scale: float = Field(2.0, description="Device scale factor for high-quality images (2.0 = Retina)", ge=1.0, le=4.0) class MarkdownToPdfRequest(BaseModel): diff --git a/services/platform/convex/agent_tools/convex_tools/files/image_tool.ts b/services/platform/convex/agent_tools/convex_tools/files/image_tool.ts index c2885ae289..9b7d8cc1bf 100644 --- a/services/platform/convex/agent_tools/convex_tools/files/image_tool.ts +++ b/services/platform/convex/agent_tools/convex_tools/files/image_tool.ts @@ -41,7 +41,7 @@ Parameters: - fileName: Base name for the generated image (without extension; .png will be added automatically) - sourceType: One of "markdown", "html", or "url" describing the content type - content: The actual Markdown/HTML text or URL to capture -- imageOptions: Advanced image options (width, height, fullPage) +- imageOptions: Advanced image options (width, height, fullPage). IMPORTANT: Do NOT set width or height unless the user explicitly requests specific dimensions. Setting dimensions can cause content truncation. Let the system auto-size based on content by default. - urlOptions: Advanced options for URL capture (navigation, timeout, etc.) - extraCss: Additional CSS to inject when rendering HTML/Markdown - wrapInTemplate: Whether to wrap raw content in a standard HTML template before rendering @@ -75,9 +75,10 @@ CRITICAL RULES FOR RESPONSE: width: z.number().optional(), height: z.number().optional(), fullPage: z.boolean().optional(), + scale: z.number().min(1).max(4).optional(), }) .optional() - .describe('Advanced image options for image output'), + .describe('Advanced image options. ONLY set width/height if user explicitly requests specific dimensions - otherwise omit to auto-size based on content. Scale defaults to 2.0 for high-quality Retina output'), urlOptions: z .object({ waitUntil: z diff --git a/services/platform/convex/documents.ts b/services/platform/convex/documents.ts index b1a0f7c5cf..85f75ff143 100644 --- a/services/platform/convex/documents.ts +++ b/services/platform/convex/documents.ts @@ -216,6 +216,7 @@ export const generateDocumentInternal = internalAction({ fullPage: v.optional(v.boolean()), width: v.optional(v.number()), height: v.optional(v.number()), + scale: v.optional(v.number()), }), ), urlOptions: v.optional( diff --git a/services/platform/convex/model/documents/generate_document_helpers.ts b/services/platform/convex/model/documents/generate_document_helpers.ts index 8e3de2a84d..bec49368da 100644 --- a/services/platform/convex/model/documents/generate_document_helpers.ts +++ b/services/platform/convex/model/documents/generate_document_helpers.ts @@ -110,9 +110,10 @@ export function buildRequestBody( } else { body.options = { image_type: imageOptions?.imageType ?? 'png', - quality: imageOptions?.quality ?? 90, + quality: imageOptions?.quality ?? 100, full_page: imageOptions?.fullPage ?? true, - width: imageOptions?.width ?? 800, + width: imageOptions?.width ?? 1200, + scale: imageOptions?.scale ?? 2.0, }; } diff --git a/services/platform/convex/model/documents/types.ts b/services/platform/convex/model/documents/types.ts index 067f701bff..70ded1765a 100644 --- a/services/platform/convex/model/documents/types.ts +++ b/services/platform/convex/model/documents/types.ts @@ -130,6 +130,7 @@ export interface GenerateDocumentImageOptions { fullPage?: boolean; width?: number; height?: number; // Only for URL screenshots + scale?: number; // Device scale factor for high-quality images (1.0-4.0, default 2.0) } /** Valid Playwright wait_until values */ From 700d4bb9d7d9723cc8ac99fb66bfaf8a8bee2515 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Fri, 12 Dec 2025 22:20:32 +0800 Subject: [PATCH 3/4] refactor(chat-agent): remove context overflow retry, improve error diagnostics - Delete context_overflow_retry.ts helper module - Instead of retrying on empty responses, throw detailed error with: - Model info, token counts, step-by-step execution details - Finish reasons, tool call counts, warnings, and provider errors - Add finishReason and warnings to stream result type - Simplify generate_agent_response by removing retry logic --- services/platform/convex/_generated/api.d.ts | 2 - .../chat_agent/context_overflow_retry.ts | 126 ------------------ .../chat_agent/generate_agent_response.ts | 76 +++++++++-- 3 files changed, 64 insertions(+), 140 deletions(-) delete mode 100644 services/platform/convex/model/chat_agent/context_overflow_retry.ts diff --git a/services/platform/convex/_generated/api.d.ts b/services/platform/convex/_generated/api.d.ts index 3cd262627e..074fb0adca 100644 --- a/services/platform/convex/_generated/api.d.ts +++ b/services/platform/convex/_generated/api.d.ts @@ -132,7 +132,6 @@ import type * as model_chat_agent_auto_summarize_if_needed from "../model/chat_a import type * as model_chat_agent_cancel_chat from "../model/chat_agent/cancel_chat.js"; import type * as model_chat_agent_chat_with_agent from "../model/chat_agent/chat_with_agent.js"; import type * as model_chat_agent_chat_with_agent_status from "../model/chat_agent/chat_with_agent_status.js"; -import type * as model_chat_agent_context_overflow_retry from "../model/chat_agent/context_overflow_retry.js"; import type * as model_chat_agent_generate_agent_response from "../model/chat_agent/generate_agent_response.js"; import type * as model_chat_agent_index from "../model/chat_agent/index.js"; import type * as model_chat_agent_message_deduplication from "../model/chat_agent/message_deduplication.js"; @@ -734,7 +733,6 @@ declare const fullApi: ApiFromModules<{ "model/chat_agent/cancel_chat": typeof model_chat_agent_cancel_chat; "model/chat_agent/chat_with_agent": typeof model_chat_agent_chat_with_agent; "model/chat_agent/chat_with_agent_status": typeof model_chat_agent_chat_with_agent_status; - "model/chat_agent/context_overflow_retry": typeof model_chat_agent_context_overflow_retry; "model/chat_agent/generate_agent_response": typeof model_chat_agent_generate_agent_response; "model/chat_agent/index": typeof model_chat_agent_index; "model/chat_agent/message_deduplication": typeof model_chat_agent_message_deduplication; diff --git a/services/platform/convex/model/chat_agent/context_overflow_retry.ts b/services/platform/convex/model/chat_agent/context_overflow_retry.ts deleted file mode 100644 index 0d9fa68472..0000000000 --- a/services/platform/convex/model/chat_agent/context_overflow_retry.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Helper for handling context overflow when the agent completes - * without generating any text. This encapsulates the "no tools" - * retry path so the main action handler stays focused. - */ - -import type { ActionCtx } from '../../_generated/server'; -import { components, internal } from '../../_generated/api'; -import { listMessages, saveMessage } from '@convex-dev/agent'; -import { createChatAgent } from '../../lib/create_chat_agent'; - -import { createDebugLog } from '../../lib/debug_log'; - -const debugLog = createDebugLog('DEBUG_CHAT_AGENT', '[ChatAgent]'); - -const MIN_SUMMARY_LENGTH_FOR_RETRY = 50000; // 50k characters -const MAX_NO_TOOL_RETRIES = 5; - -type Usage = { - inputTokens?: number; - outputTokens?: number; - totalTokens?: number; - reasoningTokens?: number; - cachedInputTokens?: number; -}; - -export interface ContextOverflowRetryParams { - threadId: string; - promptMessageId: string; - toolCallCount: number; - usage?: Usage; - /** - * Context object extended with organizationId/threadId/variables, - * passed through to Agent.generateText. - */ - contextWithOrg: unknown; -} - -export async function handleContextOverflowNoToolRetry( - ctx: ActionCtx, - params: ContextOverflowRetryParams, -): Promise { - const { threadId, promptMessageId, toolCallCount, usage, contextWithOrg } = - params; - - const tokenInfo = usage - ? `, tokens: ${usage.inputTokens ?? 0} input / ${ - usage.outputTokens ?? 0 - } output` - : ''; - - debugLog('Forcing summarization before retry check...'); - - const forcedSummaryResult: { summarized: boolean; existingSummary?: string } = - await ctx.runAction(internal.chat_agent.autoSummarizeIfNeeded, { - threadId, - }); - - const fallbackSummary = forcedSummaryResult.existingSummary || ''; - const summaryLength = fallbackSummary.length; - - if (!fallbackSummary || summaryLength <= MIN_SUMMARY_LENGTH_FOR_RETRY) { - debugLog( - `Context overflow detected (${toolCallCount} tool calls${tokenInfo}), but summary not sufficient for retry`, - { summaryLength, minRequired: MIN_SUMMARY_LENGTH_FOR_RETRY }, - ); - throw new Error( - 'Agent completed without generating a response message. Summary not sufficient for no-tool retry.', - ); - } - - debugLog( - `Context overflow detected (${toolCallCount} tool calls${tokenInfo}). Summary is ${summaryLength} chars. Retrying without tools...`, - ); - - // Get the user's original message - const userMessages = await listMessages(ctx, components.agent, { - threadId, - paginationOpts: { cursor: null, numItems: 20 }, - excludeToolMessages: true, - }); - const userMessage = userMessages.page.find((m) => m._id === promptMessageId); - const userPrompt = - typeof userMessage?.message?.content === 'string' - ? userMessage.message.content - : 'Please provide a response based on our conversation.'; - - // Retry up to MAX_NO_TOOL_RETRIES times without tools - for (let attempt = 1; attempt <= MAX_NO_TOOL_RETRIES; attempt++) { - debugLog(`No-tool retry attempt ${attempt}/${MAX_NO_TOOL_RETRIES}`); - - const noToolAgent = await createChatAgent({ - withTools: false, - maxSteps: 1, - }); - - const minimalPrompt = `## Conversation Summary\n\n${fallbackSummary}\n\n---\n\n## User's Current Request\n\n${userPrompt}\n\nPlease respond to the user's request based on the conversation summary above.`; - - try { - const retryResult = await noToolAgent.generateText( - contextWithOrg as any, - { userId: `retry-${threadId}-${attempt}` }, - { prompt: minimalPrompt }, - { storageOptions: { saveMessages: 'none' } }, - ); - - const retryText = (retryResult as { text?: string }).text?.trim(); - if (retryText) { - debugLog(`No-tool retry succeeded on attempt ${attempt}`); - await saveMessage(ctx, components.agent, { - threadId, - message: { role: 'assistant', content: retryText }, - }); - return retryText; - } - - debugLog(`No-tool retry attempt ${attempt} returned empty`); - } catch (retryError) { - debugLog(`No-tool retry attempt ${attempt} failed:`, retryError); - } - } - - throw new Error( - `Agent completed without generating a response message after ${MAX_NO_TOOL_RETRIES} no-tool retries (${toolCallCount} tool calls made${tokenInfo}). Context overflow could not be recovered.`, - ); -} diff --git a/services/platform/convex/model/chat_agent/generate_agent_response.ts b/services/platform/convex/model/chat_agent/generate_agent_response.ts index fa1e02447e..95f88ee07a 100644 --- a/services/platform/convex/model/chat_agent/generate_agent_response.ts +++ b/services/platform/convex/model/chat_agent/generate_agent_response.ts @@ -11,7 +11,6 @@ import type { ActionCtx } from '../../_generated/server'; import { components, internal } from '../../_generated/api'; import type { Id } from '../../_generated/dataModel'; import { createChatAgent } from '../../lib/create_chat_agent'; -import { handleContextOverflowNoToolRetry } from './context_overflow_retry'; import type { FileAttachment } from './chat_with_agent'; import { getFile } from '@convex-dev/agent'; import type { ImagePart as AIImagePart, FilePart as AIFilePart } from 'ai'; @@ -297,9 +296,6 @@ export async function generateAgentResponse( } } - // Determine if we need special handling for attachments - const hasAttachmentContent = promptContent !== undefined; - // Use streamText with saveStreamDeltas for real-time UI updates // This allows the UI to show tool calls as they happen const streamResult = await agent.streamText( @@ -334,10 +330,18 @@ export async function generateAgentResponse( } // Get the final result after stream completes - const result: { text?: string; steps?: unknown[]; usage?: Usage } = { + const result: { + text?: string; + steps?: unknown[]; + usage?: Usage; + finishReason?: string; + warnings?: unknown[]; + } = { text: finalText, steps: await streamResult.steps, usage: await streamResult.usage, + finishReason: await streamResult.finishReason, + warnings: await streamResult.warnings, }; clearTimeout(timeoutId); @@ -361,16 +365,64 @@ export async function generateAgentResponse( ); } - let responseText = (result.text || '').trim(); + const responseText = (result.text || '').trim(); if (!responseText) { - responseText = await handleContextOverflowNoToolRetry(ctx, { - threadId, - promptMessageId, - toolCallCount: toolCalls.length, - usage: result.usage, - contextWithOrg, + const usage = result.usage; + const inputTokens = usage?.inputTokens ?? 0; + const outputTokens = usage?.outputTokens ?? 0; + const warnings = result.warnings; + + // Build detailed step summary for debugging + const stepSummary = steps.map((step, i) => { + const finishReason = step.finishReason; + const textLen = step.text?.length ?? 0; + const toolCallsCount = step.toolCalls?.length ?? 0; + const toolResultsCount = step.toolResults?.length ?? 0; + + // Extract content info (may contain error details) + let contentInfo: string | undefined; + if (step.content) { + if (typeof step.content === 'string') { + contentInfo = step.content.slice(0, 200); + } else if (Array.isArray(step.content)) { + contentInfo = `array[${step.content.length}]`; + } + } + + // Check providerMetadata for errors + const providerError = step.providerMetadata?.error || step.providerMetadata?.openai?.error; + + const stepInfo: Record = { + step: i, + finishReason, + textLen, + toolCalls: toolCallsCount, + toolResults: toolResultsCount, + }; + + if (contentInfo) stepInfo.content = contentInfo; + if (providerError) stepInfo.providerError = providerError; + if (step.warnings?.length) stepInfo.warnings = step.warnings; + + return stepInfo; }); + + const errorDetails: Record = { + model: envModel, + inputTokens, + outputTokens, + stepCount: steps.length, + steps: stepSummary, + }; + + if (warnings && warnings.length > 0) { + errorDetails.warnings = warnings; + } + + throw new Error( + `Agent returned empty response: ${JSON.stringify(errorDetails)}`, + ); } return { From 09063a5b1cd0da77d248b7258ab9e4e49c46b2ec Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Fri, 12 Dec 2025 22:26:22 +0800 Subject: [PATCH 4/4] chore: update default AI models in env example --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index a3d1718e27..9e828c5229 100644 --- a/.env.example +++ b/.env.example @@ -32,8 +32,8 @@ ENCRYPTION_SECRET_HEX=3143246f44def075d40141fb849faffcf409fbbeb7a282a3a7c2f4396f # text-embedding-3-large). OPENAI_BASE_URL=https://openrouter.ai/api/v1 OPENAI_API_KEY=your-openrouter-api-key -OPENAI_MODEL=x-ai/grok-4.1-fast -OPENAI_CODING_MODEL=x-ai/grok-4.1-fast +OPENAI_MODEL=x-ai/grok-4-fast +OPENAI_CODING_MODEL=openai/gpt-5-mini OPENAI_EMBEDDING_MODEL=openai/text-embedding-3-large # EMBEDDING_DIMENSIONS=3072