diff --git a/examples/agents/chat-agent.json b/examples/agents/chat-agent.json index d9a02adc51..4d139c31da 100644 --- a/examples/agents/chat-agent.json +++ b/examples/agents/chat-agent.json @@ -18,8 +18,8 @@ "document_write" ], "supportedModels": [ - "moonshotai/kimi-k2.5", "deepseek/deepseek-v3.2", + "moonshotai/kimi-k2.5", "anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "anthropic/claude-haiku-4.5", diff --git a/examples/providers/openrouter.json b/examples/providers/openrouter.json index 0652388825..27be49cb84 100644 --- a/examples/providers/openrouter.json +++ b/examples/providers/openrouter.json @@ -4,7 +4,7 @@ "baseUrl": "https://openrouter.ai/api/v1", "supportsStructuredOutputs": true, "defaults": { - "chat": "moonshotai/kimi-k2.5", + "chat": "deepseek/deepseek-v3.2", "vision": "qwen/qwen3-vl-32b-instruct", "embedding": "qwen/qwen3-embedding-8b" }, @@ -13,130 +13,227 @@ "id": "moonshotai/kimi-k2.5", "displayName": "Kimi K2.5", "description": "High-performance general-purpose model", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 38, "outputCentsPerMillion": 172 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 38, + "outputCentsPerMillion": 172 + } }, { "id": "deepseek/deepseek-v3.2", "displayName": "DeepSeek V3.2", "description": "Strong reasoning and general capabilities", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 26, "outputCentsPerMillion": 38 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 26, + "outputCentsPerMillion": 38 + } }, { "id": "qwen/qwen3-next-80b-a3b-instruct", "displayName": "Qwen3 Next 80B", "description": "Fast and efficient instruction-following model", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 9, "outputCentsPerMillion": 110 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 9, + "outputCentsPerMillion": 110 + } }, { "id": "qwen/qwen3.5-35b-a3b", "displayName": "Qwen3.5 35B", "description": "Compact and fast model", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 16, "outputCentsPerMillion": 130 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 16, + "outputCentsPerMillion": 130 + } }, { "id": "anthropic/claude-opus-4.6", "displayName": "Claude Opus 4.6", "description": "Most capable model for complex reasoning and coding", - "tags": ["chat", "vision"], - "cost": { "inputCentsPerMillion": 500, "outputCentsPerMillion": 2500 } + "tags": [ + "chat", + "vision" + ], + "cost": { + "inputCentsPerMillion": 500, + "outputCentsPerMillion": 2500 + } }, { "id": "openai/gpt-5.2", "displayName": "GPT-5.2", "description": "OpenAI's latest flagship model", - "tags": ["chat", "vision"], - "cost": { "inputCentsPerMillion": 175, "outputCentsPerMillion": 1400 } + "tags": [ + "chat", + "vision" + ], + "cost": { + "inputCentsPerMillion": 175, + "outputCentsPerMillion": 1400 + } }, { "id": "qwen/qwen3-vl-32b-instruct", "displayName": "Qwen3 VL 32B", "description": "Vision-language model for image understanding", - "tags": ["chat", "vision"], - "cost": { "inputCentsPerMillion": 10, "outputCentsPerMillion": 42 } + "tags": [ + "chat", + "vision" + ], + "cost": { + "inputCentsPerMillion": 10, + "outputCentsPerMillion": 42 + } }, { "id": "qwen/qwen3-embedding-8b", "displayName": "Qwen3 Embedding 8B", "description": "Text embedding model for semantic search", - "tags": ["embedding"], + "tags": [ + "embedding" + ], "dimensions": 1536, - "cost": { "inputCentsPerMillion": 1, "outputCentsPerMillion": 0 } + "cost": { + "inputCentsPerMillion": 1, + "outputCentsPerMillion": 0 + } }, { "id": "openai/gpt-5.2-chat", "displayName": "GPT-5.2 Instant", "description": "Fast, low-latency variant of GPT-5.2 optimized for chat", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 175, "outputCentsPerMillion": 1400 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 175, + "outputCentsPerMillion": 1400 + } }, { "id": "openai/gpt-5.2-pro", "displayName": "GPT-5.2 Pro", "description": "Most advanced GPT-5.2 variant for complex reasoning tasks", - "tags": ["chat", "vision"], - "cost": { "inputCentsPerMillion": 2100, "outputCentsPerMillion": 16800 } + "tags": [ + "chat", + "vision" + ], + "cost": { + "inputCentsPerMillion": 2100, + "outputCentsPerMillion": 16800 + } }, { "id": "anthropic/claude-sonnet-4.6", "displayName": "Claude Sonnet 4.6", "description": "Balanced performance and speed for everyday tasks", - "tags": ["chat", "vision"], - "cost": { "inputCentsPerMillion": 300, "outputCentsPerMillion": 1500 } + "tags": [ + "chat", + "vision" + ], + "cost": { + "inputCentsPerMillion": 300, + "outputCentsPerMillion": 1500 + } }, { "id": "anthropic/claude-haiku-4.5", "displayName": "Claude Haiku 4.5", "description": "Fast and compact model for lightweight tasks", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 100, "outputCentsPerMillion": 500 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 100, + "outputCentsPerMillion": 500 + } }, { "id": "google/gemini-3.1-pro-preview", "displayName": "Gemini 3 Pro", "description": "Google's most capable Gemini model", - "tags": ["chat", "vision"], - "cost": { "inputCentsPerMillion": 200, "outputCentsPerMillion": 1200 } + "tags": [ + "chat", + "vision" + ], + "cost": { + "inputCentsPerMillion": 200, + "outputCentsPerMillion": 1200 + } }, { "id": "google/gemini-3-flash-preview", "displayName": "Gemini 3 Flash", "description": "Fast and efficient Gemini model", - "tags": ["chat", "vision"], - "cost": { "inputCentsPerMillion": 50, "outputCentsPerMillion": 300 } + "tags": [ + "chat", + "vision" + ], + "cost": { + "inputCentsPerMillion": 50, + "outputCentsPerMillion": 300 + } }, { "id": "mistralai/mistral-large-2512", "displayName": "Mistral Large 3", "description": "Mistral AI's flagship large language model", - "tags": ["chat"], + "tags": [ + "chat" + ], "maxOutputTokens": 8192, - "cost": { "inputCentsPerMillion": 50, "outputCentsPerMillion": 150 } + "cost": { + "inputCentsPerMillion": 50, + "outputCentsPerMillion": 150 + } }, { "id": "mistralai/mistral-medium-3", "displayName": "Mistral Medium 3", "description": "Balanced Mistral model for general tasks", - "tags": ["chat"], + "tags": [ + "chat" + ], "maxOutputTokens": 8192, - "cost": { "inputCentsPerMillion": 40, "outputCentsPerMillion": 200 } + "cost": { + "inputCentsPerMillion": 40, + "outputCentsPerMillion": 200 + } }, { "id": "meta-llama/llama-4-maverick", "displayName": "LLaMA 4 Maverick", "description": "Meta's powerful open-source large language model", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 15, "outputCentsPerMillion": 60 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 15, + "outputCentsPerMillion": 60 + } }, { "id": "meta-llama/llama-4-scout", "displayName": "LLaMA 4 Scout", "description": "Meta's efficient open-source language model", - "tags": ["chat"], - "cost": { "inputCentsPerMillion": 8, "outputCentsPerMillion": 30 } + "tags": [ + "chat" + ], + "cost": { + "inputCentsPerMillion": 8, + "outputCentsPerMillion": 30 + } } ], "i18n": { diff --git a/services/platform/app/features/chat/hooks/use-convex-file-upload.ts b/services/platform/app/features/chat/hooks/use-convex-file-upload.ts index 2d0d4854c3..62d30b0bb4 100644 --- a/services/platform/app/features/chat/hooks/use-convex-file-upload.ts +++ b/services/platform/app/features/chat/hooks/use-convex-file-upload.ts @@ -195,6 +195,7 @@ export function useConvexFileUpload(config: ConvexFileUploadConfig) { fileName: fileToUpload.name, contentType: resolvedType || 'application/octet-stream', size: fileToUpload.size, + source: 'user' as const, }); const attachment: FileAttachment = { diff --git a/services/platform/app/features/settings/governance/components/retention-editor.tsx b/services/platform/app/features/settings/governance/components/retention-editor.tsx new file mode 100644 index 0000000000..d28dbe459d --- /dev/null +++ b/services/platform/app/features/settings/governance/components/retention-editor.tsx @@ -0,0 +1,223 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useState } from 'react'; + +import { Input } from '@/app/components/ui/forms/input'; +import { Switch } from '@/app/components/ui/forms/switch'; +import { Stack } from '@/app/components/ui/layout/layout'; +import { PageSection } from '@/app/components/ui/layout/page-section'; +import { Text } from '@/app/components/ui/typography/text'; +import { useAbility } from '@/app/hooks/use-ability'; +import { useToast } from '@/app/hooks/use-toast'; +import { useT } from '@/lib/i18n/client'; +import { + retentionPolicyConfigSchema, + type RetentionPolicyConfig, +} from '@/lib/shared/schemas/governance'; +import { isRecord } from '@/lib/utils/type-guards'; + +import { useUpsertGovernancePolicy } from '../hooks/mutations'; +import { useGovernancePolicy } from '../hooks/queries'; + +interface RetentionEditorProps { + organizationId: string; +} + +function parseRetentionConfig(policy: unknown): RetentionPolicyConfig { + const config = isRecord(policy) ? policy : {}; + const result = retentionPolicyConfigSchema.safeParse(config); + if (result.success) { + return result.data; + } + return { enabled: false, retentionDays: 90 }; +} + +export function RetentionEditor({ organizationId }: RetentionEditorProps) { + const { t } = useT('governance'); + const { toast } = useToast(); + const ability = useAbility(); + + const { data: policy, isLoading } = useGovernancePolicy( + organizationId, + 'retention_policy', + ); + const upsertMutation = useUpsertGovernancePolicy(); + + const savedConfig = useMemo( + () => parseRetentionConfig(policy?.config), + [policy], + ); + + const [enabled, setEnabled] = useState(false); + const [retentionDays, setRetentionDays] = useState(90); + const [userTempEnabled, setUserTempEnabled] = useState(false); + const [userTempRetentionHours, setUserTempRetentionHours] = useState(24); + const [agentTempEnabled, setAgentTempEnabled] = useState(false); + const [agentTempRetentionHours, setAgentTempRetentionHours] = useState(24); + + useEffect(() => { + setEnabled(savedConfig.enabled); + setRetentionDays(savedConfig.retentionDays); + setUserTempEnabled(savedConfig.userTempEnabled ?? false); + setUserTempRetentionHours(savedConfig.userTempRetentionHours ?? 24); + setAgentTempEnabled(savedConfig.agentTempEnabled ?? false); + setAgentTempRetentionHours(savedConfig.agentTempRetentionHours ?? 24); + }, [savedConfig]); + + const cannotManage = ability.cannot('write', 'orgSettings'); + + const saveConfig = useCallback( + async (patch: Partial) => { + const fullConfig: RetentionPolicyConfig = { + enabled, + retentionDays, + userTempEnabled, + userTempRetentionHours, + agentTempEnabled, + agentTempRetentionHours, + ...patch, + }; + try { + await upsertMutation.mutateAsync({ + organizationId, + policyType: 'retention_policy', + config: fullConfig, + }); + toast({ title: t('retentionPolicy.saved'), variant: 'success' }); + } catch (error: unknown) { + const message = + error instanceof Error ? error.message : 'Failed to save'; + toast({ title: message, variant: 'destructive' }); + } + }, + [ + organizationId, + upsertMutation, + toast, + t, + enabled, + retentionDays, + userTempEnabled, + userTempRetentionHours, + agentTempEnabled, + agentTempRetentionHours, + ], + ); + + if (isLoading) { + return null; + } + + return ( + + { + setEnabled(checked); + void saveConfig({ enabled: checked }); + }} + disabled={cannotManage || upsertMutation.isPending} + /> + } + > +
+ + setRetentionDays(e.target.value ? Number(e.target.value) : 0) + } + onBlur={() => void saveConfig({ retentionDays })} + disabled={cannotManage || !enabled} + size="sm" + placeholder="e.g. 90" + min={0} + /> + + Documents older than this will be deleted. + +
+
+ + { + setUserTempEnabled(checked); + void saveConfig({ userTempEnabled: checked }); + }} + disabled={cannotManage || upsertMutation.isPending} + /> + } + > +
+ + setUserTempRetentionHours( + e.target.value ? Number(e.target.value) : 0, + ) + } + onBlur={() => void saveConfig({ userTempRetentionHours })} + disabled={cannotManage || !userTempEnabled} + size="sm" + placeholder="e.g. 24" + min={0} + /> + + Temporary files older than this will be deleted. + +
+
+ + { + setAgentTempEnabled(checked); + void saveConfig({ agentTempEnabled: checked }); + }} + disabled={cannotManage || upsertMutation.isPending} + /> + } + > +
+ + setAgentTempRetentionHours( + e.target.value ? Number(e.target.value) : 0, + ) + } + onBlur={() => void saveConfig({ agentTempRetentionHours })} + disabled={cannotManage || !agentTempEnabled} + size="sm" + placeholder="e.g. 24" + min={0} + /> + + Temporary files older than this will be deleted. + +
+
+
+ ); +} diff --git a/services/platform/app/routes/dashboard/$id/settings/governance.tsx b/services/platform/app/routes/dashboard/$id/settings/governance.tsx index 781f230a5f..c2b747680d 100644 --- a/services/platform/app/routes/dashboard/$id/settings/governance.tsx +++ b/services/platform/app/routes/dashboard/$id/settings/governance.tsx @@ -5,6 +5,7 @@ import { z } from 'zod'; import { AccessDenied } from '@/app/components/layout/access-denied'; import { Tabs } from '@/app/components/ui/navigation/tabs'; import { BudgetEditor } from '@/app/features/settings/governance/components/budget-editor'; +import { RetentionEditor } from '@/app/features/settings/governance/components/retention-editor'; import { SystemPromptEditor } from '@/app/features/settings/governance/components/system-prompt-editor'; import { UsageDashboard } from '@/app/features/settings/governance/components/usage-dashboard'; import { useAbility, useAbilityLoading } from '@/app/hooks/use-ability'; @@ -54,6 +55,11 @@ function GovernanceSettingsPage() { label: 'Budgets', content: , }, + { + value: 'retention', + label: 'Retention', + content: , + }, { value: 'usage', label: 'Usage', diff --git a/services/platform/convex/agent_tools/files/excel_tool.ts b/services/platform/convex/agent_tools/files/excel_tool.ts index a3c4becdf1..54a33c8ed7 100644 --- a/services/platform/convex/agent_tools/files/excel_tool.ts +++ b/services/platform/convex/agent_tools/files/excel_tool.ts @@ -209,6 +209,7 @@ To also save the file to a folder in the documents hub, call document_write with contentType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', size: blob.size, + source: 'agent', }, ); diff --git a/services/platform/convex/agent_tools/files/text_tool.ts b/services/platform/convex/agent_tools/files/text_tool.ts index b7d8d0e630..ebafcb5041 100644 --- a/services/platform/convex/agent_tools/files/text_tool.ts +++ b/services/platform/convex/agent_tools/files/text_tool.ts @@ -138,6 +138,7 @@ To also save the file to a folder in the documents hub, call document_write with fileName: filename, contentType: 'text/plain; charset=utf-8', size: blob.size, + source: 'agent', }, ); diff --git a/services/platform/convex/agents/webhooks/http_actions.ts b/services/platform/convex/agents/webhooks/http_actions.ts index 28eba2c8c9..1bb24e3229 100644 --- a/services/platform/convex/agents/webhooks/http_actions.ts +++ b/services/platform/convex/agents/webhooks/http_actions.ts @@ -109,6 +109,7 @@ export const agentWebhookHandler = httpAction(async (ctx, req) => { fileName: file.name, contentType: file.type || 'application/octet-stream', size: file.size, + source: 'agent', }, ); attachment = { diff --git a/services/platform/convex/documents/generate_document.ts b/services/platform/convex/documents/generate_document.ts index efa71fd2fa..ca2abdbf4d 100644 --- a/services/platform/convex/documents/generate_document.ts +++ b/services/platform/convex/documents/generate_document.ts @@ -120,6 +120,7 @@ export async function generateDocument( fileName: finalFileName, contentType, size, + source: 'agent', }, ); diff --git a/services/platform/convex/documents/generate_docx.ts b/services/platform/convex/documents/generate_docx.ts index 070f579d9e..2f7a652360 100644 --- a/services/platform/convex/documents/generate_docx.ts +++ b/services/platform/convex/documents/generate_docx.ts @@ -126,6 +126,7 @@ export async function generateDocx( fileName: finalFileName, contentType, size: docxBytes.length, + source: 'agent', }, ); diff --git a/services/platform/convex/documents/generate_docx_from_template.ts b/services/platform/convex/documents/generate_docx_from_template.ts index b4c828a18a..e2f9939025 100644 --- a/services/platform/convex/documents/generate_docx_from_template.ts +++ b/services/platform/convex/documents/generate_docx_from_template.ts @@ -132,6 +132,7 @@ export async function generateDocxFromTemplate( fileName: finalFileName, contentType, size: docxBytes.length, + source: 'agent', }, ); diff --git a/services/platform/convex/documents/generate_pptx.ts b/services/platform/convex/documents/generate_pptx.ts index deeadf984c..9895119b68 100644 --- a/services/platform/convex/documents/generate_pptx.ts +++ b/services/platform/convex/documents/generate_pptx.ts @@ -178,6 +178,7 @@ export async function generatePptx( fileName: finalFileName, contentType, size: pptxBytes.length, + source: 'agent', }, ); diff --git a/services/platform/convex/documents/upload_base64_to_storage.ts b/services/platform/convex/documents/upload_base64_to_storage.ts index cdf1a16b3c..82aa9c6eac 100644 --- a/services/platform/convex/documents/upload_base64_to_storage.ts +++ b/services/platform/convex/documents/upload_base64_to_storage.ts @@ -71,7 +71,7 @@ export async function uploadBase64ToStorage( await ctx.runMutation( internal.file_metadata.internal_mutations.saveFileMetadata, - { organizationId, storageId, fileName, contentType, size }, + { organizationId, storageId, fileName, contentType, size, source: 'agent' }, ); // Build download URL using our custom HTTP endpoint that sets Content-Disposition diff --git a/services/platform/convex/file_metadata/__tests__/internal_mutations.test.ts b/services/platform/convex/file_metadata/__tests__/internal_mutations.test.ts index 23e783f92e..2ea106ae08 100644 --- a/services/platform/convex/file_metadata/__tests__/internal_mutations.test.ts +++ b/services/platform/convex/file_metadata/__tests__/internal_mutations.test.ts @@ -26,6 +26,17 @@ vi.mock('../../_generated/server', async (importOriginal) => { }; }); +vi.mock('../../lib/rate_limiter/helpers', () => ({ + checkOrganizationRateLimit: vi.fn(), + RateLimitExceededError: class extends Error {}, +})); + +vi.mock('../../_generated/api', () => ({ + internal: { + governance: { retention_cleanup: { runRetentionCleanup: 'mock' } }, + }, +})); + function createMockCtx(existingDoc: Record | null = null) { const builder = { withIndex: vi.fn().mockReturnThis(), @@ -38,6 +49,9 @@ function createMockCtx(existingDoc: Record | null = null) { insert: vi.fn().mockResolvedValue('fm_new'), patch: vi.fn().mockResolvedValue(undefined), }, + scheduler: { + runAfter: vi.fn().mockResolvedValue(undefined), + }, }; return { ctx, builder }; diff --git a/services/platform/convex/file_metadata/__tests__/mutations.test.ts b/services/platform/convex/file_metadata/__tests__/mutations.test.ts index e4dee67631..a0f117ea13 100644 --- a/services/platform/convex/file_metadata/__tests__/mutations.test.ts +++ b/services/platform/convex/file_metadata/__tests__/mutations.test.ts @@ -26,6 +26,17 @@ vi.mock('../../_generated/server', async (importOriginal) => { }; }); +vi.mock('../../lib/rate_limiter/helpers', () => ({ + checkOrganizationRateLimit: vi.fn(), + RateLimitExceededError: class extends Error {}, +})); + +vi.mock('../../_generated/api', () => ({ + internal: { + governance: { retention_cleanup: { runRetentionCleanup: 'mock' } }, + }, +})); + const mockGetAuthUser = vi.fn(); vi.mock('../../auth', () => ({ authComponent: { @@ -45,6 +56,9 @@ function createMockCtx(existingDoc: Record | null = null) { insert: vi.fn().mockResolvedValue('fm_new'), patch: vi.fn().mockResolvedValue(undefined), }, + scheduler: { + runAfter: vi.fn().mockResolvedValue(undefined), + }, }; return { ctx, builder }; diff --git a/services/platform/convex/file_metadata/internal_mutations.ts b/services/platform/convex/file_metadata/internal_mutations.ts index e0944eb35c..96ad1bc5b2 100644 --- a/services/platform/convex/file_metadata/internal_mutations.ts +++ b/services/platform/convex/file_metadata/internal_mutations.ts @@ -1,6 +1,11 @@ import { v } from 'convex/values'; +import { internal } from '../_generated/api'; import { internalMutation } from '../_generated/server'; +import { + RateLimitExceededError, + checkOrganizationRateLimit, +} from '../lib/rate_limiter/helpers'; export const saveFileMetadata = internalMutation({ args: { @@ -10,6 +15,7 @@ export const saveFileMetadata = internalMutation({ contentType: v.string(), size: v.number(), documentId: v.optional(v.id('documents')), + source: v.optional(v.union(v.literal('user'), v.literal('agent'))), }, async handler(ctx, args) { const existing = await ctx.db @@ -26,18 +32,41 @@ export const saveFileMetadata = internalMutation({ if (args.documentId !== undefined) { patchData.documentId = args.documentId; } + if (args.source !== undefined) { + patchData.source = args.source; + } await ctx.db.patch(existing._id, patchData); return existing._id; } - return await ctx.db.insert('fileMetadata', { + const id = await ctx.db.insert('fileMetadata', { organizationId: args.organizationId, storageId: args.storageId, fileName: args.fileName, contentType: args.contentType, size: args.size, ...(args.documentId !== undefined && { documentId: args.documentId }), + ...(args.source !== undefined && { source: args.source }), }); + + try { + await checkOrganizationRateLimit( + ctx, + 'cleanup:retention', + args.organizationId, + ); + await ctx.scheduler.runAfter( + 0, + internal.governance.retention_cleanup.runRetentionCleanup, + {}, + ); + } catch (error) { + if (!(error instanceof RateLimitExceededError)) { + throw error; + } + } + + return id; }, }); diff --git a/services/platform/convex/file_metadata/mutations.ts b/services/platform/convex/file_metadata/mutations.ts index 32b86a1ae4..0e7e4a2189 100644 --- a/services/platform/convex/file_metadata/mutations.ts +++ b/services/platform/convex/file_metadata/mutations.ts @@ -1,7 +1,12 @@ import { v } from 'convex/values'; +import { internal } from '../_generated/api'; import { mutation } from '../_generated/server'; import { authComponent } from '../auth'; +import { + RateLimitExceededError, + checkOrganizationRateLimit, +} from '../lib/rate_limiter/helpers'; export const saveFileMetadata = mutation({ args: { @@ -11,6 +16,7 @@ export const saveFileMetadata = mutation({ contentType: v.string(), size: v.number(), documentId: v.optional(v.id('documents')), + source: v.optional(v.union(v.literal('user'), v.literal('agent'))), }, async handler(ctx, args) { const authUser = await authComponent.getAuthUser(ctx); @@ -32,17 +38,40 @@ export const saveFileMetadata = mutation({ if (args.documentId !== undefined) { patchData.documentId = args.documentId; } + if (args.source !== undefined) { + patchData.source = args.source; + } await ctx.db.patch(existing._id, patchData); return existing._id; } - return await ctx.db.insert('fileMetadata', { + const id = await ctx.db.insert('fileMetadata', { organizationId: args.organizationId, storageId: args.storageId, fileName: args.fileName, contentType: args.contentType, size: args.size, ...(args.documentId !== undefined && { documentId: args.documentId }), + ...(args.source !== undefined && { source: args.source }), }); + + try { + await checkOrganizationRateLimit( + ctx, + 'cleanup:retention', + args.organizationId, + ); + await ctx.scheduler.runAfter( + 0, + internal.governance.retention_cleanup.runRetentionCleanup, + {}, + ); + } catch (error) { + if (!(error instanceof RateLimitExceededError)) { + throw error; + } + } + + return id; }, }); diff --git a/services/platform/convex/file_metadata/schema.ts b/services/platform/convex/file_metadata/schema.ts index 27b384997e..5ffc1c87ac 100644 --- a/services/platform/convex/file_metadata/schema.ts +++ b/services/platform/convex/file_metadata/schema.ts @@ -5,10 +5,16 @@ export const fileMetadataTable = defineTable({ organizationId: v.string(), storageId: v.id('_storage'), documentId: v.optional(v.id('documents')), + source: v.optional(v.union(v.literal('user'), v.literal('agent'))), fileName: v.string(), contentType: v.string(), size: v.number(), }) .index('by_organizationId', ['organizationId']) .index('by_storageId', ['storageId']) - .index('by_organizationId_and_documentId', ['organizationId', 'documentId']); + .index('by_organizationId_and_documentId', ['organizationId', 'documentId']) + .index('by_organizationId_and_source_and_documentId', [ + 'organizationId', + 'source', + 'documentId', + ]); diff --git a/services/platform/convex/governance/internal_mutations_retention.ts b/services/platform/convex/governance/internal_mutations_retention.ts index 736cdc8887..e985c79dc4 100644 --- a/services/platform/convex/governance/internal_mutations_retention.ts +++ b/services/platform/convex/governance/internal_mutations_retention.ts @@ -1,6 +1,7 @@ import { v } from 'convex/values'; import { internalMutation } from '../_generated/server'; +import { deleteStorageWithMetadata } from '../file_metadata/helpers'; export const deleteExpiredDocument = internalMutation({ args: { @@ -27,3 +28,19 @@ export const deleteExpiredDocument = internalMutation({ return null; }, }); + +export const deleteExpiredTempFile = internalMutation({ + args: { + fileMetadataId: v.id('fileMetadata'), + }, + returns: v.null(), + handler: async (ctx, args) => { + const metadata = await ctx.db.get(args.fileMetadataId); + if (!metadata) { + return null; + } + + await deleteStorageWithMetadata(ctx, metadata.storageId); + return null; + }, +}); diff --git a/services/platform/convex/governance/internal_queries.ts b/services/platform/convex/governance/internal_queries.ts index 60a7a7793e..4e8f08424e 100644 --- a/services/platform/convex/governance/internal_queries.ts +++ b/services/platform/convex/governance/internal_queries.ts @@ -50,12 +50,40 @@ export const listRetentionPolicies = internalQuery({ }, }); +export const listExpiredTempFiles = internalQuery({ + args: { + organizationId: v.string(), + source: v.union(v.literal('user'), v.literal('agent')), + cutoffMs: v.number(), + batchSize: v.number(), + }, + returns: v.any(), + handler: async (ctx, args) => { + const files = []; + for await (const file of ctx.db + .query('fileMetadata') + .withIndex('by_organizationId_and_source_and_documentId', (q) => + q + .eq('organizationId', args.organizationId) + .eq('source', args.source) + .eq('documentId', undefined), + )) { + if (file._creationTime < args.cutoffMs) { + files.push(file); + if (files.length >= args.batchSize) { + break; + } + } + } + return files; + }, +}); + export const listExpiredDocuments = internalQuery({ args: { organizationId: v.string(), cutoffMs: v.number(), batchSize: v.number(), - scope: v.optional(v.string()), }, returns: v.any(), handler: async (ctx, args) => { diff --git a/services/platform/convex/governance/retention_cleanup.ts b/services/platform/convex/governance/retention_cleanup.ts index 22ef6384d7..f12aa504cf 100644 --- a/services/platform/convex/governance/retention_cleanup.ts +++ b/services/platform/convex/governance/retention_cleanup.ts @@ -3,12 +3,21 @@ import { v } from 'convex/values'; import type { RetentionPolicyConfig } from '../../lib/shared/schemas/governance'; -import { isRecord, getBoolean } from '../../lib/utils/type-guards'; +import { isRecord } from '../../lib/utils/type-guards'; import { internal } from '../_generated/api'; import { internalAction } from '../_generated/server'; import { getRagConfig } from '../lib/helpers/rag_config'; const DEFAULT_BATCH_SIZE = 100; +const DEFAULT_TEMP_RETENTION_HOURS = 24; + +function parseConfig(config: unknown): RetentionPolicyConfig | null { + if (!isRecord(config) || typeof config['retentionDays'] !== 'number') { + return null; + } + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- shape validated above + return config as unknown as RetentionPolicyConfig; +} export const runRetentionCleanup = internalAction({ args: {}, @@ -25,56 +34,116 @@ export const runRetentionCleanup = internalAction({ ); for (const policy of policies) { - const config = policy.config; - if ( - isRecord(config) && - getBoolean(config, 'enabled') && - typeof config['retentionDays'] === 'number' - ) { + const config = parseConfig(policy.config); + if (!config) continue; + if (config.enabled || config.userTempEnabled || config.agentTempEnabled) { orgsWithPolicies.push({ organizationId: policy.organizationId, - // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- validated above - config: config as unknown as RetentionPolicyConfig, + config, }); } } + const ragUrl = getRagConfig().serviceUrl; + for (const { organizationId, config } of orgsWithPolicies) { - const cutoffMs = Date.now() - config.retentionDays * 24 * 60 * 60 * 1000; const batchSize = config.batchSize ?? DEFAULT_BATCH_SIZE; - const expiredDocs = await ctx.runQuery( - internal.governance.internal_queries.listExpiredDocuments, - { - organizationId, - cutoffMs, - batchSize, - scope: config.scope, - }, - ); + // 1. Document retention (all documents, regardless of source) + if (config.enabled) { + const cutoffMs = + Date.now() - config.retentionDays * 24 * 60 * 60 * 1000; + + const expiredDocs = await ctx.runQuery( + internal.governance.internal_queries.listExpiredDocuments, + { organizationId, cutoffMs, batchSize }, + ); + + for (const doc of expiredDocs) { + if (doc.fileId) { + try { + await fetch( + `${ragUrl}/api/v1/documents/${encodeURIComponent(doc.fileId)}`, + { method: 'DELETE', signal: AbortSignal.timeout(30000) }, + ); + } catch (error) { + console.warn( + `[RetentionCleanup] Failed to delete RAG entry for document ${doc._id}:`, + error, + ); + } + } + + await ctx.runMutation( + internal.governance.internal_mutations_retention + .deleteExpiredDocument, + { documentId: doc._id }, + ); + } + } - const ragUrl = getRagConfig().serviceUrl; + // 2. User temporary file cleanup + if (config.userTempEnabled) { + const hours = + config.userTempRetentionHours ?? DEFAULT_TEMP_RETENTION_HOURS; + const cutoffMs = Date.now() - hours * 60 * 60 * 1000; - for (const doc of expiredDocs) { - if (doc.fileId) { + const expiredFiles = await ctx.runQuery( + internal.governance.internal_queries.listExpiredTempFiles, + { organizationId, source: 'user', cutoffMs, batchSize }, + ); + + for (const file of expiredFiles) { try { await fetch( - `${ragUrl}/api/v1/documents/${encodeURIComponent(doc.fileId)}`, + `${ragUrl}/api/v1/documents/${encodeURIComponent(file.storageId)}`, { method: 'DELETE', signal: AbortSignal.timeout(30000) }, ); } catch (error) { console.warn( - `[RetentionCleanup] Failed to delete RAG entry for document ${doc._id}:`, + `[RetentionCleanup] Failed to delete RAG entry for temp file ${file._id}:`, error, ); } + + await ctx.runMutation( + internal.governance.internal_mutations_retention + .deleteExpiredTempFile, + { fileMetadataId: file._id }, + ); } + } - await ctx.runMutation( - internal.governance.internal_mutations_retention - .deleteExpiredDocument, - { documentId: doc._id }, + // 3. Agent temporary file cleanup + if (config.agentTempEnabled) { + const hours = + config.agentTempRetentionHours ?? DEFAULT_TEMP_RETENTION_HOURS; + const cutoffMs = Date.now() - hours * 60 * 60 * 1000; + + const expiredFiles = await ctx.runQuery( + internal.governance.internal_queries.listExpiredTempFiles, + { organizationId, source: 'agent', cutoffMs, batchSize }, ); + + for (const file of expiredFiles) { + try { + await fetch( + `${ragUrl}/api/v1/documents/${encodeURIComponent(file.storageId)}`, + { method: 'DELETE', signal: AbortSignal.timeout(30000) }, + ); + } catch (error) { + console.warn( + `[RetentionCleanup] Failed to delete RAG entry for temp file ${file._id}:`, + error, + ); + } + + await ctx.runMutation( + internal.governance.internal_mutations_retention + .deleteExpiredTempFile, + { fileMetadataId: file._id }, + ); + } } } diff --git a/services/platform/convex/lib/rate_limiter/index.ts b/services/platform/convex/lib/rate_limiter/index.ts index f1167957b4..0553cb67f2 100644 --- a/services/platform/convex/lib/rate_limiter/index.ts +++ b/services/platform/convex/lib/rate_limiter/index.ts @@ -194,6 +194,16 @@ export const rateLimiter = new RateLimiter(components.rateLimiter, { period: HOUR, capacity: 120, }, + + // ============================================ + // TIER 6: Maintenance (Fixed Window) + // Background cleanup and retention tasks + // ============================================ + 'cleanup:retention': { + kind: 'fixed window', + rate: 1, + period: HOUR, + }, }); export type RateLimitName = Parameters[1]; diff --git a/services/platform/convex/node_only/integration_sandbox/helpers/create_convex_storage_provider.ts b/services/platform/convex/node_only/integration_sandbox/helpers/create_convex_storage_provider.ts index f66c466e1f..bb67b75985 100644 --- a/services/platform/convex/node_only/integration_sandbox/helpers/create_convex_storage_provider.ts +++ b/services/platform/convex/node_only/integration_sandbox/helpers/create_convex_storage_provider.ts @@ -55,6 +55,7 @@ export function createConvexStorageProvider( fileName, contentType, size: blob.size, + source: 'agent', }, ); @@ -90,6 +91,7 @@ export function createConvexStorageProvider( fileName, contentType, size: blob.size, + source: 'agent', }, ); diff --git a/services/platform/convex/onedrive/import_files_deps.ts b/services/platform/convex/onedrive/import_files_deps.ts index 2aee31a437..38bba09be0 100644 --- a/services/platform/convex/onedrive/import_files_deps.ts +++ b/services/platform/convex/onedrive/import_files_deps.ts @@ -48,7 +48,14 @@ export function createImportFilesDeps( saveFileMetadata: async (storageId, fileName, contentType, size) => { await ctx.runMutation( internal.file_metadata.internal_mutations.saveFileMetadata, - { organizationId, storageId, fileName, contentType, size }, + { + organizationId, + storageId, + fileName, + contentType, + size, + source: 'user', + }, ); }, linkDocumentToFile: async (storageId, documentId) => { diff --git a/services/platform/convex/onedrive/internal_actions.ts b/services/platform/convex/onedrive/internal_actions.ts index 641012eaef..ea2c292f6a 100644 --- a/services/platform/convex/onedrive/internal_actions.ts +++ b/services/platform/convex/onedrive/internal_actions.ts @@ -120,6 +120,7 @@ export const downloadAndStoreFile = internalAction({ fileName: args.fileName ?? args.itemId, contentType, size, + source: 'user', }, ); }, diff --git a/services/platform/convex/onedrive/upload_and_create_document_deps.ts b/services/platform/convex/onedrive/upload_and_create_document_deps.ts index b6195a10f6..72ec362ee1 100644 --- a/services/platform/convex/onedrive/upload_and_create_document_deps.ts +++ b/services/platform/convex/onedrive/upload_and_create_document_deps.ts @@ -43,7 +43,14 @@ export function createUploadAndCreateDocDeps( saveFileMetadata: async (storageId, fileName, contentType, size) => { await ctx.runMutation( internal.file_metadata.internal_mutations.saveFileMetadata, - { organizationId, storageId, fileName, contentType, size }, + { + organizationId, + storageId, + fileName, + contentType, + size, + source: 'user', + }, ); }, linkDocumentToFile: async (storageId, documentId) => { diff --git a/services/platform/convex/workflow_engine/action_defs/document/helpers/apply_docx_structured.ts b/services/platform/convex/workflow_engine/action_defs/document/helpers/apply_docx_structured.ts index 77a1a0b367..c4dddcf758 100644 --- a/services/platform/convex/workflow_engine/action_defs/document/helpers/apply_docx_structured.ts +++ b/services/platform/convex/workflow_engine/action_defs/document/helpers/apply_docx_structured.ts @@ -172,6 +172,7 @@ export async function applyDocxStructured( fileName: finalFileName, contentType: DOCX_CONTENT_TYPE, size: docxBytes.length, + source: 'agent', }, ); } diff --git a/services/platform/lib/shared/schemas/governance.ts b/services/platform/lib/shared/schemas/governance.ts index b776382780..daf61501f2 100644 --- a/services/platform/lib/shared/schemas/governance.ts +++ b/services/platform/lib/shared/schemas/governance.ts @@ -52,8 +52,11 @@ export type UploadPolicyConfig = z.infer; export const retentionPolicyConfigSchema = z.object({ enabled: z.boolean(), retentionDays: z.number().nonnegative(), - scope: z.enum(['all', 'upload', 'agent']).optional(), batchSize: z.number().nonnegative().optional(), + userTempEnabled: z.boolean().optional(), + userTempRetentionHours: z.number().nonnegative().optional(), + agentTempEnabled: z.boolean().optional(), + agentTempRetentionHours: z.number().nonnegative().optional(), }); export type RetentionPolicyConfig = z.infer; diff --git a/services/platform/messages/en.json b/services/platform/messages/en.json index 64723b8fe2..e4dd069df8 100644 --- a/services/platform/messages/en.json +++ b/services/platform/messages/en.json @@ -3685,6 +3685,7 @@ "description": "Configure automatic deletion of uploaded files after a specified period.", "retentionDays": "Retention period (days)", "scope": "Apply to", + "tempFileRetentionHours": "Temporary file retention (hours)", "saved": "Retention policy saved" }, "featureFlags": {