From 2604f08a72a1c0386704ee1f6f9990de446d9089 Mon Sep 17 00:00:00 2001 From: Emir Karabeg Date: Tue, 1 Sep 2026 17:25:47 -0700 Subject: [PATCH 01/76] feat(search): add Sim Search mode, connectors tab, and cited sources - Build/Search mode switcher in the chat composer; Search mode swaps the suggested actions for Connect Sim Search rows that open the OAuth modal - Search tab beside Integrations and Skills: a personally scoped catalog of knowledge-base connectors with connect-in-place rows and a connected-credential page without sharing or members - special tag contract: inline citation chips at the citation point plus a scrolling, right-faded sources strip under the reply - chip `shape` variant so a fully round chip is chip chrome, not a class override - keep the connector catalog out of the workflow editor's module graph and record the new routes in the boundary baseline --- .../integration-tabs-header.tsx | 13 +- .../[workspaceId]/home/components/index.ts | 1 - .../chat-content/chat-content.test.ts | 8 + .../components/chat-content/chat-content.tsx | 131 +++++++--- .../components/chat-content/chat-sanitize.ts | 10 +- .../components/chat-content/external-link.tsx | 9 +- .../message-content/components/index.ts | 2 + .../components/message-sources/index.ts | 1 + .../message-sources/message-sources.test.tsx | 81 ++++++ .../message-sources/message-sources.tsx | 40 +++ .../components/source-chip/index.ts | 1 + .../components/source-chip/source-chip.tsx | 89 +++++++ .../components/special-tags/index.ts | 1 + .../special-tags/special-tags.test.ts | 48 ++++ .../components/special-tags/special-tags.tsx | 58 ++++- .../message-content/message-content.tsx | 17 +- .../components/message-content/utils.test.ts | 37 ++- .../home/components/message-content/utils.ts | 29 ++- .../connector-actions.test.ts | 78 ++++++ .../suggested-actions/connector-actions.ts | 50 ++++ .../suggested-actions.test.tsx | 150 +++++++++++ .../suggested-actions/suggested-actions.tsx | 92 ++++--- .../components/suggested-actions/types.ts | 22 ++ .../suggested-actions/weighted-sample.ts | 27 ++ .../components/user-input/components/index.ts | 1 + .../components/mode-switcher/index.ts | 1 + .../mode-switcher/mode-switcher.test.tsx | 110 ++++++++ .../mode-switcher/mode-switcher.tsx | 63 +++++ .../home/components/user-input/user-input.tsx | 8 +- .../app/workspace/[workspaceId]/home/home.tsx | 9 +- .../search/connected/[credentialId]/page.tsx | 15 ++ .../search-credential-detail.tsx | 240 ++++++++++++++++++ .../workspace/[workspaceId]/search/error.tsx | 15 ++ .../search/hooks/use-search-credentials.ts | 48 ++++ .../workspace/[workspaceId]/search/page.tsx | 30 +++ .../[workspaceId]/search/search-params.ts | 17 ++ .../[workspaceId]/search/search.test.tsx | 176 +++++++++++++ .../workspace/[workspaceId]/search/search.tsx | 210 +++++++++++++++ apps/sim/lib/posthog/events.ts | 13 +- apps/sim/lib/sim-search/connectors.test.ts | 94 +++++++ apps/sim/lib/sim-search/connectors.ts | 79 ++++++ apps/sim/stores/mothership-mode/store.ts | 38 +++ apps/sim/stores/reset-all-stores.ts | 2 + .../chip-dropdown/chip-dropdown.tsx | 3 +- .../emcn/src/components/chip/chip-chrome.ts | 13 +- packages/emcn/src/components/chip/chip.tsx | 29 ++- packages/emcn/src/components/index.ts | 2 + ...check-tool-registry-boundary.baseline.json | 94 ++++--- 48 files changed, 2170 insertions(+), 135 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/page.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/search/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/search/page.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/search/search-params.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/search/search.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/search/search.tsx create mode 100644 apps/sim/lib/sim-search/connectors.test.ts create mode 100644 apps/sim/lib/sim-search/connectors.ts create mode 100644 apps/sim/stores/mothership-mode/store.ts diff --git a/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx b/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx index 53efc5a0175..60863ec586f 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx @@ -3,19 +3,19 @@ import { ChipLink, cn } from '@sim/emcn' import { HEADER_ACTION_CLUSTER, PAGE_HEADER_BAR } from '@/components/page-header-bar' interface IntegrationTabsHeaderProps { - active: 'integrations' | 'skills' + active: 'integrations' | 'skills' | 'search' workspaceId: string /** Trailing actions for the owning page (e.g. skills' "Add skill"). */ rightSlot?: ReactNode } /** - * Top-of-page tab header shared by the Integrations and Skills pages — two halves - * of one surface, so each highlights itself and links to its sibling. + * Top-of-page tab header shared by the Integrations, Skills, and Search pages — + * three views of one surface, so each highlights itself and links to its siblings. * * Lives in the shared workspace components rather than under `integrations/` - * because both pages own it equally; its former home made Skills reach across into - * a sibling feature for its own chrome. + * because every page owns it equally; its former home made Skills reach across + * into a sibling feature for its own chrome. * * The `gap-1` is explicit because chips carry no outer margin — the parent owns the * space between them. @@ -33,6 +33,9 @@ export function IntegrationTabsHeader({ Skills + + Search + {rightSlot &&
{rightSlot}
} ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/index.ts index 8425c127a6e..fb354716ea3 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/index.ts @@ -12,5 +12,4 @@ export { useMothershipResources, } from './mothership-resources-context' export { QueuedMessages } from './queued-messages' -export { SuggestedActions } from './suggested-actions' export { UserInput, type UserInputHandle } from './user-input' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts index 1b3e5d105d9..eb9f86f6035 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts @@ -15,6 +15,14 @@ describe('sanitizeChatDisplayContent', () => { ) }) + it('unwraps source tags from inline code spans', () => { + const content = '`Block them first. {"url":"https://docs.github.com/a"}`' + + expect(sanitizeChatDisplayContent(content)).toBe( + 'Block them first. {"url":"https://docs.github.com/a"}' + ) + }) + it('removes hidden internal references wrapped in inline code', () => { const content = 'Read `internal/tool-results/read-1.md` and found the issue.' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx index c143c72bb4a..d1bfce2c1b2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx @@ -1,6 +1,15 @@ 'use client' -import { type ComponentPropsWithoutRef, memo, useEffect, useMemo, useRef, useState } from 'react' +import { + type ComponentPropsWithoutRef, + createContext, + memo, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react' import { Streamdown } from 'streamdown' import 'streamdown/styles.css' // prismjs core must load before its language components — they register on the @@ -15,10 +24,15 @@ import { Checkbox, CopyCodeButton, cn, languages, highlight as prismHighlight } import { decodeVfsSegmentSafe } from '@/lib/copilot/vfs/path-utils' import { extractTextContent } from '@/lib/core/utils/react-node-text' import { ContextMentionIcon } from '@/app/workspace/[workspaceId]/home/components/context-mention-icon' +import { + SourceChip, + sourceLabel, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-chip' import { type ContentSegment, type CredentialSubmissionPayload, parseSpecialTags, + type SourceTagData, SpecialTags, } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import type { @@ -108,9 +122,33 @@ function nextInlineSegmentLabel(segment?: ContentSegment): string { // Thinking segments are never rendered, so they contribute no following text. if (segment.type === 'text') return segment.content if (segment.type === 'workspace_resource') return segment.data.title || segment.data.id || '' + if (segment.type === 'source') return sourceLabel(segment.data) return '' } +/** + * The `` payloads of the segment being rendered, in emission order. An + * inline citation is written into the markdown as `[label](#src-N)` so it flows + * with its paragraph, and the link renderer resolves `N` back through this + * context — the component map is static, so it is the one channel from segment + * data into it. + */ +const SourceRefsContext = createContext([]) + +const SOURCE_LINK_PREFIX = '#src-' + +interface SourceReferenceProps { + index: number + children?: React.ReactNode +} + +/** The inline citation chip; a dangling index falls back to the link text. */ +function SourceReference({ index, children }: SourceReferenceProps) { + const source = useContext(SourceRefsContext)[index] + if (!source) return <>{children} + return +} + function appendInlineReferenceMarkdown( currentMarkdown: string, referenceMarkdown: string, @@ -263,6 +301,13 @@ const MARKDOWN_COMPONENTS = { ) }, a({ children, href }: { children?: React.ReactNode; href?: string }) { + if (href?.startsWith(SOURCE_LINK_PREFIX)) { + return ( + + {children} + + ) + } if (href?.startsWith('#wsres-')) { const match = href.match(/^#wsres-(\w+)-(.+)$/) const type = match?.[1] @@ -566,14 +611,20 @@ function ChatContentInner({ type BlockSegment = Exclude< ContentSegment, - { type: 'text' } | { type: 'thinking' } | { type: 'workspace_resource' } + { type: 'text' } | { type: 'thinking' } | { type: 'workspace_resource' } | { type: 'source' } > type RenderGroup = | { kind: 'inline'; markdown: string } | { kind: 'block'; segment: BlockSegment; index: number } + const sourceRefs = useMemo( + () => parsed.segments.flatMap((segment) => (segment.type === 'source' ? [segment.data] : [])), + [parsed] + ) + const groups: RenderGroup[] = [] let pendingMarkdown = '' + let sourceIndex = 0 const flushMarkdown = () => { if (pendingMarkdown.trim()) { @@ -596,6 +647,16 @@ function ChatContentInner({ `[${label}](<#wsres-${s.data.type}-${ref}>)`, nextSegment ) + } else if (s.type === 'source') { + // A citation always stands off from the sentence it supports, even when + // the model closes the sentence on punctuation the word-boundary rule + // would otherwise glue the chip to. + if (pendingMarkdown && !/\s$/.test(pendingMarkdown)) pendingMarkdown += ' ' + pendingMarkdown = appendInlineReferenceMarkdown( + pendingMarkdown, + `[${sourceLabel(s.data)}](<${SOURCE_LINK_PREFIX}${sourceIndex++}>)`, + nextSegment + ) } else if (s.type === 'thinking') { // Model-emitted tag bodies are reasoning, not answer text — // never rendered (matches the block-level thinking omission in @@ -621,40 +682,42 @@ function ChatContentInner({ * the new special block mounts. */ return ( -
- {groups.map((group, i) => { - if (group.kind === 'inline') { - return ( -
:first-child]:mt-0 [&>:last-child]:mb-0')} - > - +
+ {groups.map((group, i) => { + if (group.kind === 'inline') { + return ( +
:first-child]:mt-0 [&>:last-child]:mb-0')} > - {group.markdown} - -
+ + {group.markdown} + +
+ ) + } + return ( + ) - } - return ( - - ) - })} -
+ })} +
+ ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts index 4e0792df0ab..1d210bd1b92 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts @@ -2,7 +2,9 @@ const HIDDEN_INLINE_REFERENCE_PATTERN = /`[^`\n]*(?:internal\/tool-results\/|internal\/blocktips\/|components\/integrations\/[^`\n]*README)[^`\n]*`/g /** - * A complete workspace-resource tag: opener, payload, closer. + * A complete inline-chip tag — `` or `` — as + * opener, payload, closer. Both are JSON-bodied tags the model places inside a + * sentence, so both attract the same stray backticks. * * Two constraints on the payload, both load-bearing: * @@ -19,10 +21,10 @@ const HIDDEN_INLINE_REFERENCE_PATTERN = * is rare; the failure it replaces corrupts a whole message and is common. */ const COMPLETE_TAG_SOURCE = - '(?:(?!)[^`])*?<\\/workspace_resource>' + '<(?workspace_resource|source)>(?:(?!<\\k>)[^`])*?<\\/\\k>' /** Non-global so {@link RegExp.test} has no `lastIndex` to carry between calls. */ -const COMPLETE_WORKSPACE_RESOURCE_TAG = new RegExp(COMPLETE_TAG_SOURCE) +const COMPLETE_INLINE_CHIP_TAG = new RegExp(COMPLETE_TAG_SOURCE) /** * One left-to-right pass over the two things that can own a backtick: an inline @@ -57,7 +59,7 @@ export function sanitizeChatDisplayContent(content: string): string { // lifts the tag out either way, so leaving the delimiters would strand a // pair of backticks around a hole. Anything else is someone else's span. const inner = match.slice(1, -1) - return COMPLETE_WORKSPACE_RESOURCE_TAG.test(inner) ? inner : match + return COMPLETE_INLINE_CHIP_TAG.test(inner) ? inner : match }) .replace(HIDDEN_INLINE_REFERENCE_PATTERN, '') } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx index 405acc7a9d4..fd9661d2151 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx @@ -6,7 +6,7 @@ import { faviconUrl } from '@/lib/core/utils/favicon' import { useLinkPreview } from '@/hooks/queries/link-preview' /** Hides a favicon img that failed to load so the link degrades to plain text. */ -function hideBrokenFavicon(e: React.SyntheticEvent): void { +export function hideBrokenFavicon(e: React.SyntheticEvent): void { e.currentTarget.style.display = 'none' } @@ -44,7 +44,10 @@ interface ExternalLinkProps { * which the shell routes to the system browser. In a web browser this is a * no-op and the link opens a new tab as usual. */ -function handleLinkClick(event: React.MouseEvent, href: string): void { +export function handleExternalLinkClick( + event: React.MouseEvent, + href: string +): void { if (event.metaKey || event.ctrlKey || event.shiftKey || event.button !== 0) return if (!shouldOpenInBrowserPanel(href)) return event.preventDefault() @@ -63,7 +66,7 @@ export function ExternalLink({ href, hostname, children }: ExternalLinkProps) { className='not-prose group text-[var(--text-primary)] no-underline' target='_blank' rel='noopener noreferrer' - onClick={(event) => handleLinkClick(event, href)} + onClick={(event) => handleExternalLinkClick(event, href)} > ({ + shouldOpenInBrowserPanel: () => false, + openInBrowserPanel: vi.fn(), +})) +vi.mock('@/lib/integrations', () => ({ + blockTypeToIconMap: { confluence_v2: () => }, +})) + +import { MessageSources } from '@/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount(ui: React.ReactNode) { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render(ui)) +} + +function chips(): HTMLAnchorElement[] { + return Array.from(container?.querySelectorAll('a') ?? []) +} + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('MessageSources', () => { + it('renders one chip per source with the site name or hostname as its label', () => { + mount( + + ) + + expect(chips().map((chip) => chip.textContent)).toEqual(['GitHub Docs', 'example.com']) + expect(chips().map((chip) => chip.getAttribute('href'))).toEqual([ + 'https://docs.github.com/en/a', + 'https://www.example.com/page', + ]) + expect(chips()[0].getAttribute('target')).toBe('_blank') + expect(chips()[0].className).toContain('rounded-full') + }) + + it('shows the connector brand mark when the source names a connector, else the favicon', () => { + mount( + + ) + + const [connector, favicon] = chips() + expect(connector.querySelector('svg[data-brand="confluence"]')).not.toBeNull() + expect(connector.querySelector('img')).toBeNull() + expect(favicon.querySelector('img')?.getAttribute('src')).toContain('docs.github.com') + }) + + it('renders nothing without sources', () => { + mount() + + expect(container?.childElementCount).toBe(0) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx new file mode 100644 index 00000000000..8c72089296e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx @@ -0,0 +1,40 @@ +'use client' + +import { cn } from '@sim/emcn' +import { SourceChip } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-chip' +import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' + +/** + * Right-edge fade so an overflowing strip reads as scrollable rather than cut. + * The strip's trailing padding matches the fade width, so the last chip scrolls + * fully clear of it. + */ +const STRIP_FADE_CLASSES = + 'pr-10 [-webkit-mask-image:linear-gradient(to_right,black_calc(100%_-_40px),transparent)] [mask-image:linear-gradient(to_right,black_calc(100%_-_40px),transparent)]' + +interface MessageSourcesProps { + sources: readonly SourceTagData[] +} + +/** + * Footer strip listing every document a reply cited, once each: one + * horizontally scrolling row of {@link SourceChip}s that fades out at the right + * edge instead of wrapping, so a long list stays a single quiet line under the + * answer. + */ +export function MessageSources({ sources }: MessageSourcesProps) { + if (sources.length === 0) return null + + return ( +
+ {sources.map((source) => ( + + ))} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts new file mode 100644 index 00000000000..4168fe2a584 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts @@ -0,0 +1 @@ +export { SourceChip, sourceLabel } from './source-chip' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx new file mode 100644 index 00000000000..4101ddffb17 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx @@ -0,0 +1,89 @@ +'use client' + +import { chipFilledFillTokens, chipHoverSurfaceClass, cn, OverflowText, Tooltip } from '@sim/emcn' +import { stripVersionSuffix } from '@sim/utils/string' +import { faviconUrl } from '@/lib/core/utils/favicon' +import { blockTypeToIconMap } from '@/lib/integrations' +import { + externalLinkHostname, + handleExternalLinkClick, + hideBrokenFavicon, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link' +import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { BrandIcon, type StyleableIcon } from '@/blocks/brand-icon' + +/** + * Brand marks by base block type. A connector id names the same product as its + * integration block (`confluence`, `google_drive`), so the block's mark serves + * the chip — through the catalog icon map rather than the connector registry, + * which would drag seventy connector modules into every surface that renders + * chat. Versioned catalog types (`gmail_v2`) collapse onto their base name. + */ +const BRAND_ICON_BY_BASE_TYPE: ReadonlyMap = new Map( + Object.entries(blockTypeToIconMap).map(([type, icon]) => [stripVersionSuffix(type), icon]) +) + +/** Chip label: the site name the model supplied, else the URL's hostname without a `www.` prefix. */ +export function sourceLabel(source: SourceTagData): string { + const siteName = source.siteName?.trim() + if (siteName) return siteName + return (externalLinkHostname(source.url) ?? source.url).replace(/^www\./, '') +} + +interface SourceChipProps { + source: SourceTagData +} + +/** + * A cited document as a small round pill — the connector's brand mark or the + * site favicon, then the site name — used inline at the citation point and + * again in the footer strip. Built on the chip fill and hover tokens at a 20px + * height so it sits inside a line of prose; the 30px `Chip` is the wrong scale + * for a citation. Opens the document like any external link in the reply. + */ +export function SourceChip({ source }: SourceChipProps) { + const hostname = externalLinkHostname(source.url) + const ConnectorIcon = source.connectorType + ? BRAND_ICON_BY_BASE_TYPE.get(source.connectorType) + : undefined + + return ( + + + handleExternalLinkClick(event, source.url)} + className={cn( + 'not-prose inline-flex h-[20px] max-w-[220px] shrink-0 items-center gap-1 rounded-full px-1.5 align-middle text-[var(--text-body)] text-caption no-underline transition-colors', + chipFilledFillTokens, + chipHoverSurfaceClass + )} + > + {ConnectorIcon ? ( + + ) : hostname ? ( + + ) : null} + + + + + {source.title ? ( + + {source.title} + {source.url} + + ) : ( + {source.url} + )} + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts index 1603a0278c5..8964244b4eb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts @@ -13,6 +13,7 @@ export type { QuestionTagData, QuestionType, RuntimeSpecialTagName, + SourceTagData, UsageUpgradeAction, UsageUpgradeTagData, WorkspaceResourceTagData, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index 5063faecdba..afddf868ad3 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -1073,6 +1073,7 @@ describe('parser properties', () => { '{"reason":"monthly cap","action":"upgrade_plan","message":"You hit your limit."}', 'mothership-error': '{"message":"The tool call failed.","code":"E_TOOL"}', + source: '{"url":"https://docs.github.com/en/x","siteName":"GitHub Docs"}', } /** Renders nothing rather than a card, so it cannot carry a card invariant. */ @@ -1374,3 +1375,50 @@ describe('ordinary JSON in prose is never turned into a card', () => { expect(segments.some((s) => s.type === 'text')).toBe(true) }) }) + +describe('source tag', () => { + it('parses a complete source tag into a source segment', () => { + const { segments } = parseSpecialTags( + 'Remove them first. {"url":"https://docs.github.com/en/x","siteName":"GitHub Docs","title":"Blocking users"} Then block.', + false + ) + + expect(segments).toEqual([ + { type: 'text', content: 'Remove them first. ' }, + { + type: 'source', + data: { + url: 'https://docs.github.com/en/x', + siteName: 'GitHub Docs', + title: 'Blocking users', + }, + }, + { type: 'text', content: ' Then block.' }, + ]) + }) + + it('keeps adjacent source tags as separate segments', () => { + const { segments } = parseSpecialTags( + 'Done. {"url":"https://a.example/1"}{"url":"https://b.example/2"}', + false + ) + + expect(segments.filter((segment) => segment.type === 'source')).toHaveLength(2) + }) + + it('rejects a source without an absolute http(s) url', () => { + const { segments } = parseSpecialTags( + 'See {"url":"docs/internal.md","siteName":"Docs"}.', + false + ) + + expect(segments.some((segment) => segment.type === 'source')).toBe(false) + }) + + it('hides a half-arrived source opener while streaming', () => { + const { segments, hasPendingTag } = parseSpecialTags('Block them. ` tag: one document the reply drew on. The tag contract for + * search answers — the model emits it inline, right after the sentence, list + * item, or paragraph the document supports, as a JSON body: + * + * `{"url":"https://docs.github.com/…","siteName":"GitHub Docs"}` + * + * Each tag renders as its own small chip where it sits (adjacent tags are + * never collapsed into a count), and every distinct `url` in the message is + * repeated in the footer strip below the reply. + */ +export interface SourceTagData { + /** Canonical http(s) link to the referenced document. */ + url: string + /** Document title, shown on hover. */ + title?: string + /** + * Short chip label — the site or product the document lives in ("GitHub + * Docs", "Confluence"). Falls back to the URL's hostname. + */ + siteName?: string + /** + * Knowledge-base connector the document was synced through + * (`CONNECTOR_META_REGISTRY` key). Lends the chip the product's brand mark; + * without it the chip shows the site favicon. + */ + connectorType?: string +} + export type ContentSegment = | { type: 'text'; content: string } | { type: 'thinking'; content: string } @@ -325,6 +354,7 @@ export type ContentSegment = | { type: 'mothership-error'; data: MothershipErrorTagData } | { type: 'workspace_resource'; data: WorkspaceResourceTagData } | { type: 'question'; data: QuestionTagData } + | { type: 'source'; data: SourceTagData } export type RuntimeSpecialTagName = | 'thinking' @@ -334,6 +364,7 @@ export type RuntimeSpecialTagName = | 'file' | 'workspace_resource' | 'question' + | 'source' export interface ParsedSpecialContent { segments: ContentSegment[] @@ -348,6 +379,7 @@ const RUNTIME_SPECIAL_TAG_NAMES = [ 'file', 'workspace_resource', 'question', + 'source', ] as const /** @@ -363,6 +395,7 @@ export const SPECIAL_TAG_NAMES = [ 'mothership-error', 'workspace_resource', 'question', + 'source', ] as const function isOptionsItemData(value: unknown): value is OptionsItemData { @@ -523,6 +556,20 @@ function isMothershipErrorTagData(value: unknown): value is MothershipErrorTagDa ) } +/** Only an absolute http(s) URL can be linked; anything else is not a source. */ +function isHttpUrl(value: unknown): value is string { + return typeof value === 'string' && /^https?:\/\/\S+$/i.test(value.trim()) +} + +function isSourceTagData(value: unknown): value is SourceTagData { + if (!isRecordLike(value)) return false + if (!isHttpUrl(value.url)) return false + if (value.title !== undefined && typeof value.title !== 'string') return false + if (value.siteName !== undefined && typeof value.siteName !== 'string') return false + if (value.connectorType !== undefined && typeof value.connectorType !== 'string') return false + return true +} + function isWorkspaceResourceTagData(value: unknown): value is WorkspaceResourceTagData { if (!isRecordLike(value)) return false if ( @@ -713,6 +760,7 @@ function parseSpecialTagData( | { type: 'mothership-error'; data: MothershipErrorTagData } | { type: 'workspace_resource'; data: WorkspaceResourceTagData } | { type: 'question'; data: QuestionTagData } + | { type: 'source'; data: SourceTagData } | null { if (tagName === 'thinking') { const content = parseTextTagBody(body) @@ -744,6 +792,11 @@ function parseSpecialTagData( return data ? { type: 'workspace_resource', data } : null } + if (tagName === 'source') { + const data = parseJsonTagBody(body, isSourceTagData) + return data ? { type: 'source', data } : null + } + if (tagName === 'question') { const data = parseQuestionTagBody(body) if (data) return { type: 'question', data } @@ -1659,7 +1712,8 @@ interface SpecialTagsProps { /** * Unified renderer for inline special tags: ``, ``, ``, - * and ``. + * and ``. A `` never reaches here — the chat renderer + * folds it into the surrounding markdown as an inline chip. */ export function SpecialTags({ segment, @@ -1692,6 +1746,8 @@ export function SpecialTags({ return case 'workspace_resource': return + case 'source': + return null case 'question': return ( (blocks.length > 0 ? parseBlocks(blocks) : []), [blocks, blockOverlayVersion] ) + const sources = useMemo(() => collectMessageSources(blocks), [blocks]) const [trailingRevealing, setTrailingRevealing] = useState(false) const handleTrailingRevealChange = useCallback((revealing: boolean) => { @@ -1004,6 +1012,11 @@ function MessageContentInner({ return null } })} + {sources.length > 0 && ( +
+ +
+ )} {thinkingExpanded && isLast ? ( // Fixed-height placeholder for the NEXT piece of output: the shimmer diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts index f0a74e539a7..c9795080947 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts @@ -2,7 +2,8 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { deriveMessagePhase, resolveToolDisplayState } from './utils' +import { ContentBlockType } from '@/app/workspace/[workspaceId]/home/types' +import { collectMessageSources, deriveMessagePhase, resolveToolDisplayState } from './utils' describe('deriveMessagePhase', () => { it('is streaming whenever the transport is live', () => { @@ -36,3 +37,37 @@ describe('resolveToolDisplayState', () => { expect(resolveToolDisplayState('rejected')).toBe('icon') }) }) + +describe('collectMessageSources', () => { + const source = (url: string, extra = '') => `{"url":"${url}"${extra}}` + + it('collects every distinct source across the message text, in first-cited order', () => { + const blocks = [ + { + type: ContentBlockType.text, + content: `First point. ${source('https://a.example/1', ',"siteName":"A"')} Second. ${source('https://b.example/2')}`, + }, + { type: ContentBlockType.tool_call }, + { + type: ContentBlockType.text, + content: `Again. ${source('https://a.example/1')} New. ${source('https://c.example/3')}`, + }, + ] + + expect(collectMessageSources(blocks).map((entry) => entry.url)).toEqual([ + 'https://a.example/1', + 'https://b.example/2', + 'https://c.example/3', + ]) + expect(collectMessageSources(blocks)[0].siteName).toBe('A') + }) + + it('ignores subagent lanes and text without sources', () => { + const blocks = [ + { type: ContentBlockType.subagent_text, content: source('https://lane.example/x') }, + { type: ContentBlockType.text, content: 'Plain prose.' }, + ] + + expect(collectMessageSources(blocks)).toEqual([]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts index 64176c822c5..286d997c1d6 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts @@ -21,10 +21,37 @@ import { } from '@sim/emcn' import { Calendar, Clock, Cursor, Globe, Table as TableIcon } from '@sim/emcn/icons' import { AgentIcon, ImageIcon, TTSIcon, VideoIcon } from '@/components/icons' -import type { ToolCallStatus } from '@/app/workspace/[workspaceId]/home/types' +import { + parseSpecialTags, + type SourceTagData, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { + type ContentBlock, + ContentBlockType, + type ToolCallStatus, +} from '@/app/workspace/[workspaceId]/home/types' export type IconComponent = ComponentType> +/** + * Every distinct `` cited in the message's own prose, in first-cited + * order, for the footer strip. Only main-lane text counts: subagent lanes fold + * into agent groups rather than the answer, and a tool's output is not a + * citation. + */ +export function collectMessageSources(blocks: ContentBlock[]): SourceTagData[] { + const byUrl = new Map() + for (const block of blocks) { + if (block.type !== ContentBlockType.text || !block.content) continue + for (const segment of parseSpecialTags(block.content, false).segments) { + if (segment.type === 'source' && !byUrl.has(segment.data.url)) { + byUrl.set(segment.data.url, segment.data) + } + } + } + return [...byUrl.values()] +} + const TOOL_ICONS: Record = { mothership: Blimp, glob: FolderCode, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts new file mode 100644 index 00000000000..bacd4157759 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockRandomFloat } = vi.hoisted(() => ({ + mockRandomFloat: vi.fn<() => number>(), +})) + +vi.mock('@sim/utils/random', () => ({ randomFloat: mockRandomFloat })) + +vi.mock('@/lib/sim-search/connectors', () => { + const icon = () => null + const connector = (type: string, name: string, providerId: string) => ({ + type, + meta: { id: type, name, description: `Sync ${name}`, icon }, + providerId, + requiredScopes: [], + serviceName: name, + serviceIcon: icon, + blockType: type, + }) + return { + SEARCH_CONNECTORS: [ + connector('airtable', 'Airtable', 'airtable'), + connector('confluence', 'Confluence', 'confluence'), + connector('jira', 'Jira', 'jira'), + connector('jsm', 'Jira Service Management', 'jira'), + connector('notion', 'Notion', 'notion'), + connector('slack', 'Slack', 'slack'), + ], + } +}) + +import { computeConnectorActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions' + +describe('computeConnectorActions', () => { + beforeEach(() => { + /** A zero roll always samples the first remaining candidate, so the rotation is catalog order. */ + mockRandomFloat.mockReturnValue(0) + }) + + it('pins Confluence, Jira, and JSM first and fills the last slot from the rotation', () => { + const actions = computeConnectorActions(new Set()) + + expect(actions.map((action) => action.id)).toEqual([ + 'connect-confluence', + 'connect-jira', + 'connect-jsm', + 'connect-airtable', + ]) + expect(actions[0]).toMatchObject({ + kind: 'connector', + label: 'Connect Confluence', + target: { providerId: 'confluence', serviceName: 'Confluence' }, + }) + }) + + it('drops every connector on a connected provider and refills from the rotation', () => { + const actions = computeConnectorActions(new Set(['jira', 'airtable'])) + + expect(actions.map((action) => action.id)).toEqual([ + 'connect-confluence', + 'connect-notion', + 'connect-slack', + ]) + }) + + it('returns fewer than four rows once the rotation is exhausted', () => { + const actions = computeConnectorActions(new Set(['airtable', 'notion', 'slack'])) + + expect(actions.map((action) => action.id)).toEqual([ + 'connect-confluence', + 'connect-jira', + 'connect-jsm', + ]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts new file mode 100644 index 00000000000..360ccc8064e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts @@ -0,0 +1,50 @@ +import { SEARCH_CONNECTORS, type SearchConnector } from '@/lib/sim-search/connectors' +import type { Action } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/types' +import { weightedSample } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample' + +/** Rows shown in Search mode — the same count as the Build-mode set. */ +const CONNECTOR_ACTION_COUNT = 4 + +/** + * Connectors pinned to the head of the Search-mode list, in this order. Keys + * are `CONNECTOR_META_REGISTRY` keys, as on every {@link SearchConnector}. + */ +export const PINNED_CONNECTOR_TYPES = ['confluence', 'jira', 'jsm'] as const + +const PINNED_TYPES: ReadonlySet = new Set(PINNED_CONNECTOR_TYPES) + +/** The pinned connectors, in pinned order. */ +const PINNED: readonly SearchConnector[] = PINNED_CONNECTOR_TYPES.flatMap((type) => { + const connector = SEARCH_CONNECTORS.find((candidate) => candidate.type === type) + return connector ? [connector] : [] +}) + +/** Every other Sim Search connector — the pool the remaining slots rotate through. */ +const ROTATING: readonly SearchConnector[] = SEARCH_CONNECTORS.filter( + (connector) => !PINNED_TYPES.has(connector.type) +) + +function toConnectorAction(connector: SearchConnector): Action { + return { + kind: 'connector', + id: `connect-${connector.type}`, + label: `Connect ${connector.meta.name}`, + icon: connector.meta.icon, + target: connector, + } +} + +/** + * Builds the Search-mode rows: the pinned connectors first, then a uniform + * sample of the rest to fill four slots. A connector whose provider the viewer + * has already connected is dropped from both halves — so Jira and Jira Service + * Management, which share one provider, leave together — and a pinned slot + * freed that way is taken by the rotation. + */ +export function computeConnectorActions(connectedProviderIds: ReadonlySet): Action[] { + const isConnected = (connector: SearchConnector) => connectedProviderIds.has(connector.providerId) + const pinned = PINNED.filter((connector) => !isConnected(connector)) + const pool = ROTATING.filter((connector) => !isConnected(connector)) + const rotating = weightedSample(pool, CONNECTOR_ACTION_COUNT - pinned.length, () => 1) + return [...pinned, ...rotating].map(toConnectorAction) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx new file mode 100644 index 00000000000..e7446072101 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx @@ -0,0 +1,150 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCaptureEvent, mockUseSearchCredentials } = vi.hoisted(() => ({ + mockCaptureEvent: vi.fn(), + mockUseSearchCredentials: vi.fn(), +})) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) +vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) +vi.mock('@sim/utils/random', () => ({ randomFloat: () => 0 })) + +vi.mock('@/hooks/queries/credentials', () => ({ + useWorkspaceCredentials: () => ({ data: [] }), +})) +vi.mock('@/hooks/queries/oauth/oauth-connections', () => ({ + useOAuthConnections: () => ({ data: [] }), +})) +vi.mock('@/hooks/queries/tables', () => ({ + useTablesList: () => ({ data: [] }), +})) +vi.mock('@/hooks/queries/kb/knowledge', () => ({ + useKnowledgeBasesQuery: () => ({ data: [] }), +})) +vi.mock('@/app/workspace/[workspaceId]/search/hooks/use-search-credentials', () => ({ + useSearchCredentials: mockUseSearchCredentials, +})) + +/** The Build-mode pool is built from the block catalog at module load; an empty catalog keeps it to the table starters. */ +vi.mock('@/blocks/registry', () => ({ getAllBlockMeta: () => ({}), getAllBlocks: () => [] })) + +vi.mock('@/lib/sim-search/connectors', () => { + const icon = () => null + const connector = (type: string, name: string, providerId: string) => ({ + type, + meta: { id: type, name, description: `Sync ${name}`, icon }, + providerId, + requiredScopes: ['read'], + serviceName: name, + serviceIcon: icon, + blockType: type, + }) + return { + SEARCH_CONNECTORS: [ + connector('airtable', 'Airtable', 'airtable'), + connector('confluence', 'Confluence', 'confluence'), + connector('jira', 'Jira', 'jira'), + connector('jsm', 'Jira Service Management', 'jira'), + connector('notion', 'Notion', 'notion'), + ], + } +}) + +vi.mock('@/app/workspace/[workspaceId]/components/connect-oauth-modal', () => ({ + ConnectOAuthModal: ({ open, providerId }: { open: boolean; providerId: string }) => + open ?
{providerId}
: null, +})) + +import { SuggestedActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions' +import { useMothershipModeStore } from '@/stores/mothership-mode/store' + +let root: Root | null = null +let container: HTMLDivElement | null = null +const onSelectPrompt = vi.fn() + +function mount() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function heading(): string { + return container?.querySelector('button[aria-expanded] span')?.textContent ?? '' +} + +function rows(): HTMLButtonElement[] { + return Array.from( + container?.querySelectorAll('button:not([aria-expanded])') ?? [] + ) +} + +function connectModal(): string | null { + return document.querySelector('[data-testid="connect-modal"]')?.textContent ?? null +} + +beforeEach(() => { + onSelectPrompt.mockClear() + mockCaptureEvent.mockClear() + mockUseSearchCredentials.mockReturnValue({ + credentials: [{ id: 'cred-jira', providerId: 'jira' }], + isPending: false, + }) + useMothershipModeStore.getState().reset() +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('SuggestedActions', () => { + it('shows the Build starters by default', () => { + mount() + + expect(heading()).toBe('Suggested actions') + expect(rows().map((row) => row.textContent)).toContain('Integrate with Slack') + }) + + it('swaps to the connector list in Search mode, minus providers already connected', () => { + mount() + + act(() => useMothershipModeStore.getState().setMode('search')) + + expect(heading()).toBe('Connect Sim Search') + expect(rows().map((row) => row.textContent)).toEqual([ + 'Connect Confluence', + 'Connect Airtable', + 'Connect Notion', + ]) + }) + + it('opens the OAuth connect modal for a connector row instead of populating the input', () => { + mount() + act(() => useMothershipModeStore.getState().setMode('search')) + expect(connectModal()).toBeNull() + + act(() => { + rows()[0].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) + + expect(connectModal()).toBe('confluence') + expect(onSelectPrompt).not.toHaveBeenCalled() + expect(mockCaptureEvent).toHaveBeenCalledWith( + null, + 'suggested_action_clicked', + expect.objectContaining({ kind: 'connector', action_id: 'connect-confluence', position: 0 }) + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx index 32f878c1a27..ac16d620a49 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx @@ -1,21 +1,27 @@ 'use client' -import { type ComponentType, type CSSProperties, useMemo, useState } from 'react' +import { useMemo, useState } from 'react' import { ArrowRight, ChevronDown, cn, Expandable, ExpandableContent } from '@sim/emcn' import { Table } from '@sim/emcn/icons' -import { randomFloat } from '@sim/utils/random' import { stripVersionSuffix } from '@sim/utils/string' import { useParams } from 'next/navigation' import { usePostHog } from 'posthog-js/react' import { GmailIcon, SlackIcon } from '@/components/icons' import { INTEGRATIONS, - type OAuthServiceMatch, resolveOAuthServiceForIntegration, resolveOAuthServiceForSlug, } from '@/lib/integrations' import { captureEvent } from '@/lib/posthog/client' import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' +import { computeConnectorActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions' +import type { + Action, + ActionIcon, + OAuthConnectTarget, +} from '@/app/workspace/[workspaceId]/home/components/suggested-actions/types' +import { weightedSample } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample' +import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' import { BrandIcon } from '@/blocks/brand-icon' import { getAllBlockMeta } from '@/blocks/registry' import type { ModuleTag } from '@/blocks/types' @@ -23,12 +29,7 @@ import { useWorkspaceCredentials } from '@/hooks/queries/credentials' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useOAuthConnections } from '@/hooks/queries/oauth/oauth-connections' import { useTablesList } from '@/hooks/queries/tables' - -type Icon = ComponentType<{ className?: string; style?: CSSProperties }> - -type Action = - | { kind: 'prompt'; id: string; label: string; prompt: string; icon: Icon } - | { kind: 'integration'; id: string; label: string; icon: Icon; slug: string } +import { type MothershipMode, useMothershipModeStore } from '@/stores/mothership-mode/store' /** Lookup integration slug by OAuth service display name (case-insensitive). */ const SLUG_BY_LOWER_NAME: ReadonlyMap = new Map( @@ -51,7 +52,7 @@ interface Candidate { blockType: string label: string prompt: string - icon: Icon + icon: ActionIcon modules: readonly ModuleTag[] featured: boolean popular: boolean @@ -101,7 +102,7 @@ const CANDIDATES: readonly Candidate[] = (() => { blockType, label: template.title, prompt: template.prompt, - icon: template.icon as Icon, + icon: template.icon as ActionIcon, modules: template.modules, featured: template.featured ?? false, popular: template.category === 'popular', @@ -147,34 +148,13 @@ function scoreCandidate(c: Candidate, signals: Signals): number { return weight } -/** - * Weighted sampling without replacement. Each pick's probability is - * proportional to its weight, so the set stays varied while staying relevant. - */ -function weightedSample(pool: readonly T[], n: number, weightOf: (item: T) => number): T[] { - const remaining = pool.map((item) => ({ item, weight: Math.max(weightOf(item), 0) })) - const out: T[] = [] - while (out.length < n && remaining.length > 0) { - const total = remaining.reduce((sum, entry) => sum + entry.weight, 0) - if (total <= 0) break - let roll = randomFloat() * total - const index = remaining.findIndex((entry) => { - roll -= entry.weight - return roll <= 0 - }) - const [picked] = remaining.splice(index === -1 ? remaining.length - 1 : index, 1) - out.push(picked.item) - } - return out -} - const EMPTY_CREDENTIALS: NonNullable['data']> = [] const EMPTY_SERVICES: NonNullable['data']> = [] type ServiceInfo = NonNullable['data']>[number] function toPromptAction(c: Candidate): Action { - return { kind: 'prompt', id: c.id, label: c.label, prompt: c.prompt, icon: c.icon } + return { kind: 'prompt', id: c.id, label: c.label, icon: c.icon, prompt: c.prompt } } function toIntegrationAction(service: ServiceInfo, slug: string): Action { @@ -251,6 +231,12 @@ const INITIAL_ACTIONS: Action[] = [ .map(toPromptAction), ] +/** Section heading per composer mode — Search reads as a connect-your-sources list. */ +const HEADINGS: Record = { + build: 'Suggested actions', + search: 'Connect Sim Search', +} + interface SuggestedActionsProps { onSelectPrompt: (prompt: string) => void } @@ -258,6 +244,7 @@ interface SuggestedActionsProps { export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { const { workspaceId } = useParams<{ workspaceId: string }>() const posthog = usePostHog() + const mode = useMothershipModeStore((state) => state.mode) const { data: credentials = EMPTY_CREDENTIALS } = useWorkspaceCredentials({ workspaceId, @@ -268,6 +255,8 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { const { data: knowledgeBases = [] } = useKnowledgeBasesQuery(workspaceId, { enabled: Boolean(workspaceId), }) + const { credentials: searchCredentials, isPending: searchCredentialsPending } = + useSearchCredentials(workspaceId) const [expanded, setExpanded] = useState(true) /** @@ -282,7 +271,7 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { * to `null` (via `onOpenChange(false)`) closes it. Mirrors the local-state * pattern used by the integrations detail page. */ - const [oauthTarget, setOAuthTarget] = useState(null) + const [oauthTarget, setOAuthTarget] = useState(null) const connectedProviders = useMemo( () => @@ -304,17 +293,39 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { [connectedProviders, tables.length, knowledgeBases.length] ) + const connectedSearchProviders = useMemo( + () => + new Set( + searchCredentials + .map((credential) => credential.providerId) + .filter((providerId): providerId is string => Boolean(providerId)) + ), + [searchCredentials] + ) + /** - * Personalized suggestions, re-sampled whenever signals resolve. Falls back to + * Each mode's list is memoized on its own inputs alone, so switching modes — + * or the other mode's signals settling — never re-samples it. + * + * Search lists connectors to attach, and waits for the viewer's credentials: + * sampling against an empty set would list connected providers and then + * reshuffle when the query lands. Build lists personalized suggestions, + * re-sampled whenever signals resolve, and falls back to * {@link INITIAL_ACTIONS} until the credential and service queries have loaded * — and stays there for users with no connections — so first paint never - * flashes. + * flashes. The store's default mode is Build, so the server render never + * shows the sampled Search list. */ - const actions = useMemo(() => { + const searchActions = useMemo( + () => (searchCredentialsPending ? [] : computeConnectorActions(connectedSearchProviders)), + [searchCredentialsPending, connectedSearchProviders] + ) + const buildActions = useMemo(() => { const personalized = services.length > 0 && connectedProviders.size > 0 if (!personalized) return INITIAL_ACTIONS return computeActions(services, signals) }, [connectedProviders, services, signals]) + const actions = mode === 'search' ? searchActions : buildActions const handleSelect = (action: Action, position: number) => { captureEvent(posthog, 'suggested_action_clicked', { @@ -329,8 +340,9 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { onSelectPrompt(action.prompt) return } - const match = resolveOAuthServiceForSlug(action.slug) - if (match) setOAuthTarget(match) + const target = + action.kind === 'connector' ? action.target : resolveOAuthServiceForSlug(action.slug) + if (target) setOAuthTarget(target) } const handleToggleExpanded = () => { @@ -351,7 +363,7 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { aria-expanded={expanded} className='group/toggle flex w-full cursor-pointer items-center gap-2' > - Suggested actions + {HEADINGS[mode]} {/* * Revealed by hovering anywhere in the section — the group sits on the * section wrapper rather than this row, so the action rows below arm it just diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts new file mode 100644 index 00000000000..98c51898c9c --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts @@ -0,0 +1,22 @@ +import type { ComponentType, CSSProperties } from 'react' + +export type ActionIcon = ComponentType<{ className?: string; style?: CSSProperties }> + +/** What the OAuth connect modal needs to start a connection for one service. */ +export interface OAuthConnectTarget { + providerId: string + requiredScopes: readonly string[] + serviceName: string + serviceIcon: ComponentType<{ className?: string }> +} + +/** + * One suggested-action row. `prompt` rows populate the input with a curated + * prompt; `integration` rows resolve their OAuth service from the catalog slug + * on click; `connector` rows — the Search-mode "Connect X" rows — carry their + * connect target directly. Both connecting kinds open the OAuth connect modal. + */ +export type Action = + | { kind: 'prompt'; id: string; label: string; icon: ActionIcon; prompt: string } + | { kind: 'integration'; id: string; label: string; icon: ActionIcon; slug: string } + | { kind: 'connector'; id: string; label: string; icon: ActionIcon; target: OAuthConnectTarget } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample.ts new file mode 100644 index 00000000000..b8b6c66607a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample.ts @@ -0,0 +1,27 @@ +import { randomFloat } from '@sim/utils/random' + +/** + * Weighted sampling without replacement. Each pick's probability is + * proportional to its weight, so the set stays varied while staying relevant. + * A constant weight yields a uniform sample. + */ +export function weightedSample( + pool: readonly T[], + n: number, + weightOf: (item: T) => number +): T[] { + const remaining = pool.map((item) => ({ item, weight: Math.max(weightOf(item), 0) })) + const out: T[] = [] + while (out.length < n && remaining.length > 0) { + const total = remaining.reduce((sum, entry) => sum + entry.weight, 0) + if (total <= 0) break + let roll = randomFloat() * total + const index = remaining.findIndex((entry) => { + roll -= entry.weight + return roll <= 0 + }) + const [picked] = remaining.splice(index === -1 ? remaining.length - 1 : index, 1) + out.push(picked.item) + } + return out +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts index 95d472588c1..7d8bdca03af 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts @@ -24,6 +24,7 @@ export { export { DropOverlay } from './drop-overlay' export { MicButton } from './mic-button' export { MicrophonePermissionHelp } from './microphone-permission-help' +export { ModeSwitcher } from './mode-switcher' export { PlusMenuDropdown } from './plus-menu-dropdown' export type { PromptEditorInstance, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/index.ts new file mode 100644 index 00000000000..46800468812 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/index.ts @@ -0,0 +1 @@ +export { ModeSwitcher } from './mode-switcher' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx new file mode 100644 index 00000000000..f44dbb101a5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx @@ -0,0 +1,110 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCaptureEvent } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn() })) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) +vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) + +import { ModeSwitcher } from '@/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher' +import { useMothershipModeStore } from '@/stores/mothership-mode/store' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function trigger(): HTMLButtonElement { + const node = container?.querySelector('button') + if (!node) throw new Error('Switcher trigger did not render') + return node +} + +/** Opens the menu the way a pointer does — Radix opens on `pointerdown`. */ +function openMenu() { + act(() => { + trigger().dispatchEvent(new MouseEvent('pointerdown', { bubbles: true, button: 0 })) + }) +} + +function items(): HTMLElement[] { + return Array.from(document.querySelectorAll('[role="menuitem"]')) +} + +beforeEach(() => { + mockCaptureEvent.mockClear() + useMothershipModeStore.getState().reset() +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('ModeSwitcher', () => { + it('renders the active mode as a label-only round chip and defaults to Build', () => { + mount() + + const button = trigger() + expect(button.textContent).toBe('Build') + expect(button.getAttribute('aria-label')).toBe('Mode: Build') + expect(button.className).toContain('h-[30px]') + expect(button.className).toContain('rounded-full') + expect(button.className).not.toContain('rounded-lg') + expect(button.className).toContain('hover-hover:bg-[var(--surface-hover)]') + expect(button.querySelector('svg')).toBeNull() + }) + + it('lists both modes and checks the active one', () => { + mount() + openMenu() + + const rows = items() + expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Search']) + expect(rows[0].querySelector('svg')).not.toBeNull() + expect(rows[1].querySelector('svg')).toBeNull() + }) + + it('switches the shared mode and reports the change', () => { + mount() + openMenu() + + act(() => { + items()[1].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) + + expect(useMothershipModeStore.getState().mode).toBe('search') + expect(trigger().textContent).toBe('Search') + expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_mode_changed', { + workspace_id: 'workspace-1', + mode: 'search', + }) + }) + + it('does not report re-selecting the active mode', () => { + mount() + openMenu() + + act(() => { + items()[0].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) + + expect(useMothershipModeStore.getState().mode).toBe('build') + expect(mockCaptureEvent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx new file mode 100644 index 00000000000..60d58ada236 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -0,0 +1,63 @@ +'use client' + +import { memo } from 'react' +import { + Chip, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuItemLabel, + DropdownMenuTrigger, +} from '@sim/emcn' +import { Check } from '@sim/emcn/icons' +import { useParams } from 'next/navigation' +import { usePostHog } from 'posthog-js/react' +import { captureEvent } from '@/lib/posthog/client' +import { + MOTHERSHIP_MODES, + type MothershipMode, + useMothershipModeStore, +} from '@/stores/mothership-mode/store' + +const MODE_LABELS: Record = { + build: 'Build', + search: 'Search', +} + +/** + * The composer's Build / Search switcher: a label-only `Chip` in its `round` + * shape — chip chrome throughout (`--text-body` label, `--surface-hover` on + * hover, no text-color shift), fully round to sit in the toolbar's row of + * round controls — opening a two-row menu that checks the active mode, as + * `ChipDropdown` does. + */ +export const ModeSwitcher = memo(function ModeSwitcher() { + const { workspaceId } = useParams<{ workspaceId: string }>() + const posthog = usePostHog() + const mode = useMothershipModeStore((state) => state.mode) + const setMode = useMothershipModeStore((state) => state.setMode) + + const handleSelect = (next: MothershipMode) => { + if (next === mode) return + setMode(next) + captureEvent(posthog, 'chat_mode_changed', { workspace_id: workspaceId, mode: next }) + } + + return ( + + + + {MODE_LABELS[mode]} + + + + {MOTHERSHIP_MODES.map((option) => ( + handleSelect(option)}> + + {option === mode && } + + ))} + + + ) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 13c23d419cf..3b0bb976a7b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -25,6 +25,7 @@ import { DropOverlay, MicButton, MicrophonePermissionHelp, + ModeSwitcher, PromptEditor, SendButton, usePromptEditor, @@ -512,8 +513,12 @@ const UserInputImpl = forwardRef(function UserI return () => window.cancelAnimationFrame(raf) }, [textareaRef]) + /** + * Menu rows are excluded alongside buttons: the mode switcher's items are + * portaled, so their clicks still bubble here through the React tree. + */ const handleContainerClick = (e: React.MouseEvent) => { - if ((e.target as HTMLElement).closest('button, [role="dialog"]')) return + if ((e.target as HTMLElement).closest('button, [role="dialog"], [role="menu"]')) return textareaRef.current?.focus() } @@ -678,6 +683,7 @@ const UserInputImpl = forwardRef(function UserI
+ {isSttSupported && ( +}) { + const { workspaceId, credentialId } = await params + return +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx b/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx new file mode 100644 index 00000000000..8f00ee19c60 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx @@ -0,0 +1,240 @@ +'use client' + +import { useState } from 'react' +import { + Chip, + ChipConfirmModal, + ChipCopyInput, + ChipInput, + ChipLink, + ChipTextarea, + cn, + toast, +} from '@sim/emcn' +import { ArrowLeft } from '@sim/emcn/icons' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { useRouter } from 'next/navigation' +import { SaveDiscardChips } from '@/components/settings/save-discard-actions' +import { writeOAuthReturnContext } from '@/lib/credentials/client-state' +import { resolveCredentialDisplay } from '@/lib/integrations' +import { + CredentialDetailHeading, + CredentialDetailLayout, + DetailSection, + UnsavedChangesModal, + useCredentialDetailForm, +} from '@/app/workspace/[workspaceId]/components/credential-detail' +import { + RESOURCE_TILE_BASE, + RESOURCE_TILE_PLAIN, +} from '@/app/workspace/[workspaceId]/components/resource-tile' +import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' +import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' +import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' +import { useCreateCredentialDraft, useDeleteWorkspaceCredential } from '@/hooks/queries/credentials' +import { useConnectOAuthService } from '@/hooks/queries/oauth/oauth-connections' +import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' + +const logger = createLogger('SearchCredentialDetail') + +interface SearchCredentialDetailProps { + workspaceId: string + credentialId: string +} + +/** + * A connected Sim Search credential: the integrations connected-credential page + * without its sharing. Connections here are personal, so there is no Share + * action and no members section — the credential resolves only when the viewer + * created it, and anyone else lands on "not found". Reconnect and Disconnect + * go through the same credential draft and delete paths as integrations. + */ +export function SearchCredentialDetail({ workspaceId, credentialId }: SearchCredentialDetailProps) { + const router = useRouter() + const searchHref = `/workspace/${workspaceId}/search` + + useOAuthReturnRouter() + + const { credentials, isPending: credentialsLoading } = useSearchCredentials(workspaceId) + const connectOAuthService = useConnectOAuthService() + const createDraft = useCreateCredentialDraft() + const deleteCredential = useDeleteWorkspaceCredential() + + const credential = credentials.find((c) => c.id === credentialId) ?? null + const isAdmin = credential?.role === 'admin' + + const [showDeleteConfirmDialog, setShowDeleteConfirmDialog] = useState(false) + + const form = useCredentialDetailForm({ credential, isAdmin, backHref: searchHref }) + + const display = credential ? resolveCredentialDisplay(credential) : null + + const handleReconnect = async () => { + if (!credential?.providerId) return + try { + const draft = await createDraft.mutateAsync({ + workspaceId, + providerId: credential.providerId, + displayName: credential.displayName, + description: credential.description || undefined, + credentialId: credential.id, + }) + writeOAuthReturnContext({ + origin: 'integrations', + displayName: credential.displayName, + providerId: credential.providerId, + preCount: credentials.filter((c) => c.providerId === credential.providerId).length, + workspaceId, + reconnect: true, + requestedAt: Date.now(), + }) + await connectOAuthService.mutateAsync({ + providerId: credential.providerId, + callbackURL: window.location.href, + draftId: draft.draftId, + }) + } catch (error: unknown) { + toast.error("Couldn't start reconnect", { + description: getErrorMessage(error, 'Please try again in a moment.'), + }) + logger.error('Failed to reconnect Sim Search credential', error) + } + } + + const handleConfirmDelete = async () => { + if (!credential) return + try { + await deleteCredential.mutateAsync(credential.id) + setShowDeleteConfirmDialog(false) + router.push(searchHref) + } catch (error) { + toast.error("Couldn't disconnect", { + description: getErrorMessage(error, 'Please try again in a moment.'), + }) + logger.error('Failed to disconnect Sim Search credential', error) + } + } + + const back = ( + + Search + + ) + + const actions = + credential && isAdmin ? ( + <> + + Reconnect + + setShowDeleteConfirmDialog(true)} + disabled={deleteCredential.isPending} + > + Disconnect + + + + ) : null + + if (credentialsLoading && !credential) { + return ( + + Loading… + + ) + } + + if (!credential) { + return ( + + Credential not found. + + ) + } + + return ( + <> + + + ) : ( +
+ + {credential.displayName.slice(0, 1) || '?'} + +
+ ) + } + title={display?.detailTitle ?? credential.displayName} + subtitle={display?.detailSubtitle ?? 'Connected service'} + /> + + + + + + + form.setDisplayNameDraft(event.target.value)} + autoComplete='off' + data-lpignore='true' + disabled={!isAdmin} + /> + + + + form.setDescriptionDraft(event.target.value)} + placeholder='Add a description...' + maxLength={500} + autoComplete='off' + data-lpignore='true' + disabled={!isAdmin} + /> + +
+ + + + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/error.tsx b/apps/sim/app/workspace/[workspaceId]/search/error.tsx new file mode 100644 index 00000000000..d4520d2d64f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function SearchError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts b/apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts new file mode 100644 index 00000000000..c298ca4e737 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts @@ -0,0 +1,48 @@ +'use client' + +import { useMemo } from 'react' +import { useSession } from '@/lib/auth/auth-client' +import { isSearchConnectorProvider } from '@/lib/sim-search/connectors' +import { useWorkspaceCredentials, type WorkspaceCredential } from '@/hooks/queries/credentials' + +const EMPTY_CREDENTIALS: readonly WorkspaceCredential[] = [] + +interface UseSearchCredentialsResult { + /** The viewer's own OAuth credentials for Sim Search connector providers. */ + credentials: readonly WorkspaceCredential[] + isPending: boolean +} + +/** + * The credentials the Sim Search surface shows as connected. Sim Search + * connections are personal: the surface lists only credentials the viewer + * created, so a workspace admin — who can see every shared credential — still + * sees just their own here. Service accounts are excluded, as they are from the + * knowledge-base connector picker: no connector can authenticate with one. + * + * Reads the same unfiltered workspace credential query the integrations pages + * use, so the two surfaces share one cache and one invalidation path. + */ +export function useSearchCredentials(workspaceId: string): UseSearchCredentialsResult { + const { data: session, isPending: sessionPending } = useSession() + const userId = session?.user?.id + const { data: allCredentials, isPending: credentialsPending } = useWorkspaceCredentials({ + workspaceId, + enabled: Boolean(workspaceId), + }) + + const credentials = useMemo(() => { + if (!userId || !allCredentials) return EMPTY_CREDENTIALS + return allCredentials.filter( + (credential) => + credential.type === 'oauth' && + credential.createdBy === userId && + isSearchConnectorProvider(credential.providerId) + ) + }, [allCredentials, userId]) + + return { + credentials, + isPending: sessionPending || (Boolean(workspaceId) && credentialsPending), + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/page.tsx b/apps/sim/app/workspace/[workspaceId]/search/page.tsx new file mode 100644 index 00000000000..1623fd074e2 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/page.tsx @@ -0,0 +1,30 @@ +import { Suspense } from 'react' +import type { Metadata } from 'next' +import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/components' +import { Search } from '@/app/workspace/[workspaceId]/search/search' + +export const metadata: Metadata = { + title: 'Search', +} + +/** + * Sim Search page entry. `Search` reads URL query params via nuqs (which uses + * `useSearchParams` internally), so it must sit under a Suspense boundary. The + * fallback renders the real page chrome (background + tab header) so a suspend + * never shows a blank frame. + */ +export default async function SearchPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + return ( + + +
+ } + > + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/search-params.ts b/apps/sim/app/workspace/[workspaceId]/search/search-params.ts new file mode 100644 index 00000000000..c55f7709913 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/search-params.ts @@ -0,0 +1,17 @@ +import { parseAsString } from 'nuqs/server' + +/** + * `search` filters the Sim Search connector list by name and description. The + * input is controlled directly by the instant nuqs value; only its URL write is + * debounced via `useDebouncedSearchSetter` — never written on every keystroke. + */ +export const connectorSearchParam = { + key: 'search', + parser: parseAsString.withDefault(''), +} as const + +/** Search is filter view-state: clean URLs, no back-stack churn. */ +export const connectorSearchUrlKeys = { + history: 'replace', + clearOnDefault: true, +} as const diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx new file mode 100644 index 00000000000..37cd5cf59b0 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx @@ -0,0 +1,176 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('nuqs', () => ({ + useQueryState: () => ['', vi.fn()], +})) +vi.mock('@/hooks/use-debounced-search-setter', () => ({ + useDebouncedSearchSetter: (write: (value: string) => void) => write, +})) +vi.mock('@/lib/auth/auth-client', () => ({ + useSession: () => ({ data: { user: { id: 'user-1' } }, isPending: false }), +})) +vi.mock('@/hooks/use-permission-config', () => ({ + usePermissionConfig: () => ({ + integrationAvailability: new Map([ + ['confluence', { state: 'ready', oauthAvailable: true }], + /* A service-account-only deployment: the block is usable, the OAuth path is not. */ + ['slack', { state: 'limited', oauthAvailable: false }], + ]), + }), +})) +vi.mock('@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration', () => ({ + useScrollRestoration: () => {}, +})) +vi.mock('@/hooks/use-oauth-return', () => ({ + useOAuthReturnRouter: () => {}, +})) +vi.mock('@/app/workspace/[workspaceId]/components', () => ({ + IntegrationTabsHeader: () => null, +})) +vi.mock('@/app/workspace/[workspaceId]/components/connect-oauth-modal', () => ({ + ConnectOAuthModal: ({ open, providerId }: { open: boolean; providerId: string }) => + open ?
{providerId}
: null, +})) +vi.mock('@/blocks', () => ({ getBlock: () => undefined })) +vi.mock('@/lib/integrations', () => ({ + resolveCredentialDisplay: () => ({ icon: () => null, blockType: 'confluence', subtitle: 'Sub' }), +})) + +vi.mock('@/lib/sim-search/connectors', () => { + const icon = () => null + const connector = (type: string, name: string, description: string) => ({ + type, + meta: { id: type, name, description, icon }, + providerId: type, + requiredScopes: [], + serviceName: name, + serviceIcon: icon, + blockType: type, + }) + const providers = new Set(['confluence', 'jira', 'slack']) + return { + SEARCH_CONNECTORS: [ + connector('confluence', 'Confluence', 'Sync Confluence pages'), + connector('jira', 'Jira', 'Sync Jira issues'), + connector('slack', 'Slack', 'Sync Slack messages'), + ], + isSearchConnectorProvider: (providerId: string | null) => + providerId !== null && providers.has(providerId), + } +}) + +const credential = (overrides: Record) => ({ + id: 'cred', + workspaceId: 'workspace-1', + type: 'oauth', + displayName: 'Credential', + description: null, + unredacted: false, + providerId: 'confluence', + accountId: null, + envKey: null, + envOwnerUserId: null, + createdBy: 'user-1', + createdAt: '', + updatedAt: '', + role: 'admin', + ...overrides, +}) + +vi.mock('@/hooks/queries/credentials', () => ({ + useWorkspaceCredentials: () => ({ + isPending: false, + data: [ + credential({ id: 'cred-mine', displayName: 'My Confluence' }), + credential({ + id: 'cred-theirs', + displayName: 'Teammate Jira', + providerId: 'jira', + createdBy: 'user-2', + }), + credential({ id: 'cred-sa', displayName: 'Service Account', type: 'service_account' }), + credential({ id: 'cred-github', displayName: 'My GitHub', providerId: 'github' }), + ], + }), +})) + +import { Search } from '@/app/workspace/[workspaceId]/search/search' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function sectionLabels(): string[] { + return Array.from(container?.querySelectorAll('section > div > span') ?? []).map( + (node) => node.textContent ?? '' + ) +} + +function hrefs(): Array { + return Array.from(container?.querySelectorAll('a') ?? []).map((a) => a.getAttribute('href')) +} + +function connectButton(name: string): HTMLButtonElement | null { + return container?.querySelector(`button[aria-label="Connect ${name}"]`) ?? null +} + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('Search', () => { + it('lists the viewer’s own search-connector credentials under Connected', () => { + mount() + + expect(sectionLabels()).toEqual(['Connected', 'Sim Search Connectors']) + const text = container?.textContent ?? '' + expect(text).toContain('My Confluence') + expect(text).not.toContain('Teammate Jira') + expect(text).not.toContain('Service Account') + expect(text).not.toContain('My GitHub') + expect(hrefs()).toContain('/workspace/workspace-1/search/connected/cred-mine') + }) + + it('opens the connect modal for a connector instead of navigating', () => { + mount() + + const connect = connectButton('Confluence') + expect(connect).not.toBeNull() + expect(hrefs()).not.toContain('/workspace/workspace-1/search/confluence') + expect(document.querySelector('[data-testid="connect-modal"]')).toBeNull() + + act(() => { + connect?.dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) + + expect(document.querySelector('[data-testid="connect-modal"]')?.textContent).toBe('confluence') + }) + + it('disables a connector whose OAuth path is unavailable, even when the block is usable', () => { + mount() + + expect(connectButton('Jira')).not.toBeNull() + expect(connectButton('Slack')).toBeNull() + expect(container?.textContent).toContain( + 'Unavailable in this deployment. Contact your administrator.' + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx new file mode 100644 index 00000000000..a6ff812c978 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -0,0 +1,210 @@ +'use client' + +import { useRef, useState } from 'react' +import { ChipInput } from '@sim/emcn' +import { Search as SearchIcon } from '@sim/emcn/icons' +import { useParams } from 'next/navigation' +import { useQueryState } from 'nuqs' +import { resolveCredentialDisplay } from '@/lib/integrations' +import { SEARCH_CONNECTORS, type SearchConnector } from '@/lib/sim-search/connectors' +import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/components' +import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' +import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' +import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' +import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' +import { CONNECTED_LABEL } from '@/app/workspace/[workspaceId]/integrations/search-params' +import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' +import { + connectorSearchParam, + connectorSearchUrlKeys, +} from '@/app/workspace/[workspaceId]/search/search-params' +import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' +import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import type { WorkspaceCredential } from '@/hooks/queries/credentials' +import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' +import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' +import { usePermissionConfig } from '@/hooks/use-permission-config' + +const CONNECTORS_LABEL = 'Sim Search Connectors' + +interface ConnectorItemProps { + connector: SearchConnector + unavailable: boolean + onConnect: (connector: SearchConnector) => void +} + +/** + * A connector row acts in place — it opens the connect modal — so it carries no + * navigation chevron; only a connected credential leads to a page of its own. + */ +function ConnectorItem({ connector, unavailable, onConnect }: ConnectorItemProps) { + return ( + } + title={connector.meta.name} + description={ + unavailable + ? 'Unavailable in this deployment. Contact your administrator.' + : connector.meta.description + } + onClick={unavailable ? undefined : () => onConnect(connector)} + clickLabel={`Connect ${connector.meta.name}`} + disabled={unavailable} + /> + ) +} + +interface ConnectedItemProps { + credential: WorkspaceCredential + workspaceId: string +} + +function ConnectedItem({ credential, workspaceId }: ConnectedItemProps) { + const display = resolveCredentialDisplay(credential) + if (!display.icon) return null + return ( + } + title={credential.displayName} + description={credential.description || display.subtitle} + href={`/workspace/${workspaceId}/search/connected/${credential.id}`} + clickLabel={`Open ${credential.displayName}`} + navigable + /> + ) +} + +/** + * The Sim Search connector catalog: the viewer's own connections first, then + * every connector a personal OAuth connection can power. Same shell and rows + * as the Integrations page, minus its showcase and category filter — the + * connector set is small enough that the search box alone narrows it. A + * connector row opens the OAuth connect modal right here; the OAuth redirect + * lands back on this page, where the return router reports the outcome. + */ +export function Search() { + const scrollContainerRef = useRef(null) + useOAuthReturnRouter() + const params = useParams() + const workspaceId = (params?.workspaceId as string) || '' + const { integrationAvailability } = usePermissionConfig() + const [connectTarget, setConnectTarget] = useState(null) + + const [searchTerm, setSearchTermParam] = useQueryState(connectorSearchParam.key, { + ...connectorSearchParam.parser, + ...connectorSearchUrlKeys, + }) + /** + * The input is controlled directly by the instant nuqs value; only the URL + * write is debounced. Filtering below is cheap in-memory over a static list, + * so it reads the instant value too. + */ + const setSearchTerm = useDebouncedSearchSetter(setSearchTermParam) + + const { credentials, isPending: credentialsLoading } = useSearchCredentials(workspaceId) + + useScrollRestoration(scrollContainerRef, { ready: !credentialsLoading }) + + const normalizedSearch = searchTerm.trim().toLowerCase() + + const visibleCredentials = normalizedSearch + ? credentials.filter((credential) => { + const display = resolveCredentialDisplay(credential) + return [credential.displayName, credential.description ?? '', display.subtitle].some( + (text) => text.toLowerCase().includes(normalizedSearch) + ) + }) + : credentials + + const visibleConnectors = normalizedSearch + ? SEARCH_CONNECTORS.filter( + (connector) => + connector.meta.name.toLowerCase().includes(normalizedSearch) || + connector.meta.description.toLowerCase().includes(normalizedSearch) + ) + : SEARCH_CONNECTORS + + const showNoResults = + Boolean(normalizedSearch) && visibleCredentials.length === 0 && visibleConnectors.length === 0 + + return ( +
+ +
+
+ setSearchTerm(e.target.value)} + disabled={credentialsLoading} + /> + +
+ {visibleCredentials.length > 0 && ( + + {visibleCredentials.map((credential) => ( + + ))} + + )} + + {visibleConnectors.length > 0 && ( + + {visibleConnectors.map((connector) => { + /** + * The OAuth path specifically: an integration's `state` can + * read `limited` on a service-account-only deployment, but a + * connector authenticates with OAuth alone. + */ + const availability = integrationAvailability.get( + connector.blockType.toLowerCase() + ) + const unavailable = availability ? !availability.oauthAvailable : false + return ( + + ) + })} + + )} + + {showNoResults && ( + + No connectors found matching “{searchTerm}” + + )} +
+
+
+ {connectTarget && workspaceId && ( + { + if (!open) setConnectTarget(null) + }} + workspaceId={workspaceId} + providerId={connectTarget.providerId} + requiredScopes={connectTarget.requiredScopes} + serviceName={connectTarget.serviceName} + serviceIcon={connectTarget.serviceIcon} + /> + )} +
+ ) +} diff --git a/apps/sim/lib/posthog/events.ts b/apps/sim/lib/posthog/events.ts index 85da54723a7..71c9643839a 100644 --- a/apps/sim/lib/posthog/events.ts +++ b/apps/sim/lib/posthog/events.ts @@ -614,10 +614,19 @@ export interface PostHogEventMap { action_id?: string } - /** A home-page suggested action was clicked. `action_id` is the candidate id (e.g. `gmail-0`). */ + /** The chat composer's mode switcher picked a different mode. */ + chat_mode_changed: { + workspace_id: string + mode: 'build' | 'search' + } + + /** + * A home-page suggested action was clicked. `action_id` is the candidate id + * (e.g. `gmail-0`); `connector` rows are the Search-mode "Connect X" rows. + */ suggested_action_clicked: { workspace_id: string - kind: 'prompt' | 'integration' + kind: 'prompt' | 'integration' | 'connector' action_id: string label: string position: number diff --git a/apps/sim/lib/sim-search/connectors.test.ts b/apps/sim/lib/sim-search/connectors.test.ts new file mode 100644 index 00000000000..e7c3aa5f641 --- /dev/null +++ b/apps/sim/lib/sim-search/connectors.test.ts @@ -0,0 +1,94 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/connectors/registry', () => { + const icon = () => null + return { + CONNECTOR_META_REGISTRY: { + mintlify: { id: 'mintlify', name: 'Mintlify', auth: { mode: 'apiKey' }, icon }, + jsm: { + id: 'jsm', + name: 'Jira Service Management', + auth: { mode: 'oauth', provider: 'jira' }, + icon, + }, + jira: { id: 'jira', name: 'Jira', auth: { mode: 'oauth', provider: 'jira' }, icon }, + google_drive: { + id: 'google_drive', + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive' }, + icon, + }, + gmail: { + id: 'gmail', + name: 'Gmail', + auth: { mode: 'oauth', provider: 'google-email' }, + icon, + }, + unknown: { + id: 'unknown', + name: 'Unknown', + auth: { mode: 'oauth', provider: 'not-a-service' }, + icon, + }, + }, + } +}) + +vi.mock('@/lib/oauth', () => { + const services = { + jira: { providerId: 'jira', name: 'Jira', icon: () => null }, + 'google-drive': { providerId: 'google-drive', name: 'Google Drive', icon: () => null }, + gmail: { providerId: 'google-email', name: 'Gmail', icon: () => null }, + } + return { + getServiceConfigByServiceId: (serviceId: string) => + services[serviceId as keyof typeof services] ?? null, + getServiceConfigByProviderId: (providerId: string) => + Object.values(services).find((service) => service.providerId === providerId) ?? null, + getCanonicalScopesForProvider: (providerId: string) => [`${providerId}:read`], + } +}) + +vi.mock('@/lib/integrations/credential-display', () => ({ + getIntegrationsForCredentialProvider: (providerId: string) => + providerId === 'jira' ? [{ type: 'jira' }] : [], +})) + +import { isSearchConnectorProvider, SEARCH_CONNECTORS } from '@/lib/sim-search/connectors' + +describe('SEARCH_CONNECTORS', () => { + it('lists OAuth connectors with a registered service, alphabetically', () => { + expect(SEARCH_CONNECTORS.map((connector) => connector.type)).toEqual([ + 'gmail', + 'google_drive', + 'jira', + 'jsm', + ]) + }) + + it('resolves the provider, scopes, and brand block type per connector', () => { + const jsm = SEARCH_CONNECTORS.find((connector) => connector.type === 'jsm') + expect(jsm).toMatchObject({ + providerId: 'jira', + requiredScopes: ['jira:read'], + serviceName: 'Jira', + blockType: 'jira', + }) + const drive = SEARCH_CONNECTORS.find((connector) => connector.type === 'google_drive') + expect(drive).toMatchObject({ blockType: 'google_drive' }) + const gmail = SEARCH_CONNECTORS.find((connector) => connector.type === 'gmail') + expect(gmail).toMatchObject({ providerId: 'google-email', serviceName: 'Gmail' }) + }) +}) + +describe('isSearchConnectorProvider', () => { + it('matches credentials by provider, including shared providers', () => { + expect(isSearchConnectorProvider('jira')).toBe(true) + expect(isSearchConnectorProvider('google-drive')).toBe(true) + expect(isSearchConnectorProvider('slack')).toBe(false) + expect(isSearchConnectorProvider(null)).toBe(false) + }) +}) diff --git a/apps/sim/lib/sim-search/connectors.ts b/apps/sim/lib/sim-search/connectors.ts new file mode 100644 index 00000000000..4995ff64717 --- /dev/null +++ b/apps/sim/lib/sim-search/connectors.ts @@ -0,0 +1,79 @@ +import type { ComponentType } from 'react' +import { getIntegrationsForCredentialProvider } from '@/lib/integrations/credential-display' +import { + getCanonicalScopesForProvider, + getServiceConfigByProviderId, + getServiceConfigByServiceId, +} from '@/lib/oauth' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import type { ConnectorMeta } from '@/connectors/types' + +/** + * A knowledge-base connector offered on the Sim Search surface: the connector's + * client-safe meta paired with the OAuth service a user connects it through. + * Only OAuth connectors qualify — an API-key connector has nowhere to keep a + * personal key outside a knowledge base, so it stays a knowledge-base flow. + */ +export interface SearchConnector { + /** `CONNECTOR_META_REGISTRY` key — the id a knowledge base reports in `connectorTypes`. */ + type: string + meta: ConnectorMeta + /** Canonical OAuth provider id the connection is stored under. */ + providerId: string + /** + * Scopes listed in the connect modal — the provider's canonical set, which is + * what the knowledge-base connector flow requests for the same provider. + */ + requiredScopes: readonly string[] + /** The OAuth service's own name and mark, for the connect modal. */ + serviceName: string + serviceIcon: ComponentType<{ className?: string }> + /** + * Block type lending the brand tile and the deployment-availability lookup: + * the first catalog integration on the provider, else the connector type. + */ + blockType: string +} + +/** + * Every Sim Search connector, alphabetical by name. Built once at module load. + * + * A connector names its service by service id (`confluence`) or, for Gmail, by + * the provider id (`google-email`); the knowledge-base connector flow accepts + * both through `getProviderIdFromServiceId`'s raw fallback, so the lookup here + * tries the service id first and the provider id second. + */ +export const SEARCH_CONNECTORS: readonly SearchConnector[] = Object.entries(CONNECTOR_META_REGISTRY) + .flatMap(([type, meta]): SearchConnector[] => { + if (meta.auth.mode !== 'oauth') return [] + const service = + getServiceConfigByServiceId(meta.auth.provider) ?? + getServiceConfigByProviderId(meta.auth.provider) + if (!service) return [] + return [ + { + type, + meta, + providerId: service.providerId, + requiredScopes: getCanonicalScopesForProvider(service.providerId), + serviceName: service.name, + serviceIcon: service.icon as ComponentType<{ className?: string }>, + blockType: getIntegrationsForCredentialProvider(service.providerId)[0]?.type ?? type, + }, + ] + }) + .sort((a, b) => a.meta.name.localeCompare(b.meta.name)) + +/** + * Provider ids some Sim Search connector connects through. Several connectors + * share one (Jira and Jira Service Management both use `jira`), so a credential + * is matched to the surface by provider rather than to a single connector. + */ +const SEARCH_PROVIDER_IDS: ReadonlySet = new Set( + SEARCH_CONNECTORS.map((connector) => connector.providerId) +) + +/** Whether a stored credential's provider powers a Sim Search connector. */ +export function isSearchConnectorProvider(providerId: string | null): boolean { + return providerId !== null && SEARCH_PROVIDER_IDS.has(providerId) +} diff --git a/apps/sim/stores/mothership-mode/store.ts b/apps/sim/stores/mothership-mode/store.ts new file mode 100644 index 00000000000..049c2cd16bf --- /dev/null +++ b/apps/sim/stores/mothership-mode/store.ts @@ -0,0 +1,38 @@ +import { create } from 'zustand' +import { devtools } from 'zustand/middleware' + +export const MOTHERSHIP_MODES = ['build', 'search'] as const + +export type MothershipMode = (typeof MOTHERSHIP_MODES)[number] + +interface MothershipModeState { + mode: MothershipMode + setMode: (mode: MothershipMode) => void + reset: () => void +} + +const initialState: Pick = { mode: 'build' } + +/** + * The chat composer's mode — Build (default) or Search — read by the input's + * mode switcher and by the suggested actions beneath the input. + * + * A store rather than `Home` state because `Home` remounts per chat + * (`key={chatId}`) and the new-chat → `/chat/[chatId]` handoff must carry the + * mode across. Not a URL param: that handoff rewrites the path with + * `history.replaceState`, which would drop a query key, and the mode is a + * composer preference rather than a destination (the same reasoning that keeps + * canvas mode out of the URL). Deliberately not persisted, so the server render + * and the first client render agree — a persisted `search` would hydrate over + * server-rendered Build chrome. + */ +export const useMothershipModeStore = create()( + devtools( + (set) => ({ + ...initialState, + setMode: (mode) => set({ mode }), + reset: () => set(initialState), + }), + { name: 'mothership-mode-store' } + ) +) diff --git a/apps/sim/stores/reset-all-stores.ts b/apps/sim/stores/reset-all-stores.ts index 9665d01a09f..211d8180377 100644 --- a/apps/sim/stores/reset-all-stores.ts +++ b/apps/sim/stores/reset-all-stores.ts @@ -3,6 +3,7 @@ import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { useExecutionStore } from '@/stores/execution' import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store' +import { useMothershipModeStore } from '@/stores/mothership-mode/store' import { useMothershipQueueStore } from '@/stores/mothership-queue/store' import { useOperationQueueStore } from '@/stores/operation-queue/store' import { @@ -55,5 +56,6 @@ export async function resetAllStores(): Promise { clearAllExecutionPointers() useMothershipDraftsStore.setState({ drafts: {} }) useMothershipQueueStore.getState().reset() + useMothershipModeStore.getState().reset() await consolePersistence.persist({ merge: false }) } diff --git a/packages/emcn/src/components/chip-dropdown/chip-dropdown.tsx b/packages/emcn/src/components/chip-dropdown/chip-dropdown.tsx index 16ab7f0fbf9..7932ebee96e 100644 --- a/packages/emcn/src/components/chip-dropdown/chip-dropdown.tsx +++ b/packages/emcn/src/components/chip-dropdown/chip-dropdown.tsx @@ -181,6 +181,7 @@ const ChipDropdown = forwardRef( leftIcon: LeftIcon, className, variant = 'filled', + shape, active, fullWidth, 'aria-label': ariaLabel, @@ -325,7 +326,7 @@ const ChipDropdown = forwardRef( aria-label={ariaLabel} aria-labelledby={ariaLabelledBy} className={cn( - chipVariants({ variant, active, fullWidth }), + chipVariants({ variant, shape, active, fullWidth }), hasTriggerBorder && TRIGGER_BORDER_CLASS, className )} diff --git a/packages/emcn/src/components/chip/chip-chrome.ts b/packages/emcn/src/components/chip/chip-chrome.ts index c0ed45f02e6..979076d5682 100644 --- a/packages/emcn/src/components/chip/chip-chrome.ts +++ b/packages/emcn/src/components/chip/chip-chrome.ts @@ -15,8 +15,10 @@ export const chipFilledSurfaceTokens = `border border-[var(--border-1)] ${chipFi */ export const chipPrimaryFillTokens = 'bg-[var(--text-primary)] text-[var(--text-inverse)] dark:bg-white dark:text-[var(--bg)]' +/** The default chip corner radius. `chipVariants`' `shape: 'round'` swaps it for `rounded-full`. */ +export const chipRadiusClass = 'rounded-lg' /** Filled surface shared by the chip text fields ({@link ChipInput}, {@link ChipTextarea}) — aligned with `Chip` / `ChipDropdown`. */ -export const chipFieldSurfaceClass = `rounded-lg ${chipFilledSurfaceTokens} transition-colors` +export const chipFieldSurfaceClass = `${chipRadiusClass} ${chipFilledSurfaceTokens} transition-colors` /** * The raised "border + drop shadow" ring of the `border-shadow` chip variant: a * 1px hairline ring plus a soft drop shadow, in both light and dark. Single @@ -47,6 +49,13 @@ export const chipFieldTextClass = */ export const chipContentGap = 'gap-1.5' +/** + * Chip pill geometry minus its corner radius — height, centering, gap, padding, + * text size. `chipVariants` composes this with its `shape` variant so a raw + * (non-`cn`) consumer never emits two competing radii; everything else reads + * {@link chipGeometryClass}, which adds the default radius back. + */ +export const chipGeometryUnroundedClass = `h-[30px] items-center ${chipContentGap} px-2 text-left text-sm` /** * Chip pill geometry — height, centering, gap, radius, padding, text size — with * NO interactivity (no `cursor-pointer`, no hover). `chipVariants` composes this @@ -54,7 +63,7 @@ export const chipContentGap = 'gap-1.5' * current-location label or a non-navigable breadcrumb) reuse it directly to * match a chip's shape without inheriting its hover. */ -export const chipGeometryClass = `h-[30px] items-center ${chipContentGap} rounded-lg px-2 text-left text-sm` +export const chipGeometryClass = `${chipGeometryUnroundedClass} ${chipRadiusClass}` /** Chip-content icon (non-inverse): 16px, non-shrinking, `--text-icon`. Inverse chip variants override the color to `currentColor`. */ export const chipContentIconClass = 'size-[16px] flex-shrink-0 text-[var(--text-icon)]' /** Fade-free single-line fallback for rich chip content. Plain text labels should render through `OverflowText`. */ diff --git a/packages/emcn/src/components/chip/chip.tsx b/packages/emcn/src/components/chip/chip.tsx index 1063e1822c2..f06411da8a6 100644 --- a/packages/emcn/src/components/chip/chip.tsx +++ b/packages/emcn/src/components/chip/chip.tsx @@ -16,9 +16,10 @@ import { chipContentIconClass, chipContentLabelClass, chipFilledFillTokens, - chipGeometryClass, + chipGeometryUnroundedClass, chipHoverSurfaceClass, chipPrimaryFillTokens, + chipRadiusClass, } from './chip-chrome' /** @@ -40,6 +41,9 @@ import { * no CSS border, no fill). * `active` renders the default/filled chip in its selected state — `--surface-active`, held through hover. * `fullWidth` swaps `inline-flex` for block-level `flex`. + * `shape` picks the corner radius: the implicit `default` is the `rounded-lg` pill; `round` is fully round + * (`rounded-full`) for a chip sitting in a row of round controls. The radius lives in this variant rather than + * in the base string so a raw `chipVariants({ shape: 'round' })` consumer emits exactly one radius. * * The chip carries NO outer margin — spacing between chips belongs to the parent, as a `gap`. It used to ship a * default `mx-0.5` "cluster margin" with a `flush` prop to switch it off, which meant a chip's visual box was not @@ -55,7 +59,7 @@ import { * {@link chipHoverSurfaceClass}. */ const chipVariants = cva( - `group cursor-pointer ${chipGeometryClass} transition-colors disabled:cursor-not-allowed disabled:opacity-60`, + `group cursor-pointer ${chipGeometryUnroundedClass} transition-colors disabled:cursor-not-allowed disabled:opacity-60`, { variants: { variant: { @@ -68,6 +72,7 @@ const chipVariants = cva( 'bg-[var(--surface-2)] shadow-[0_0_0_1px_rgba(28,40,64,0.08),0_1px_3px_0_rgba(28,40,64,0.1)] hover-hover:bg-[var(--surface-3)] dark:shadow-[0_0_0_1px_var(--border-1),0_1px_3px_0_rgba(0,0,0,0.3)] dark:hover-hover:bg-[var(--surface-4)]', border: `shadow-[0_0_0_1px_rgba(28,40,64,0.08),0_1px_3px_0_rgba(28,40,64,0.1)] ${chipHoverSurfaceClass} dark:shadow-[0_0_0_1px_var(--border-1),0_1px_3px_0_rgba(0,0,0,0.3)]`, }, + shape: { default: chipRadiusClass, round: 'rounded-full' }, active: { true: '', false: '' }, fullWidth: { true: 'flex w-full', false: 'inline-flex' }, }, @@ -75,7 +80,7 @@ const chipVariants = cva( { variant: ['default', 'filled'], active: false, className: chipHoverSurfaceClass }, { variant: ['default', 'filled'], active: true, className: chipActiveSurfaceClass }, ], - defaultVariants: { variant: 'default', active: false, fullWidth: false }, + defaultVariants: { variant: 'default', shape: 'default', active: false, fullWidth: false }, } ) @@ -141,6 +146,7 @@ const Chip = forwardRef(function Chip( { className, variant, + shape, active, fullWidth, leftIcon, @@ -156,7 +162,7 @@ const Chip = forwardRef(function Chip( + ) : ( + { + const parsed = parseOptionValue(next) + if (parsed) onChange(parsed) + }} + placeholder='Select a credential group' + isLoading={isLoading} + disabled={disabled || Boolean(loadError)} + /> + )}

- {membersUnavailable - ? 'Create a credential group with a ' - : 'Members join through a credential group; manage them in '} + Members connect their own {connectorConfig.name} account after you invite them in{' '} - {membersUnavailable ? `${connectorConfig.name} option in Settings` : 'Settings'} + Settings .

)} + + {footer} ) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index a838cd1dc90..a4ddc1bfe72 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -92,6 +92,15 @@ const MEMBER_SYNC_IN_FLIGHT_TOOLTIP = { running: 'Syncing members', } as const +/** How each member-engine status reads on the card's badge. */ +const MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS = { + idle: 'active', + pending: 'pending', + running: 'syncing', + error: 'error', + disabled: 'disabled', +} as const satisfies Record + const CONNECTOR_ACTION_BUTTON_CLASSES = 'size-7 rounded-lg p-0 text-[var(--text-muted)] hover-hover:bg-[var(--surface-active)] hover-hover:text-[var(--text-primary)]' @@ -286,8 +295,19 @@ function ConnectorCard({ const connectorDef = CONNECTOR_META_REGISTRY[connector.connectorType] const Icon = connectorDef?.icon const brandBg = getBlock(connector.connectorType)?.bgColor ?? null + /** + * A members-mode connector's content status stays `active` while the member + * engine does the work, so its badge reads the member engine's status. A + * paused or disabled content status still wins: the user set it. + */ + const effectiveStatus = + connector.accessMode === 'members' && connector.status === 'active' + ? (MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS[ + connector.memberSyncStatus as keyof typeof MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS + ] ?? 'active') + : connector.status const statusConfig = - STATUS_CONFIG[connector.status as keyof typeof STATUS_CONFIG] || STATUS_CONFIG.active + STATUS_CONFIG[effectiveStatus as keyof typeof STATUS_CONFIG] || STATUS_CONFIG.active const serviceId = connectorDef?.auth.mode === 'oauth' ? connectorDef.auth.provider : undefined const providerId = serviceId ? getProviderIdFromServiceId(serviceId) : undefined @@ -452,6 +472,15 @@ function ConnectorCard({ {lastSyncError} )} + {connector.accessRewritePending && ( + <> + · + + + Updating access + + + )} @@ -571,6 +600,22 @@ function ConnectorCard({ + {syncsPerMember && connector.memberSyncStatus === 'disabled' && ( +
+
+
+ + Per-member sync is disabled +
+

+ {connector.lastMemberSyncError ?? 'The connector can no longer sync per member.'}{' '} + Members keep no access until it is fixed; switch the connector's access to re-enable + it. +

+
+
+ )} + {connector.status === 'disabled' && (
diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index dd08e19182e..c0550872bfa 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -225,12 +225,18 @@ export function EditConnectorModal({ initialCanonicalModes, }) - const { ownerBilling } = useWorkspaceHostContext() + const { ownerBilling, features } = useWorkspaceHostContext() const { canAdmin } = useUserPermissionsContext() const { workspaceId } = useParams<{ workspaceId: string }>() const { mutate: updateConnector, isPending: isSavingSettings } = useUpdateConnector() - const { mutate: updateAccess, isPending: isSavingAccess } = useUpdateConnectorAccess() - const isSaving = isSavingSettings || isSavingAccess + const { mutate: updateAccess, isPending: isSwitchingAccess } = useUpdateConnectorAccess() + const isSaving = isSavingSettings || isSwitchingAccess + /** + * The field shows where the flag is on, and stays visible read-only for a + * connector already syncing per member where it has since been turned off. + */ + const showAccessField = + features?.knowledgeMemberAccess === true || connector.accessMode === 'members' const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling) @@ -260,7 +266,6 @@ export function EditConnectorModal({ ) const hasChanges = useMemo(() => { - if (accessDirty) return true if (syncInterval !== connector.syncIntervalMinutes) return true if (didCanonicalModesChange(canonicalModes, persistedCanonicalModes)) return true const resolved = resolveSourceConfig() @@ -269,7 +274,6 @@ export function EditConnectorModal({ } return false }, [ - accessDirty, resolveSourceConfig, syncInterval, connector.syncIntervalMinutes, @@ -305,52 +309,15 @@ export function EditConnectorModal({ updates.sourceConfig = next } - /** - * The mode switch is its own admin operation and rewrites document access, - * so it runs after the ordinary settings save has landed rather than - * alongside it — a refused settings edit must not leave a half-switched - * connector behind. - */ - const switchAccess = () => { - updateAccess( - { - knowledgeBaseId, - connectorId: connector.id, - access: - access.accessMode === 'members' - ? { - accessMode: 'members', - credentialGroupId: access.credentialGroupId, - credentialGroupOptionId: access.credentialGroupOptionId, - } - : { - accessMode: 'workspace', - credentialId: workspaceCredentialId ?? undefined, - }, - }, - { - onSuccess: () => onOpenChange(false), - onError: (err) => { - logger.error('Failed to switch connector access', { error: err.message }) - setError(err.message) - }, - } - ) - } - if (Object.keys(updates).length === 0) { - if (accessDirty) switchAccess() - else onOpenChange(false) + onOpenChange(false) return } updateConnector( { knowledgeBaseId, connectorId: connector.id, updates }, { - onSuccess: () => { - if (accessDirty) switchAccess() - else onOpenChange(false) - }, + onSuccess: () => onOpenChange(false), onError: (err) => { logger.error('Failed to update connector', { error: err.message }) setError(err.message) @@ -359,6 +326,39 @@ export function EditConnectorModal({ ) } + /** + * The mode switch is its own admin operation: it rewrites document access + * and queues a run of the other engine, so it is applied on its own rather + * than folded into a settings save that would race the run it starts. + */ + const handleApplyAccess = () => { + setError(null) + updateAccess( + { + knowledgeBaseId, + connectorId: connector.id, + access: + access.accessMode === 'members' + ? { + accessMode: 'members', + credentialGroupId: access.credentialGroupId, + credentialGroupOptionId: access.credentialGroupOptionId, + } + : { + accessMode: 'workspace', + credentialId: workspaceCredentialId ?? undefined, + }, + }, + { + onSuccess: () => setWorkspaceCredentialId(null), + onError: (err) => { + logger.error('Failed to switch connector access', { error: err.message }) + setError(err.message) + }, + } + ) + } + const displayName = connectorConfig?.name ?? connector.connectorType const Icon = connectorConfig?.icon @@ -403,6 +403,12 @@ export function EditConnectorModal({ access={access} onAccessChange={setAccess} canAdmin={canAdmin} + showAccessField={showAccessField} + accessDirty={accessDirty} + accessComplete={accessComplete} + isSwitchingAccess={isSwitchingAccess} + onApplyAccess={handleApplyAccess} + onResetAccess={() => setAccess(currentAccess(connector))} workspaceId={workspaceId} needsWorkspaceCredential={needsWorkspaceCredential} workspaceCredentialId={workspaceCredentialId} @@ -419,7 +425,7 @@ export function EditConnectorModal({ primaryAction={{ label: isSaving ? 'Saving…' : 'Save', onClick: handleSave, - disabled: !hasChanges || !accessComplete || isSaving, + disabled: !hasChanges || isSaving, }} /> )} @@ -444,6 +450,12 @@ interface SettingsTabProps { access: ConnectorAccessSelection onAccessChange: (access: ConnectorAccessSelection) => void canAdmin: boolean + showAccessField: boolean + accessDirty: boolean + accessComplete: boolean + isSwitchingAccess: boolean + onApplyAccess: () => void + onResetAccess: () => void workspaceId: string needsWorkspaceCredential: boolean workspaceCredentialId: string | null @@ -467,6 +479,12 @@ function SettingsTab({ access, onAccessChange, canAdmin, + showAccessField, + accessDirty, + accessComplete, + isSwitchingAccess, + onApplyAccess, + onResetAccess, workspaceId, needsWorkspaceCredential, workspaceCredentialId, @@ -493,7 +511,7 @@ function SettingsTab({ return ( <> - {connectorConfig && connectorConfig.auth.mode === 'oauth' && ( + {connectorConfig && connectorConfig.auth.mode === 'oauth' && showAccessField && ( + {needsWorkspaceCredential && ( + + )} +
+ + +
+

+ {access.accessMode === 'members' + ? 'Documents stay hidden until members connect and sync. Listing caps are cleared.' + : 'Every workspace member can read every synced document once the next sync completes.'} +

+
+ ) : undefined + } /> )} - {needsWorkspaceCredential && connectorConfig && ( - - - - )} - {connectorConfig && ( 'hybrid', + value: () => 'auto', mode: 'advanced', condition: { field: 'operation', value: 'search' }, }, diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 2596295e7f6..445fde1e2eb 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -145,20 +145,22 @@ export function useConnectorDetail(knowledgeBaseId?: string, connectorId?: strin * never starting that poll, and showing stale sync history behind the list's * spinner. */ +type ConnectorStatusPatch = Pick | Pick + function setCachedConnectorStatus( queryClient: QueryClient, knowledgeBaseId: string, connectorId: string, - status: ConnectorData['status'] + patch: ConnectorStatusPatch ) { queryClient.setQueryData(connectorKeys.lists(knowledgeBaseId), (connectors) => connectors?.map((connector) => - connector.id === connectorId ? { ...connector, status } : connector + connector.id === connectorId ? { ...connector, ...patch } : connector ) ) queryClient.setQueryData( connectorKeys.detail(knowledgeBaseId, connectorId), - (detail) => (detail ? { ...detail, status } : detail) + (detail) => (detail ? { ...detail, ...patch } : detail) ) } @@ -186,11 +188,35 @@ function optimisticallySetConnectorStatus( .getQueryData(connectorKeys.lists(knowledgeBaseId)) ?.find((connector) => connector.id === connectorId)?.status - setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, status) + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { status }) return previousStatus } +/** + * The optimistic "queued" write for a sync trigger, on whichever engine the + * connector runs: a members connector queues a member run, so its content + * status must not flip. Returns what to restore if the trigger is refused. + */ +function optimisticallyQueueSync( + queryClient: QueryClient, + knowledgeBaseId: string, + connectorId: string +): ConnectorStatusPatch | undefined { + const cached = queryClient + .getQueryData(connectorKeys.lists(knowledgeBaseId)) + ?.find((connector) => connector.id === connectorId) + if (!cached) return undefined + if (cached.accessMode === 'members') { + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { + memberSyncStatus: 'pending', + }) + return { memberSyncStatus: cached.memberSyncStatus } + } + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { status: 'pending' }) + return { status: cached.status } +} + interface CreateConnectorParams { knowledgeBaseId: string connectorType: string @@ -227,6 +253,7 @@ export function useCreateConnector() { */ onSettled: (_data, _error, { knowledgeBaseId }) => { queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) }, }) } @@ -271,7 +298,9 @@ export function useUpdateConnector() { }, onError: (_error, { knowledgeBaseId, connectorId }, previousStatus) => { if (previousStatus) { - setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, previousStatus) + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { + status: previousStatus, + }) } }, onSettled: (_data, _error, { knowledgeBaseId }) => { @@ -317,6 +346,8 @@ export function useUpdateConnectorAccess() { queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, }) + /** The base list says whether any connector syncs per member. */ + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) }, }) } @@ -365,6 +396,7 @@ export function useDeleteConnector() { if (deleteDocuments) { queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentDetails(knowledgeBaseId) }) } + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) }, }) } @@ -400,15 +432,15 @@ export function useTriggerSync() { */ onMutate: async ({ knowledgeBaseId, connectorId }) => { await queryClient.cancelQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) - return optimisticallySetConnectorStatus(queryClient, knowledgeBaseId, connectorId, 'pending') + return optimisticallyQueueSync(queryClient, knowledgeBaseId, connectorId) }, /** * Rolling back also stops the poll the optimistic `pending` started, so a * refused sync does not leave the row spinning. */ - onError: (_error, { knowledgeBaseId, connectorId }, previousStatus) => { - if (previousStatus) { - setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, previousStatus) + onError: (_error, { knowledgeBaseId, connectorId }, previous) => { + if (previous) { + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, previous) } queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) }, diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index fd3e0279d3b..0909ec8ae38 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -16,16 +16,15 @@ export const knowledgeSearchTagFilterSchema = z.object({ export const KNOWLEDGE_SEARCH_MODES = ['vector', 'hybrid'] as const /** - * Shared by the internal and v1 search contracts. Defaults to `hybrid`: the - * full-text leg recovers the exact tokens embeddings rank poorly, and a caller - * that wants semantic-only ranking asks for `vector`. + * Shared by the internal and v1 search contracts. Omitted, the workspace's + * default applies: `hybrid` where permission-aware knowledge is on, else + * `vector`. The use case resolves that, so the schema carries no default. */ export const knowledgeSearchModeSchema = z .enum(KNOWLEDGE_SEARCH_MODES) .optional() .nullable() - .default('hybrid') - .transform((val) => val ?? 'hybrid') + .transform((val) => val ?? undefined) export const knowledgeSearchBodySchema = z .object({ @@ -52,10 +51,10 @@ export const knowledgeSearchBodySchema = z .nullable() .transform((val) => val || undefined), /** - * `hybrid` (default) runs a full-text leg alongside semantic retrieval and - * fuses the two by reciprocal rank, which recovers exact tokens (error codes, - * ticket keys, identifiers) that embeddings rank poorly. `vector` is - * semantic-only retrieval. + * `hybrid` runs a full-text leg alongside semantic retrieval and fuses the + * two by reciprocal rank, which recovers exact tokens (error codes, ticket + * keys, identifiers) that embeddings rank poorly. `vector` is semantic-only + * retrieval. Omitted, the workspace's default applies. */ searchMode: knowledgeSearchModeSchema, rerankerEnabled: z.boolean().optional().default(false), diff --git a/apps/sim/lib/api/contracts/v1/knowledge/index.ts b/apps/sim/lib/api/contracts/v1/knowledge/index.ts index f4cc24e51ec..412fa140c2d 100644 --- a/apps/sim/lib/api/contracts/v1/knowledge/index.ts +++ b/apps/sim/lib/api/contracts/v1/knowledge/index.ts @@ -182,8 +182,8 @@ export const v1KnowledgeSearchBodySchema = z topK: z.number().min(1).max(100).default(10), tagFilters: z.array(v1SearchTagFilterSchema).optional(), /** - * `hybrid` (default) fuses a full-text leg with semantic retrieval by - * reciprocal rank; `vector` is semantic-only. + * `hybrid` fuses a full-text leg with semantic retrieval by reciprocal + * rank; `vector` is semantic-only. Omitted, the workspace's default applies. */ searchMode: knowledgeSearchModeSchema, }) diff --git a/apps/sim/lib/api/contracts/workspaces.ts b/apps/sim/lib/api/contracts/workspaces.ts index 3b2c99e3a52..883c9375c0c 100644 --- a/apps/sim/lib/api/contracts/workspaces.ts +++ b/apps/sim/lib/api/contracts/workspaces.ts @@ -262,6 +262,8 @@ export const workspaceHostContextSchema = z.object({ features: z .object({ credentialGroups: z.boolean(), + /** Optional for rolling compatibility with app versions that predate the flag. */ + knowledgeMemberAccess: z.boolean().optional(), }) .optional(), }) diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index 7ba5579ee6d..606e2449663 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -68,6 +68,9 @@ function canonicalGoogleScope(scope: string): string { return scope } +const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive' +const DRIVE_READONLY_SCOPE = 'https://www.googleapis.com/auth/drive.readonly' + function hasRequiredGoogleScopes( providerId: string, grantedScopes: string[], @@ -77,12 +80,18 @@ function hasRequiredGoogleScopes( return requiredScopes.every((requestedScope) => { const required = canonicalGoogleScope(requestedScope) if (granted.has(required)) return true - return ( + if ( providerId === 'google-email' && granted.has(GMAIL_MODIFY_SCOPE) && (required === GMAIL_READONLY_SCOPE || required === GMAIL_SEND_SCOPE || required === GMAIL_LABELS_SCOPE) + ) { + return true + } + /** Full Drive access implies read-only access, which is all a crawler asks for. */ + return ( + providerId === 'google-drive' && granted.has(DRIVE_SCOPE) && required === DRIVE_READONLY_SCOPE ) }) } diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index a3992c6cbd9..241c82feff2 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -589,6 +589,7 @@ export const env = createEnv({ TABLES_V2_API: z.boolean().optional(), // Enable the v2 tables HTTP API (public /api/v2/tables + internal /api/table/[tableId]/query predicate-grammar route) TABLE_ROW_TTL: z.boolean().optional(), CREDENTIAL_GROUPS: z.boolean().optional(), // Enable enterprise Credential Groups globally + KNOWLEDGE_MEMBER_ACCESS: z.boolean().optional(), // Enable per-member knowledge connectors and hybrid-by-default retrieval globally // Organizations - for self-hosted deployments ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements) diff --git a/apps/sim/lib/core/config/feature-flags.test.ts b/apps/sim/lib/core/config/feature-flags.test.ts index fefd67589a3..2d28ffa4cf4 100644 --- a/apps/sim/lib/core/config/feature-flags.test.ts +++ b/apps/sim/lib/core/config/feature-flags.test.ts @@ -14,6 +14,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({ TABLES_V2_API: undefined as boolean | undefined, TABLE_ROW_TTL: undefined as boolean | undefined, CREDENTIAL_GROUPS: undefined as boolean | undefined, + KNOWLEDGE_MEMBER_ACCESS: undefined as boolean | undefined, }, })) @@ -123,6 +124,40 @@ describe('isFeatureEnabled', () => { vi.clearAllMocks() setEnvFlags({ isAppConfigEnabled: false }) envRef.CREDENTIAL_GROUPS = undefined + envRef.KNOWLEDGE_MEMBER_ACCESS = undefined + }) + + describe('knowledge-member-access flag', () => { + it('uses a global fallback switch off AppConfig', async () => { + expect(await isFeatureEnabled('knowledge-member-access')).toBe(false) + + envRef.KNOWLEDGE_MEMBER_ACCESS = true + expect(await isFeatureEnabled('knowledge-member-access')).toBe(true) + }) + + it('opens for an allowlisted workspace only', async () => { + withAppConfig({ 'knowledge-member-access': { workspaceIds: ['ws-1'] } }) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-1', userId: 'u1' }) + ).toBe(true) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2', userId: 'u1' }) + ).toBe(false) + expect(mockIsPlatformAdmin).not.toHaveBeenCalled() + }) + + it('opens for a platform admin in any workspace', async () => { + withAppConfig({ 'knowledge-member-access': { workspaceIds: ['ws-1'], adminEnabled: true } }) + mockIsPlatformAdmin.mockResolvedValue(true) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2', userId: 'admin' }) + ).toBe(true) + mockIsPlatformAdmin.mockResolvedValue(false) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2', userId: 'u1' }) + ).toBe(false) + expect(await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2' })).toBe(false) + }) }) describe('credential-groups flag', () => { diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index e0d457d69c9..aab00d1d8ad 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -85,6 +85,16 @@ const FEATURE_FLAGS = { 'Enterprise subscription. Off-AppConfig falls back to CREDENTIAL_GROUPS.', fallback: 'CREDENTIAL_GROUPS', }, + 'knowledge-member-access': { + description: + 'Permission-aware knowledge bases: lets a workspace admin sync a connector once per ' + + 'Credential Group member so each person sees only what their own account can read, and ' + + 'makes hybrid retrieval with a source-recency boost the default for searches in that ' + + 'workspace. Gated by workspaceId and platform admins via AppConfig; off-AppConfig falls ' + + 'back to KNOWLEDGE_MEMBER_ACCESS. Requires the credential-groups flag for the connector ' + + 'side to do anything.', + fallback: 'KNOWLEDGE_MEMBER_ACCESS', + }, } satisfies Record /** diff --git a/apps/sim/lib/credential-groups/credentials.ts b/apps/sim/lib/credential-groups/credentials.ts index 985d6b60b72..735959a3607 100644 --- a/apps/sim/lib/credential-groups/credentials.ts +++ b/apps/sim/lib/credential-groups/credentials.ts @@ -54,6 +54,28 @@ export interface ManagedCredentialGroupBinding { credentialGroupId: string credentialGroupOptionId: string managedOauthStatus: ManagedOAuthCredentialStatus + enrollmentStatus: CredentialGroupEnrollmentStatus + groupStatus: 'active' | 'disabled' + /** Null when the option was removed from the group. */ + optionStatus: 'active' | 'disabled' | null +} + +/** + * Whether a managed credential may be used right now: the credential, its + * enrollment, its option, and its group are all live. Every consumer that + * mints a token from a binding checks this, so a disabled option or a revoked + * enrollment denies without waiting for a scope bump to invalidate the + * credential itself. + */ +export function isManagedCredentialGroupBindingLive( + binding: ManagedCredentialGroupBinding +): boolean { + return ( + binding.managedOauthStatus === 'active' && + (binding.enrollmentStatus === 'in_progress' || binding.enrollmentStatus === 'completed') && + binding.groupStatus === 'active' && + binding.optionStatus === 'active' + ) } export interface CredentialGroupEnrollmentAccess { @@ -175,12 +197,16 @@ export async function loadManagedCredentialGroupBinding( credentialGroupId: credentialGroupEnrollment.credentialGroupId, credentialGroupOptionId: credential.credentialGroupOptionId, managedOauthStatus: credential.managedOauthStatus, + enrollmentStatus: credentialGroupEnrollment.status, + groupStatus: credentialGroup.status, + groupOptions: credentialGroup.options, }) .from(credential) .innerJoin( credentialGroupEnrollment, eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) ) + .innerJoin(credentialGroup, eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId)) .where(and(eq(credential.id, credentialId), eq(credential.type, 'managed_oauth'))) .limit(1) if (!row) return null @@ -198,6 +224,10 @@ export async function loadManagedCredentialGroupBinding( credentialGroupId: row.credentialGroupId, credentialGroupOptionId: row.credentialGroupOptionId, managedOauthStatus: row.managedOauthStatus, + enrollmentStatus: row.enrollmentStatus, + groupStatus: row.groupStatus, + optionStatus: + row.groupOptions.find((option) => option.id === row.credentialGroupOptionId)?.status ?? null, } } diff --git a/apps/sim/lib/knowledge/access/availability.ts b/apps/sim/lib/knowledge/access/availability.ts new file mode 100644 index 00000000000..b6f31798840 --- /dev/null +++ b/apps/sim/lib/knowledge/access/availability.ts @@ -0,0 +1,26 @@ +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' + +/** + * Who is asking, for the flag's workspace allowlist and platform-admin + * clauses. `userId` is the signed-in user when there is one; an actorless + * caller (a schedule, a cron, an API key) passes none and is judged by the + * workspace alone. + */ +export interface KnowledgeMemberAccessContext { + workspaceId: string + userId?: string +} + +/** + * Whether permission-aware knowledge is on for this workspace: members-mode + * connectors, their per-member change feeds, and hybrid-by-default retrieval + * with the source-recency boost. Everything the feature adds checks this one + * gate, so turning the flag off freezes members-mode connectors (their + * documents stay hidden, nothing is deleted) and returns search to the + * semantic-only default. + */ +export async function isKnowledgeMemberAccessAvailable( + context: KnowledgeMemberAccessContext +): Promise { + return isFeatureEnabled('knowledge-member-access', context) +} diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index 996cd12ad0a..8b0084e494c 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -1,5 +1,5 @@ import { AuditAction, AuditResourceType } from '@sim/audit' -import type { Principal } from '@sim/auth/principal' +import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' @@ -69,6 +69,7 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ accessMode: 'members' as const, binding: await resolveKnowledgeConnectorMembersBinding({ workspaceId, + actingUserId: resolvePrincipalSubjectUserId(principal) ?? undefined, connectorMeta, binding: { credentialGroupId: requireBindingField( diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index b78e3b3172e..6425cb7bf07 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -108,6 +108,7 @@ const crossWorkspaceContext = { const connectorContext = { ...crossWorkspaceContext, + access: { get: async () => ({ kind: 'workspace' as const, tokens: ['ws', 'pub'] as const }) }, connectorId: 'connector-b', connector: { id: 'connector-b', diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 0233b7ee282..43f785efe27 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -20,6 +20,7 @@ import { getCredentialActorContext, resolveCredentialTokenIdentity, } from '@/lib/credentials/access' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeAttributedUserId, @@ -432,6 +433,7 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ } membersBinding = await resolveKnowledgeConnectorMembersBinding({ workspaceId, + actingUserId: subjectUserId, connectorMeta, binding: { credentialGroupId: input.credentialGroupId, @@ -720,6 +722,7 @@ export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase( eq(document.connectorId, context.connectorId), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(await context.access.get()), ] as const const [[activeCount], excludedCountRows] = await Promise.all([ db diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index 89e6af871df..78a9ed8eba6 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -1002,6 +1002,7 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, input.operation, input.enabledFilter, + await context.access.get(), generateRequestId() ) : input.documentIds?.length diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 227d23cab69..d7949ab6735 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -162,7 +162,8 @@ describe('knowledge search application use case', () => { expect.objectContaining({ knowledgeBaseIds: ['knowledge-1'], topK: 5, - searchMode: 'hybrid', + searchMode: 'vector', + boostRecency: false, }) ) expect(result.results[0]).toMatchObject({ diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 611f6aa7bb2..7b95ddfa468 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -38,6 +38,7 @@ import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' import { rerank } from '@/lib/knowledge/reranker' import type { RerankerStatus } from '@/lib/knowledge/reranker-models' +import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' import { executeKnowledgeSearch, generateSearchEmbedding, @@ -288,6 +289,11 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ : Promise.resolve(null) /** Resolved alongside the embedding call; both are needed before the first leg runs. */ const accessPromise = context.access.get() + const searchDefaults = await resolveKnowledgeSearchDefaults({ + workspaceId: context.workspaceId, + userId, + requestedMode: input.searchMode, + }) const useReranker = Boolean(input.rerankerEnabled && hasQuery) const candidateTopK = useReranker ? input.rerankerInputCount !== undefined @@ -302,7 +308,8 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ knowledgeBaseIds, topK: candidateTopK, access, - searchMode: input.searchMode ?? 'hybrid', + searchMode: searchDefaults.searchMode, + boostRecency: searchDefaults.boostRecency, query: input.query, queryVector: hasQuery ? JSON.stringify((await queryEmbeddingPromise)?.embedding ?? null) diff --git a/apps/sim/lib/knowledge/application/tags.ts b/apps/sim/lib/knowledge/application/tags.ts index d9a88888c72..c2e836f1075 100644 --- a/apps/sim/lib/knowledge/application/tags.ts +++ b/apps/sim/lib/knowledge/application/tags.ts @@ -354,7 +354,13 @@ export const readKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ resolveContext: ({ principal, input }: { principal: Principal; input: ListKnowledgeTagsInput }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ context }) { - return { usage: await getTagUsageStats(context.knowledgeBaseId, generateRequestId()) } + return { + usage: await getTagUsageStats( + context.knowledgeBaseId, + await context.access.get(), + generateRequestId() + ), + } }, }) diff --git a/apps/sim/lib/knowledge/connectors/member-access.test.ts b/apps/sim/lib/knowledge/connectors/member-access.test.ts index d8517d20626..6bcd5f50a7c 100644 --- a/apps/sim/lib/knowledge/connectors/member-access.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-access.test.ts @@ -35,6 +35,16 @@ vi.mock('@/lib/resource-policies/repository', async () => { vi.mock('@/lib/credential-groups/credentials', () => ({ loadManagedCredentialGroupBinding: mocks.loadBinding, listCredentialGroupOptionCredentialReferences: mocks.listOptionCredentials, + isManagedCredentialGroupBindingLive: (binding: { + managedOauthStatus: string + enrollmentStatus: string + groupStatus: string + optionStatus: string | null + }) => + binding.managedOauthStatus === 'active' && + (binding.enrollmentStatus === 'in_progress' || binding.enrollmentStatus === 'completed') && + binding.groupStatus === 'active' && + binding.optionStatus === 'active', })) vi.mock('@/lib/credentials/managed-oauth', () => ({ @@ -44,6 +54,7 @@ vi.mock('@/lib/credentials/managed-oauth', () => ({ import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' import { CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT } from '@/lib/credential-groups/workflow-access-limits' import { + findListingCapViolation, grantKnowledgeConnectorCredentialAccess, KnowledgeConnectorMemberAccessDeniedError, listKnowledgeConnectorMemberCredentials, @@ -279,6 +290,9 @@ describe('knowledge connector member access', () => { credentialGroupId: GROUP_ID, credentialGroupOptionId: 'option-drive', managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', }) mocks.resolveManagedOAuthToken.mockResolvedValue({ accessToken: 'token', refreshed: false }) }) @@ -342,6 +356,32 @@ describe('knowledge connector member access', () => { expect(mocks.resolveManagedOAuthToken).not.toHaveBeenCalled() }) + it.each([ + ['a revoked enrollment', { enrollmentStatus: 'revoked' }], + ['a disabled option', { optionStatus: 'disabled' }], + ['a removed option', { optionStatus: null }], + ['a disabled group', { groupStatus: 'disabled' }], + ['a credential needing re-auth', { managedOauthStatus: 'needs_reauth' }], + ] as const)('denies %s before consulting any policy', async (_name, overrides) => { + mocks.loadBinding.mockResolvedValue({ + credentialId: 'credential-1', + workspaceId: 'workspace-1', + providerId: 'google-drive', + credentialGroupId: GROUP_ID, + credentialGroupOptionId: 'option-drive', + managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', + ...overrides, + }) + + await expect(mintKnowledgeConnectorMemberToken(mintInput)).rejects.toBeInstanceOf( + KnowledgeConnectorMemberAccessDeniedError + ) + expect(mocks.requireResourcePolicy).not.toHaveBeenCalled() + }) + it('denies a credential from another workspace before consulting any policy', async () => { mocks.loadBinding.mockResolvedValue({ credentialId: 'credential-1', @@ -350,6 +390,9 @@ describe('knowledge connector member access', () => { credentialGroupId: GROUP_ID, credentialGroupOptionId: 'option-drive', managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', }) await expect(mintKnowledgeConnectorMemberToken(mintInput)).rejects.toBeInstanceOf( @@ -484,3 +527,18 @@ describe('knowledge connector member access', () => { }) }) }) + +describe('findListingCapViolation', () => { + const meta = { + permissionScopedListing: { capFieldIds: ['maxFiles'] }, + configFields: [{ id: 'maxFiles', title: 'Max Files' }], + } as never + + it.each([[undefined], [null], [''], ['0'], [0], [' 0 ']])('treats %j as unlimited', (value) => { + expect(findListingCapViolation(meta, { maxFiles: value })).toBeNull() + }) + + it.each([['5'], [5], ['abc']])('refuses %j', (value) => { + expect(findListingCapViolation(meta, { maxFiles: value })).toContain('Max Files') + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-access.ts b/apps/sim/lib/knowledge/connectors/member-access.ts index ef73cc100dd..2b2cfd71e5d 100644 --- a/apps/sim/lib/knowledge/connectors/member-access.ts +++ b/apps/sim/lib/knowledge/connectors/member-access.ts @@ -14,6 +14,7 @@ import { import { type CredentialGroupCredentialListContext, type CredentialGroupOptionCredentialReference, + isManagedCredentialGroupBindingLive, listCredentialGroupOptionCredentialReferences, loadManagedCredentialGroupBinding, } from '@/lib/credential-groups/credentials' @@ -298,6 +299,11 @@ export async function mintKnowledgeConnectorMemberToken( 'Managed credential is not enrolled in a Credential Group in this workspace' ) } + if (!isManagedCredentialGroupBindingLive(binding)) { + throw new KnowledgeConnectorMemberAccessDeniedError( + 'Managed credential is not currently usable: its enrollment, option, or group is not active' + ) + } await assertKnowledgeConnectorCredentialAccess({ workspaceId: binding.workspaceId, credentialGroupId: binding.credentialGroupId, @@ -358,12 +364,35 @@ export type KnowledgeConnectorMembersBindingValidation = | { ok: true; option: CredentialGroupOptionConfig } | { ok: false; message: string } +/** Whether a listing cap is in force. Blank, `0`, and `'0'` all mean unlimited. */ function isCapFieldSet(value: unknown): boolean { if (value === undefined || value === null) return false - if (typeof value === 'string') return value.trim().length > 0 + if (typeof value === 'number') return value > 0 + if (typeof value === 'string') { + const trimmed = value.trim() + if (trimmed.length === 0) return false + const parsed = Number(trimmed) + return !(Number.isFinite(parsed) && parsed <= 0) + } return true } +/** + * The source config without its listing caps. A cap has no meaning once a + * connector syncs per member, so the switch clears it rather than refusing a + * connector the admin can no longer see the field on. + */ +export function stripListingCapFields( + connectorMeta: Pick, + sourceConfig: Record +): Record { + const capFieldIds = connectorMeta.permissionScopedListing?.capFieldIds ?? [] + if (capFieldIds.length === 0) return sourceConfig + const stripped = { ...sourceConfig } + for (const fieldId of capFieldIds) delete stripped[fieldId] + return stripped +} + /** * The message refusing a source config that caps a per-member listing, or * null when nothing caps it. A cap would hide part of a member's corpus and diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index 188c0066d66..ecefbb838bf 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -178,7 +178,13 @@ export async function materializeDocumentAcls( const rows = await db .update(document) .set({ acl: observedAcl() }) - .where(and(inArray(document.id, batch), eq(document.connectorId, connectorId))) + .where( + and( + inArray(document.id, batch), + eq(document.connectorId, connectorId), + sql`${document.acl} IS DISTINCT FROM ${observedAcl()}` + ) + ) .returning({ id: document.id }) updated += rows.length } @@ -324,10 +330,13 @@ export interface StaleMemberSweepResult { * * Fail-closed but schedule-relative: an active member is swept only when both * their last start and their last complete listing are older than - * `max(24 h, 2 × interval)`, so queue lag in a large group never trips it, and - * a suspended member only once they have been suspended past the same window. - * The member row survives; the next run that lists for them rebuilds their - * observations. Purging is left to a run holding the lease. + * `max(24 h, 2 × interval)`, so queue lag in a large group never trips it. A + * suspended member is not swept at all: suspension already drops their token + * from every ACL, and their observations are kept so a re-auth restores access + * without a re-crawl until membership reconciliation purges the row after + * `MEMBER_SUSPENDED_PURGE_DAYS`. The member row survives; the next run that + * lists for them rebuilds their observations. Purging is left to a run holding + * the lease. */ export async function sweepStaleMemberObservations(now: Date): Promise { const staleWindow = sql`GREATEST( @@ -353,23 +362,15 @@ export async function sweepStaleMemberObservations(now: Date): Promise { const group = await loadCredentialGroupCredentialListContext(binding.credentialGroupId) - const option = group?.options.find( - (candidate) => candidate.id === binding.credentialGroupOptionId - ) + if (!group) { + throw new MemberBindingGoneError( + 'The Credential Group this connector synced through was deleted' + ) + } + const option = group.options.find((candidate) => candidate.id === binding.credentialGroupOptionId) + if (!option) { + throw new MemberBindingGoneError( + 'The Credential Group option this connector synced through was removed' + ) + } const optionState = { - groupActive: group?.status === 'active', - optionActive: option?.status === 'active', + groupActive: group.status === 'active', + optionActive: option.status === 'active', } const snapshots = new Map() @@ -623,6 +646,12 @@ async function claimNextMember(run: MemberSyncRun): Promise { * a member this run claimed but could not finish is re-armed for now, and the * immediate re-dispatch this count triggers is what lets them finish. */ +/** + * Members with a due timestamp, which is what re-dispatch waits for. A NULL + * `nextAttemptAt` means "with the connector's next run" — a new member, or a + * completed one on a manual-only connector — and must not keep the connector + * re-dispatching itself; only an explicit time that has passed does that. + */ async function countDueMembers(run: MemberSyncRun): Promise { const [row] = await db .select({ count: sql`count(*)::int` }) @@ -631,10 +660,7 @@ async function countDueMembers(run: MemberSyncRun): Promise { and( eq(knowledgeConnectorMember.connectorId, run.connectorId), eq(knowledgeConnectorMember.status, 'active'), - or( - isNull(knowledgeConnectorMember.nextAttemptAt), - lte(knowledgeConnectorMember.nextAttemptAt, new Date()) - ) + lte(knowledgeConnectorMember.nextAttemptAt, new Date()) ) ) return row?.count ?? 0 @@ -672,11 +698,13 @@ interface MemberListing { documents: ExternalDocument[] removedExternalIds: string[] complete: boolean + /** See {@link MemberListingOutcome.resumable}. */ + resumable: boolean /** The source itself said this member reaches nothing; not a listing shape to doubt. */ authoritative: boolean startedAt: Date - /** Cursor to store once the listing lands; undefined leaves the member's cursor alone. */ - changeCursor: string | undefined + /** Cursor to store once the listing lands: a value, null to close the feed, undefined to leave it. */ + changeCursor: string | null | undefined } async function listForMember(input: { @@ -720,18 +748,22 @@ async function listForMember(input: { }) } catch (error) { if (connectorConfig.isChangeCursorInvalidError?.(error) !== true) throw error - logger.info('Member change feed cursor expired; reopening it from a full listing', { + logger.warn('Member change feed cursor rejected; reopening it from a full listing', { connectorId: run.connectorId, memberId: member.id, + error: getErrorMessage(error), }) return listForMember({ ...input, forceFull: true }) } + const complete = pass.exhausted && !pass.budgetAborted return { kind: 'listed', mode: 'changes', documents: pass.upserts, removedExternalIds: pass.removedExternalIds, - complete: pass.exhausted && !pass.budgetAborted, + complete, + /** The cursor already sits past every page read, so the next run continues. */ + resumable: !complete, authoritative: false, startedAt, changeCursor: pass.cursor, @@ -777,6 +809,8 @@ async function listForMember(input: { documents: listing.documents, removedExternalIds: [], complete, + /** Only the deadline is worth retrying at once; a capped or truncated source reads the same next time. */ + resumable: listing.budgetAborted, authoritative: false, startedAt, changeCursor: full && complete ? openedCursor : undefined, @@ -797,9 +831,11 @@ async function listForMember(input: { documents: [], removedExternalIds: [], complete: true, + resumable: false, authoritative: true, startedAt, - changeCursor: undefined, + /** A feed over a scope the member cannot reach says nothing; the next full listing reopens one. */ + changeCursor: null, } } logger.warn('Member listing failed', { @@ -837,7 +873,13 @@ async function applyMemberListing( await db.transaction(async (tx) => { const added = await recordMemberObservations(tx, outcome.member.id, seenDocumentIds, run.runId) run.result.observationsAdded += added - if (added > 0) for (const documentId of seenDocumentIds) affected.add(documentId) + /** + * Every seen document is rematerialised, not only the newly observed ones: + * a run that died between writing observations and writing ACLs left them + * hidden, and the observation graph is the only record that says so. + * Rematerialising an already-correct ACL is a no-op write. + */ + for (const documentId of seenDocumentIds) affected.add(documentId) if (removesAllowed) { const removed = await removeUnseenMemberObservations(tx, outcome.member.id, run.runId) run.result.observationsRemoved += removed.length @@ -862,7 +904,9 @@ async function applyMemberListing( consecutiveFailures: 0, lastError: null, ...(outcome.mode === 'full' ? { lastListedCount: outcome.listedCount } : {}), - nextAttemptAt: outcome.complete ? nextMemberSyncTime(now, syncIntervalMinutes, false) : now, + nextAttemptAt: outcome.resumable + ? now + : nextMemberSyncTime(now, syncIntervalMinutes, false), ...(removesAllowed ? { lastCompleteListingAt: now, memberSyncedThrough: outcome.listingStartedAt } : {}), @@ -870,7 +914,10 @@ async function applyMemberListing( ? { memberSyncedThrough: outcome.listingStartedAt } : {}), ...(outcome.changeCursor !== undefined - ? { changeCursor: outcome.changeCursor, changeCursorAt: now } + ? { + changeCursor: outcome.changeCursor, + changeCursorAt: outcome.changeCursor === null ? null : now, + } : {}), updatedAt: now, }) @@ -909,7 +956,20 @@ async function completeMemberSync( .from(knowledgeBase) .where(and(eq(knowledgeBase.id, run.knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .for('update') - if (!activeKnowledgeBase) return false + if (!activeKnowledgeBase) { + /** Nothing to record against a deleted knowledge base; hand the lease back rather than let it expire as a failure. */ + await tx + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'idle', + nextMemberSyncAt: null, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where(stillHoldsMemberSyncLock(run.connectorId, run.runId)) + return false + } const [held] = await tx .select({ id: knowledgeConnector.id }) .from(knowledgeConnector) @@ -1157,6 +1217,13 @@ export async function executeMemberSync( error: 'Credential Groups are not available', } } + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId: run.workspaceId }))) { + await disableMemberSync(run, 'Per-member access is not enabled for this workspace') + return { + ...skipped(result, 'connector_not_syncable'), + error: 'Per-member access is not enabled', + } + } if (!connector.credentialGroupId || !connector.credentialGroupOptionId) { await disableMemberSync(run, 'Connector is no longer attached to a Credential Group option') return { @@ -1254,6 +1321,7 @@ export async function executeMemberSync( removedExternalIds: listed.removedExternalIds, listedCount: admitted.seenExternalIds.size, complete: listed.complete, + resumable: listed.resumable, suspect, /** A doubted listing does not open the feed either: the next full listing decides. */ changeCursor: suspect ? undefined : listed.changeCursor, @@ -1369,6 +1437,10 @@ export async function executeMemberSync( await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch(() => undefined) return skipped(result, 'connector_deleted_during_sync') } + if (error instanceof MemberBindingGoneError) { + await disableMemberSync(run, error.message) + return { ...skipped(result, 'connector_not_syncable'), error: error.message } + } const errorMessage = toError(error).message const retryAfterMs = getRetryAfterMs(error) diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.ts index 7b108cfacfa..868d809b217 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.ts @@ -4,7 +4,6 @@ * so the persisted column is derived here rather than in each of them. */ const SOURCE_MODIFIED_AT_KEYS = [ - 'sourceModifiedAt', 'modifiedTime', 'lastModified', 'lastModifiedDateTime', diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index ae6cdafbbca..1ab922bbc5c 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -2687,6 +2687,8 @@ describe('completeSuccessfulSync', () => { queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) queueTableRows(schemaMock.document, [{ count: 4 }]) dbChainMockFns.returning + /** The workspace ACL restore finds nothing drifted. */ + .mockResolvedValueOnce([]) .mockResolvedValueOnce([{ id: 'log-1' }]) .mockResolvedValueOnce([{ id: 'c-1' }]) @@ -2724,7 +2726,7 @@ describe('completeSuccessfulSync', () => { queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) queueTableRows(schemaMock.document, [{ count: 4 }]) - dbChainMockFns.returning.mockResolvedValueOnce([]) + dbChainMockFns.returning.mockResolvedValueOnce([]).mockResolvedValueOnce([]) await expect(completeSuccessfulSync('c-1', 'kb-1', 'log-1', 60, RESULT, null)).resolves.toBe( false diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index ee4c1dbd422..06c76c14ad0 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -211,6 +211,21 @@ export async function completeSuccessfulSync( .for('update') if (!lockedConnector) throw new SyncCompletionOwnershipLost() + /** + * Self-healing invariant of workspace mode: a mode switch back from + * members that was interrupted, or any other drift, leaves no document + * of this connector hidden from the workspace once a sync completes. + * Inside the completion transaction, after the lock is proven held, so a + * reclaimed run cannot rewrite a connector that has since changed mode. + */ + const restoredAcls = await restoreWorkspaceDocumentAcls(tx, connectorId) + if (restoredAcls > 0) { + logger.warn('Restored workspace access on connector documents that had drifted', { + connectorId, + restoredAcls, + }) + } + const [{ count: actualDocCount }] = await tx .select({ count: sql`count(*)::int` }) .from(document) @@ -247,15 +262,17 @@ export async function completeSuccessfulSync( const [writtenConnector] = await tx .update(knowledgeConnector) - .set( - buildSyncSuccessUpdate( + .set({ + ...buildSyncSuccessUpdate( now, actualDocCount, calculateNextSyncTime(syncIntervalMinutes), reconciliationHoldNotice, result.docsFailed === 0 - ) - ) + ), + /** Restored above, under this same lock. */ + accessRewritePending: false, + }) .where(stillHoldsSyncLock(connectorId, syncLogId)) .returning({ id: knowledgeConnector.id }) if (!writtenConnector) throw new SyncCompletionOwnershipLost() @@ -666,6 +683,7 @@ export async function executeSync( .set(buildSyncLockAcquisition(syncLogId, new Date())) .where( and( + eq(knowledgeConnector.accessMode, 'workspace'), eq(knowledgeConnector.id, connectorId), inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), /** @@ -934,19 +952,6 @@ export async function executeSync( lease, }) - /** - * Self-healing invariant of workspace mode: a mode switch back from - * members that was interrupted, or any other drift, leaves no document of - * this connector hidden from the workspace once a sync completes. - */ - const restoredAcls = await restoreWorkspaceDocumentAcls(connectorId) - if (restoredAcls > 0) { - logger.warn('Restored workspace access on connector documents that had drifted', { - connectorId, - restoredAcls, - }) - } - const completionLanded = await completeSuccessfulSync( connectorId, connector.knowledgeBaseId, diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index ef00de21593..0d9b3b869cb 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -1,10 +1,11 @@ import { db } from '@sim/db' -import { document, embedding, knowledgeBase } from '@sim/db/schema' +import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, eq, isNull, sql } from 'drizzle-orm' +import { and, eq, exists, isNull, sql } from 'drizzle-orm' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import type { DbOrTx } from '@/lib/db/types' import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' import type { DocumentData } from '@/lib/knowledge/documents/service' @@ -41,7 +42,10 @@ function updatedDocumentAcl(access: SyncDocumentAccess): { acl?: string[] } { * whatever a mode switch or an interrupted rewrite left behind. Idempotent and * a no-op on a healthy connector. */ -export async function restoreWorkspaceDocumentAcls(connectorId: string): Promise { +export async function restoreWorkspaceDocumentAcls( + executor: DbOrTx, + connectorId: string +): Promise { /** * The comparison array is assembled from scalar binds: the shared pool runs * with `fetch_types: false`, under which a JS array bound as one parameter @@ -52,10 +56,26 @@ export async function restoreWorkspaceDocumentAcls(connectorId: string): Promise WORKSPACE_ACL.map((token) => sql`${token}`), sql`, ` )}]::text[]` - const restored = await db + const restored = await executor .update(document) .set({ acl: [...WORKSPACE_ACL] }) - .where(and(eq(document.connectorId, connectorId), sql`${document.acl} <> ${workspaceAcl}`)) + .where( + and( + eq(document.connectorId, connectorId), + sql`${document.acl} <> ${workspaceAcl}`, + exists( + executor + .select({ one: sql`1` }) + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.accessMode, 'workspace') + ) + ) + ) + ) + ) .returning({ id: document.id }) return restored.length } @@ -243,8 +263,8 @@ export async function persistSkippedDocuments( existingId?: string extDoc: ExternalDocument }>, - sourceConfig?: Record, - access: SyncDocumentAccess = 'workspace' + sourceConfig: Record | undefined, + access: SyncDocumentAccess ): Promise { if (skipOps.length === 0) { return 0 @@ -304,6 +324,7 @@ export async function persistSkippedDocuments( storageKey: skipped.storageKey, fileSize: skipped.fileSize, mimeType: skipped.mimeType, + sourceModifiedAt: skipped.sourceModifiedAt, processingStatus: skipped.processingStatus, processingError: skipped.processingError, processingStartedAt: null, @@ -395,8 +416,8 @@ export async function addDocument( connectorType: string, extDoc: ExternalDocument, kbOwner: KnowledgeBaseOwner, - sourceConfig?: Record, - access: SyncDocumentAccess = 'workspace' + sourceConfig: Record | undefined, + access: SyncDocumentAccess ): Promise { const documentId = generateId() const artifact = connectorStoredArtifact(extDoc) @@ -492,8 +513,8 @@ export async function updateDocument( connectorType: string, extDoc: ExternalDocument, kbOwner: KnowledgeBaseOwner, - sourceConfig?: Record, - access: SyncDocumentAccess = 'workspace' + sourceConfig: Record | undefined, + access: SyncDocumentAccess ): Promise { const existingRows = await db .select({ fileUrl: document.fileUrl }) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index a4266757492..5322726a991 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -3035,6 +3035,7 @@ export async function bulkDocumentOperationByFilter( knowledgeBaseId: string, operation: 'enable' | 'disable' | 'delete', enabledFilter: 'all' | 'enabled' | 'disabled' | undefined, + access: KnowledgeAccessScope, requestId: string ): Promise<{ success: boolean @@ -3054,6 +3055,8 @@ export async function bulkDocumentOperationByFilter( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + /** "Every document" means every document the caller can see. */ + knowledgeAccessCondition(access), ] if (enabledFilter === 'enabled') { diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts index bcff731f977..44cd5ea9aca 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts @@ -11,6 +11,7 @@ const mocks = vi.hoisted(() => ({ loadGroup: vi.fn(), dispatchSync: vi.fn(), dispatchMemberSync: vi.fn(), + memberAccessAvailable: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -33,10 +34,14 @@ vi.mock('@/lib/knowledge/connectors/member-access', () => ({ revokeKnowledgeConnectorCredentialAccess: mocks.revoke, validateKnowledgeConnectorMembersBinding: mocks.validateBinding, findListingCapViolation: vi.fn(() => null), + stripListingCapFields: (_meta: unknown, sourceConfig: Record) => sourceConfig, })) vi.mock('@/lib/credential-groups/credentials', () => ({ loadCredentialGroupCredentialListContext: mocks.loadGroup, })) +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: mocks.memberAccessAvailable, +})) vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mocks.dispatchSync })) vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ dispatchMemberSync: mocks.dispatchMemberSync, @@ -95,13 +100,35 @@ function switchTo(target: Parameters { - beforeEach(() => vi.clearAllMocks()) + beforeEach(() => { + vi.clearAllMocks() + mocks.memberAccessAvailable.mockResolvedValue(true) + }) + + it('refuses members mode where the feature is off, before loading anything', async () => { + mocks.memberAccessAvailable.mockResolvedValue(false) + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + actingUserId: 'admin-1', + connectorMeta: {} as never, + binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + sourceConfig: {}, + }) + ).rejects.toMatchObject({ message: 'Per-member access is not available for this workspace' }) + expect(mocks.memberAccessAvailable).toHaveBeenCalledWith({ + workspaceId: 'ws-1', + userId: 'admin-1', + }) + expect(mocks.loadGroup).not.toHaveBeenCalled() + }) it('refuses a group from another workspace before validating anything', async () => { mocks.loadGroup.mockResolvedValue({ workspaceId: 'ws-2', status: 'active', options: [] }) await expect( resolveKnowledgeConnectorMembersBinding({ workspaceId: 'ws-1', + actingUserId: 'admin-1', connectorMeta: {} as never, binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, sourceConfig: {}, @@ -116,6 +143,7 @@ describe('resolveKnowledgeConnectorMembersBinding', () => { await expect( resolveKnowledgeConnectorMembersBinding({ workspaceId: 'ws-1', + actingUserId: 'admin-1', connectorMeta: {} as never, binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, sourceConfig: { maxFiles: '5' }, diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index 0fc4251ef2a..4b4134b400d 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -1,16 +1,19 @@ import { db } from '@sim/db' import { document, knowledgeConnector, knowledgeConnectorMember } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { loadCredentialGroupCredentialListContext } from '@/lib/credential-groups/credentials' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' import { grantKnowledgeConnectorCredentialAccess, revokeKnowledgeConnectorCredentialAccess, + stripListingCapFields, validateKnowledgeConnectorMembersBinding, } from '@/lib/knowledge/connectors/member-access' import { @@ -28,6 +31,14 @@ import type { ConnectorMeta } from '@/connectors/types' const logger = createLogger('KnowledgeConnectorAccessOrchestration') +/** The switch lease was taken away between acquiring it and writing the flip. */ +class SwitchLeaseLostError extends Error { + constructor() { + super('Connector changed during the switch') + this.name = 'SwitchLeaseLostError' + } +} + /** Documents rewritten per statement while switching modes. */ const ACCESS_REWRITE_BATCH_SIZE = 1000 /** Wall-clock the request spends rewriting before handing the rest to the member run. */ @@ -51,6 +62,8 @@ export interface KnowledgeConnectorMembersBinding { export interface ResolvedMembersBinding extends KnowledgeConnectorMembersBinding { workspaceId: string + /** The connector's source config with the listing caps cleared, which members mode stores. */ + sourceConfig: Record } /** @@ -60,22 +73,36 @@ export interface ResolvedMembersBinding extends KnowledgeConnectorMembersBinding */ export async function resolveKnowledgeConnectorMembersBinding(input: { workspaceId: string + /** The signed-in admin, for the feature gate's platform-admin clause. */ + actingUserId: string | undefined connectorMeta: Pick binding: KnowledgeConnectorMembersBinding sourceConfig: Record }): Promise { + if ( + !(await isKnowledgeMemberAccessAvailable({ + workspaceId: input.workspaceId, + userId: input.actingUserId, + })) + ) { + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) + } const group = await loadCredentialGroupCredentialListContext(input.binding.credentialGroupId) if (!group || group.workspaceId !== input.workspaceId) { throw new OrchestrationError('validation', 'Credential Group was not found in this workspace') } + const sourceConfig = stripListingCapFields(input.connectorMeta, input.sourceConfig) const validation = validateKnowledgeConnectorMembersBinding({ connectorMeta: input.connectorMeta, group, credentialGroupOptionId: input.binding.credentialGroupOptionId, - sourceConfig: input.sourceConfig, + sourceConfig, }) if (!validation.ok) throw new OrchestrationError('validation', validation.message) - return { ...input.binding, workspaceId: input.workspaceId } + return { ...input.binding, workspaceId: input.workspaceId, sourceConfig } } /** @@ -188,12 +215,19 @@ export type PerformUpdateKnowledgeConnectorAccessResult = KnowledgeOrchestration }> /** - * Moves a connector between access modes: rewrite the documents' ACLs to the - * new mode's default, then flip the mode, all under the connector's content - * lease so neither engine runs against a half-rewritten corpus. A rewrite that - * outgrows the request budget is left for the member run to finish before the - * mode takes effect (`accessRewritePending`); documents are only ever hidden - * early, never shown early. + * Moves a connector between access modes under the connector's content lease, + * so neither engine runs against a half-rewritten corpus. + * + * Into members mode: grant the option's credentials first (a reversible policy + * write), rewrite every ACL to nobody, then flip. A rewrite that outgrows the + * request budget is finished by the first member run before it lists + * (`accessRewritePending`); documents are hidden early, never shown early. + * + * Back to workspace mode: rewrite every ACL to the workspace first — the admin + * asked for exactly that visibility, and a rewrite interrupted here is + * corrected by the still-members-mode engine — then drop the members and flip + * in one transaction, then revoke the grant. A rewrite that outgrows the budget + * is finished by the next content sync (`accessRewritePending`). */ export async function performUpdateKnowledgeConnectorAccess( params: PerformUpdateKnowledgeConnectorAccessParams @@ -215,21 +249,35 @@ export async function performUpdateKnowledgeConnectorAccess( return { success: true, connector, changed: false } } + /** + * Staying in workspace mode with a different credential is a plain credential + * change: no document's visibility moves, so nothing needs the lease. + */ + if (target.accessMode === 'workspace' && existing.accessMode === 'workspace') { + const now = new Date() + const [updated] = await db + .update(knowledgeConnector) + .set({ credentialId: target.credentialId, updatedAt: now }) + .where(and(eq(knowledgeConnector.id, connectorId), isNull(knowledgeConnector.deletedAt))) + .returning() + if (!updated) return fail('Connector not found', 'not_found') + const { encryptedApiKey: _secret, ...connector } = updated + return { success: true, connector, changed: true } + } + const switchId = generateId() /** * The status to restore is the one the row had before the lease, which the - * lease itself asserts: a row returned by the lease update already reads - * `syncing`, and a status that moved between the read and the lease makes - * the lease fail rather than be restored wrongly. + * lease itself asserts: a status that moved between the read and the lease + * makes the lease fail rather than be restored wrongly. */ - const previousStatus = existing.status === 'pending' ? 'active' : existing.status + const previousStatus = existing.status const leased = await acquireSwitchLease(connectorId, kb.id, switchId, existing.status) if (!leased) return fail('Sync already in progress', 'conflict') const deadlineAt = Date.now() + ACCESS_REWRITE_REQUEST_BUDGET_MS try { if (target.accessMode === 'members') { - const rewritten = await rewriteConnectorAcls(connectorId, EMPTY_ACL, deadlineAt) await grantKnowledgeConnectorCredentialAccess( { workspaceId: kb.workspaceId, @@ -239,33 +287,96 @@ export async function performUpdateKnowledgeConnectorAccess( }, params.userId ) - if ( - existing.credentialGroupId && - existing.credentialGroupId !== target.binding.credentialGroupId - ) { + try { + const rewritten = await rewriteConnectorAcls(connectorId, EMPTY_ACL, deadlineAt) + const now = new Date() + const [updated] = await db + .update(knowledgeConnector) + .set({ + accessMode: 'members', + credentialId: null, + credentialGroupId: target.binding.credentialGroupId, + credentialGroupOptionId: target.binding.credentialGroupOptionId, + sourceConfig: target.binding.sourceConfig, + accessRewritePending: !rewritten, + memberSyncStatus: 'idle', + memberSyncConsecutiveFailures: 0, + lastMemberSyncError: null, + nextMemberSyncAt: now, + nextSyncAt: null, + status: previousStatus, + syncLockToken: null, + syncLockLeaseAt: null, + updatedAt: now, + }) + .where(switchLeaseHeld(connectorId, switchId)) + .returning() + if (!updated) throw new SwitchLeaseLostError() + if ( + existing.credentialGroupId && + existing.credentialGroupId !== target.binding.credentialGroupId + ) { + await revokeKnowledgeConnectorCredentialAccess( + { + workspaceId: kb.workspaceId, + credentialGroupId: existing.credentialGroupId, + connectorId, + }, + params.userId + ).catch((error) => { + logger.error(`[${requestId}] Failed to revoke the previous group's grant`, { + connectorId, + error: getErrorMessage(error), + }) + }) + } + logger.info(`[${requestId}] Switched connector ${connectorId} to members mode`, { + rewritten, + }) + const { encryptedApiKey: _secret, ...connector } = updated + if (previousStatus !== 'paused') { + await dispatchMemberSyncBestEffort(connectorId, params, requestId, now) + } + return { success: true, connector, changed: true } + } catch (error) { + /** The grant is the one write a failed switch must not leave behind. */ await revokeKnowledgeConnectorCredentialAccess( { workspaceId: kb.workspaceId, - credentialGroupId: existing.credentialGroupId, + credentialGroupId: target.binding.credentialGroupId, connectorId, }, params.userId - ) + ).catch((revokeError) => { + logger.error(`[${requestId}] Failed to revoke the grant of an abandoned switch`, { + connectorId, + error: getErrorMessage(revokeError), + }) + }) + throw error } - const now = new Date() - const [updated] = await db + } + + const rewritten = await rewriteConnectorAcls(connectorId, WORKSPACE_ACL, deadlineAt) + const now = new Date() + const updated = await db.transaction(async (tx) => { + await tx + .delete(knowledgeConnectorMember) + .where(eq(knowledgeConnectorMember.connectorId, connectorId)) + const [row] = await tx .update(knowledgeConnector) .set({ - accessMode: 'members', - credentialId: null, - credentialGroupId: target.binding.credentialGroupId, - credentialGroupOptionId: target.binding.credentialGroupOptionId, + accessMode: 'workspace', + credentialId: target.credentialId, + credentialGroupId: null, + credentialGroupOptionId: null, + /** The next content sync finishes a rewrite the budget cut short. */ accessRewritePending: !rewritten, memberSyncStatus: 'idle', memberSyncConsecutiveFailures: 0, lastMemberSyncError: null, - nextMemberSyncAt: now, - nextSyncAt: null, + nextMemberSyncAt: null, + nextSyncAt: now, status: previousStatus, syncLockToken: null, syncLockLeaseAt: null, @@ -273,51 +384,20 @@ export async function performUpdateKnowledgeConnectorAccess( }) .where(switchLeaseHeld(connectorId, switchId)) .returning() - if (!updated) - return fail('Connector changed during the switch; retry the request', 'conflict') - logger.info(`[${requestId}] Switched connector ${connectorId} to members mode`, { - rewritten, - }) - const { encryptedApiKey: _secret, ...connector } = updated - if (previousStatus !== 'paused') { - await dispatchMemberSyncBestEffort(connectorId, params, requestId, now) - } - return { success: true, connector, changed: true } - } - - await db - .delete(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.connectorId, connectorId)) - const rewritten = await rewriteConnectorAcls(connectorId, WORKSPACE_ACL, deadlineAt) + if (!row) throw new SwitchLeaseLostError() + return row + }) if (existing.credentialGroupId) { await revokeKnowledgeConnectorCredentialAccess( { workspaceId: kb.workspaceId, credentialGroupId: existing.credentialGroupId, connectorId }, params.userId - ) - } - const now = new Date() - const [updated] = await db - .update(knowledgeConnector) - .set({ - accessMode: 'workspace', - credentialId: target.credentialId, - credentialGroupId: null, - credentialGroupOptionId: null, - /** The content engine restores workspace access on completion; no member run will. */ - accessRewritePending: false, - memberSyncStatus: 'idle', - memberSyncConsecutiveFailures: 0, - lastMemberSyncError: null, - nextMemberSyncAt: null, - nextSyncAt: now, - status: previousStatus, - syncLockToken: null, - syncLockLeaseAt: null, - updatedAt: now, + ).catch((error) => { + logger.error(`[${requestId}] Failed to revoke the grant after leaving members mode`, { + connectorId, + error: getErrorMessage(error), + }) }) - .where(switchLeaseHeld(connectorId, switchId)) - .returning() - if (!updated) return fail('Connector changed during the switch; retry the request', 'conflict') + } logger.info(`[${requestId}] Switched connector ${connectorId} to workspace mode`, { rewritten, }) @@ -327,6 +407,9 @@ export async function performUpdateKnowledgeConnectorAccess( } return { success: true, connector, changed: true } } catch (error) { + if (error instanceof SwitchLeaseLostError) { + return fail('Connector changed during the switch; retry the request', 'conflict') + } await releaseSwitchLease(connectorId, switchId, previousStatus).catch((releaseError) => { logger.error(`[${requestId}] Failed to release the access switch lease`, { connectorId, diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index 7da695bef40..c5b4fb894de 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -893,10 +893,10 @@ export async function performDeleteKnowledgeConnector( ]) } - if (existing.credentialGroupId) { + if (existing.credentialGroupId && kb.workspaceId) { await revokeKnowledgeConnectorCredentialAccess( { - workspaceId: kb.workspaceId ?? '', + workspaceId: kb.workspaceId, credentialGroupId: existing.credentialGroupId, connectorId, }, diff --git a/apps/sim/lib/knowledge/search/defaults.ts b/apps/sim/lib/knowledge/search/defaults.ts new file mode 100644 index 00000000000..a88d914d746 --- /dev/null +++ b/apps/sim/lib/knowledge/search/defaults.ts @@ -0,0 +1,32 @@ +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import type { KnowledgeSearchMode } from '@/lib/knowledge/search/queries' + +/** How a search runs when the caller did not choose a mode. */ +export interface KnowledgeSearchDefaults { + searchMode: KnowledgeSearchMode + /** Whether a recently modified document may edge past a stale one of similar relevance. */ + boostRecency: boolean +} + +/** + * The retrieval defaults for one workspace. Where permission-aware knowledge + * is on, hybrid retrieval and the recency boost are the default; elsewhere + * search stays semantic-only with no boost, exactly as before. An explicit + * `searchMode` from the caller always wins over the default mode. + */ +export async function resolveKnowledgeSearchDefaults(input: { + workspaceId: string | undefined + userId: string | undefined + requestedMode: KnowledgeSearchMode | undefined +}): Promise { + const enabled = input.workspaceId + ? await isKnowledgeMemberAccessAvailable({ + workspaceId: input.workspaceId, + userId: input.userId, + }) + : false + return { + searchMode: input.requestedMode ?? (enabled ? 'hybrid' : 'vector'), + boostRecency: enabled, + } +} diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index a4a5bf18d89..e473e4b6d84 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -5,6 +5,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { and, eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import { RRF_K } from '@/lib/knowledge/search/rank' import { applyRecencyBoost } from '@/lib/knowledge/search/recency' import { coerceTagFilterValue, @@ -362,12 +363,6 @@ export function getStructuredTagFilters(filters: StructuredFilter[], embeddingTa */ const FTS_CONFIG = 'english' -/** - * Reciprocal-rank-fusion damping constant. 60 is the value from the original RRF - * paper and matches the docs search retriever (`apps/docs/app/api/search/route.ts`). - */ -export const RRF_K = 60 - /** * Row visibility predicates shared by every search leg: a chunk is only * retrievable when both it and its document are enabled, the document finished @@ -763,6 +758,8 @@ export interface ExecuteKnowledgeSearchParams { /** What the caller may read; resolved from the principal by the use case, never from input. */ access: KnowledgeAccessScope searchMode: KnowledgeSearchMode + /** Lets a recently modified document edge past a stale one of similar relevance; off by default. */ + boostRecency?: boolean query?: string /** Required whenever `query` is present. */ queryVector?: string @@ -777,8 +774,16 @@ export interface ExecuteKnowledgeSearchParams { export async function executeKnowledgeSearch( params: ExecuteKnowledgeSearchParams ): Promise { - const { knowledgeBaseIds, topK, searchMode, query, queryVector, structuredFilters, access } = - params + const { + knowledgeBaseIds, + topK, + searchMode, + query, + queryVector, + structuredFilters, + access, + boostRecency = false, + } = params const hasQuery = Boolean(query?.trim()) const hasFilters = Boolean(structuredFilters && structuredFilters.length > 0) @@ -808,7 +813,8 @@ export async function executeKnowledgeSearch( : handleVectorOnlySearch({ knowledgeBaseIds, topK, queryVector, distanceThreshold, access }) if (searchMode === 'vector') { - return applyRecencyBoost(await vectorSearch) + const results = await vectorSearch + return boostRecency ? applyRecencyBoost(results) : results } /** @@ -837,5 +843,6 @@ export async function executeKnowledgeSearch( * threshold is precisely what a caller opted into hybrid to recover, and at * `topK: 1` something has to win. */ - return applyRecencyBoost(fuseByReciprocalRank([keywordResults, vectorResults], topK)) + const fused = fuseByReciprocalRank([keywordResults, vectorResults], topK) + return boostRecency ? applyRecencyBoost(fused) : fused } diff --git a/apps/sim/lib/knowledge/search/rank.ts b/apps/sim/lib/knowledge/search/rank.ts new file mode 100644 index 00000000000..500316b6992 --- /dev/null +++ b/apps/sim/lib/knowledge/search/rank.ts @@ -0,0 +1,7 @@ +/** + * Reciprocal-rank-fusion damping constant, shared by fusion and the recency + * boost so a rank means the same to both: `score = 1 / (RRF_K + rank)`. 60 is + * the value from the original RRF paper and matches the docs search retriever + * (`apps/docs/app/api/search/route.ts`). + */ +export const RRF_K = 60 diff --git a/apps/sim/lib/knowledge/search/recency.ts b/apps/sim/lib/knowledge/search/recency.ts index 004f73c8613..f3d53c13dab 100644 --- a/apps/sim/lib/knowledge/search/recency.ts +++ b/apps/sim/lib/knowledge/search/recency.ts @@ -1,9 +1,9 @@ +import { RRF_K } from '@/lib/knowledge/search/rank' + /** Age at which a document's recency boost has decayed to half. */ export const RECENCY_HALF_LIFE_DAYS = 90 /** The most a fully fresh document's rank score is raised, as a fraction. */ export const RECENCY_WEIGHT = 0.15 -/** The rank-score denominator offset; shared with reciprocal-rank fusion so the two agree. */ -export const RECENCY_RANK_K = 60 const DAY_MS = 24 * 60 * 60 * 1000 @@ -33,7 +33,7 @@ export function applyRecencyBoost( const boosted = rows.map((row, index) => ({ row, score: - (1 / (RECENCY_RANK_K + index + 1)) * + (1 / (RRF_K + index + 1)) * (1 + RECENCY_WEIGHT * recencyFreshness(row.sourceModifiedAt, now)), })) return boosted.sort((a, b) => b.score - a.score).map((entry) => entry.row) diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 2fb32169da0..c8d6d9cc4fc 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -930,13 +930,14 @@ export async function updateKnowledgeBase( logger.info(`[${requestId}] Updated knowledge base: ${knowledgeBaseId}`) - return { - ...updatedKb[0], - chunkingConfig: updatedKb[0].chunkingConfig as ChunkingConfig, - docCount: Number(updatedKb[0].docCount), - connectorTypes: [], - hasMemberScopedConnector: false, - } + const [withConnectors] = await attachConnectorTypes([ + { + ...updatedKb[0], + chunkingConfig: updatedKb[0].chunkingConfig as ChunkingConfig, + docCount: Number(updatedKb[0].docCount), + }, + ]) + return withConnectors } /** @@ -1007,13 +1008,14 @@ export async function getKnowledgeBaseById( return null } - return { - ...result[0], - chunkingConfig: result[0].chunkingConfig as ChunkingConfig, - docCount: Number(result[0].docCount), - connectorTypes: [], - hasMemberScopedConnector: false, - } + const [withConnectors] = await attachConnectorTypes([ + { + ...result[0], + chunkingConfig: result[0].chunkingConfig as ChunkingConfig, + docCount: Number(result[0].docCount), + }, + ]) + return withConnectors } /** diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 3de8800e069..6d1d112cf60 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -871,6 +871,7 @@ export async function getTagUsage( */ export async function getTagUsageStats( knowledgeBaseId: string, + access: KnowledgeAccessScope, requestId: string ): Promise< Array<{ @@ -898,6 +899,7 @@ export async function getTagUsageStats( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(access), sql`${sql.raw(tagSlot)} IS NOT NULL` ) ) @@ -912,6 +914,7 @@ export async function getTagUsageStats( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(access), sql`${sql.raw(`embedding.${tagSlot}`)} IS NOT NULL` ) ) diff --git a/apps/sim/lib/workspaces/host-context.ts b/apps/sim/lib/workspaces/host-context.ts index 6fdd788d90a..2a53015de45 100644 --- a/apps/sim/lib/workspaces/host-context.ts +++ b/apps/sim/lib/workspaces/host-context.ts @@ -2,6 +2,7 @@ import { cache } from 'react' import type { WorkspaceHostContext } from '@/lib/api/contracts/workspaces' import { getWorkspaceOwnerSubscriptionAccess } from '@/lib/billing/core/workspace-access' import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { getOrganizationSettingsAccess } from '@/lib/organizations/settings-access' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' @@ -28,7 +29,10 @@ async function resolveWorkspaceHostContextForViewer( ? getOrganizationSettingsAccess(hostOrganizationId, userId) : Promise.resolve({ role: null, isMember: false, isAdmin: false }), ]) - const credentialGroupsAvailable = await isCredentialGroupsAvailable({ workspaceId, ownerBilling }) + const [credentialGroupsAvailable, knowledgeMemberAccessAvailable] = await Promise.all([ + isCredentialGroupsAvailable({ workspaceId, ownerBilling }), + isKnowledgeMemberAccessAvailable({ workspaceId, userId }), + ]) return { workspace: { @@ -48,6 +52,7 @@ async function resolveWorkspaceHostContextForViewer( }, features: { credentialGroups: credentialGroupsAvailable, + knowledgeMemberAccess: knowledgeMemberAccessAvailable, }, } } diff --git a/apps/sim/tools/generated/tool-metadata.ts b/apps/sim/tools/generated/tool-metadata.ts index adf36f35c23..8ed92264ac3 100644 --- a/apps/sim/tools/generated/tool-metadata.ts +++ b/apps/sim/tools/generated/tool-metadata.ts @@ -3,7 +3,7 @@ /** Serializable metadata for every built-in tool, keyed by tool id. */ const toolMetadata: Record = JSON.parse( - '{"a2a_cancel_task":{"id":"a2a_cancel_task","name":"A2A Cancel Task","description":"Request cancellation of an in-progress A2A task.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The task ID to cancel"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"a2a_get_agent_card":{"id":"a2a_get_agent_card","name":"A2A Get Agent Card","description":"Fetch the Agent Card (discovery document) for an external A2A agent.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"a2a_get_task":{"id":"a2a_get_task","name":"A2A Get Task","description":"Retrieve the current state and result of an A2A task.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The task ID to retrieve"},"historyLength":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of history messages to include"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"a2a_send_message":{"id":"a2a_send_message","name":"A2A Send Message","description":"Send a message to an external A2A agent and return its response.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"message":{"type":"string","required":true,"visibility":"user-or-llm","description":"The message text to send"},"data":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional structured JSON data to attach"},"files":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional files to attach"},"taskId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Existing task ID to continue"},"contextId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Conversation context ID to continue"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"affinity_batch_update_entity_fields":{"id":"affinity_batch_update_entity_fields","name":"Affinity Batch Update Entity Fields","description":"Write up to 100 non-list field values on one company or person in a single request.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to write the fields on: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"updates":{"type":"json","required":true,"visibility":"user-or-llm","description":"Up to 100 field updates as [{\\"id\\":\\"\\",\\"value\\":{\\"type\\":\\"…\\",\\"data\\":…}}], using the same value shapes as a single field update"}},"hostedApiKey":"none"},"affinity_batch_update_list_entry_fields":{"id":"affinity_batch_update_list_entry_fields","name":"Affinity Batch Update List Entry Fields","description":"Write up to 100 field values on one list row in a single request. Requires the \\"Export data from Lists\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"updates":{"type":"json","required":true,"visibility":"user-or-llm","description":"Up to 100 field updates as [{\\"id\\":\\"\\",\\"value\\":{\\"type\\":\\"…\\",\\"data\\":…}}], using the same value shapes as a single field update"}},"hostedApiKey":"none"},"affinity_create_list":{"id":"affinity_create_list","name":"Affinity Create List","description":"Create a list. Its type fixes which entities it can hold, and the API key holder becomes its creator and owner.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the new list"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Entity kind the list holds: company, opportunity, or person"},"isPublic":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether everyone in the organization can see the list"}},"hostedApiKey":"none"},"affinity_create_list_field_dropdown_option":{"id":"affinity_create_list_field_dropdown_option","name":"Affinity Create List Field Dropdown Option","description":"Add a selectable option to a dropdown field on a list. A ranked or status option also needs a rank and a color.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Kind of option to create, matching the field. dropdown takes only a label; ranked-dropdown also requires rank and color; status-dropdown additionally requires a status category. Sending a field the kind does not accept is rejected"},"text":{"type":"string","required":true,"visibility":"user-or-llm","description":"The option label"},"rank":{"type":"number","required":false,"visibility":"user-or-llm","description":"Sort order. Required on a ranked-dropdown or status-dropdown option"},"color":{"type":"string","required":false,"visibility":"user-or-llm","description":"Option color: white, gray, blue, green, purple, orange, or red. Required on a ranked-dropdown or status-dropdown option"},"statusCategory":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pipeline meaning of the option: open, won, lost, or on-hold. Status-dropdown options only"},"winRate":{"type":"number","required":false,"visibility":"user-or-llm","description":"Expected win rate of the status. Status-dropdown options only"}},"hostedApiKey":"none"},"affinity_create_merge":{"id":"affinity_create_merge","name":"Affinity Create Merge","description":"Fold a duplicate company or person into the record you are keeping. The merge runs asynchronously — poll the returned task to see it finish. Requires the \\"Manage duplicates\\" permission and an admin role.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to merge: companies or persons"},"primaryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to keep"},"duplicateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the duplicate record to fold in"}},"hostedApiKey":"none"},"affinity_create_note":{"id":"affinity_create_note","name":"Affinity Create Note","description":"Write a note — attached to companies, persons, and opportunities, anchored to a meeting, call, or chat message, or posted as a reply to an existing note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Note shape: entities to attach it to records, interaction to anchor it to a meeting, call, or chat message, or user-reply to reply to a note"},"html":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note body as HTML"},"companyIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Companies to attach the note to, e.g. [1, 2]. Not used on a reply"},"personIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Persons to attach the note to, e.g. [1, 2]. Not used on a reply"},"opportunityIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Opportunities to attach the note to, e.g. [1, 2]. Not used on a reply"},"interactionId":{"type":"string","required":false,"visibility":"user-or-llm","description":"The interaction to anchor the note to. Required for an interaction note"},"interactionType":{"type":"string","required":false,"visibility":"user-or-llm","description":"Kind of the anchoring interaction: meeting, call, or chat-message. Required for an interaction note"},"parentId":{"type":"string","required":false,"visibility":"user-or-llm","description":"The note being replied to. Required for a user-reply note"},"creatorId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Attribute the note to another internal person. Defaults to the API key holder"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"Backdate the note to this ISO 8601 timestamp"}},"hostedApiKey":"none"},"affinity_create_reminder":{"id":"affinity_create_reminder","name":"Affinity Create Reminder","description":"Create a reminder on one company, person, or opportunity. A recurring reminder resets whenever the chosen signal happens instead of firing once.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"one-time to fire once, or recurring to reset on a signal"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"What the reminder is about: company, person, or opportunity"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company, person, or opportunity"},"dueDate":{"type":"string","required":false,"visibility":"user-or-llm","description":"When the reminder is due, as an ISO 8601 timestamp. Required for a one-time reminder; on a recurring one Affinity computes it from the period when omitted"},"content":{"type":"string","required":false,"visibility":"user-or-llm","description":"What the reminder says"},"ownerId":{"type":"string","required":true,"visibility":"user-or-llm","description":"User the reminder is assigned to. Must be an internal user. The API key holder is recorded as the creator, which is a separate field"},"resetTrigger":{"type":"string","required":false,"visibility":"user-or-llm","description":"What restarts a recurring reminder: interaction, email, or event. Required when the type is recurring"},"periodDays":{"type":"number","required":false,"visibility":"user-or-llm","description":"Days between firings of a recurring reminder. Required when the type is recurring"}},"hostedApiKey":"none"},"affinity_delete_list_field_dropdown_option":{"id":"affinity_delete_list_field_dropdown_option","name":"Affinity Delete List Field Dropdown Option","description":"Permanently delete a dropdown option on a list field. Every list entry currently set to it is cleared, and those values cannot be recovered.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"dropdownOptionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown option ID to delete"}},"hostedApiKey":"none"},"affinity_delete_note":{"id":"affinity_delete_note","name":"Affinity Delete Note","description":"Delete a note you created. Deleting a root note also deletes its replies; deleting a reply removes only that reply.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID to delete"}},"hostedApiKey":"none"},"affinity_get_company":{"id":"affinity_get_company","name":"Affinity Get Company","description":"Look up one company by ID. Field data is returned only for the Field IDs or Field Types asked for.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"companyId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The company ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"}},"hostedApiKey":"none"},"affinity_get_current_user":{"id":"affinity_get_current_user","name":"Affinity Get Current User","description":"Verify an Affinity API key and return the tenant, the user behind the key, and the scopes the grant carries.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"}},"hostedApiKey":"none"},"affinity_get_entity_field_value":{"id":"affinity_get_entity_field_value","name":"Affinity Get Entity Field Value","description":"Read one non-list field value from a company or person.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to read the field from: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to read"}},"hostedApiKey":"none"},"affinity_get_list":{"id":"affinity_get_list","name":"Affinity Get List","description":"Read one list — its name, type, owner, and privacy setting.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"}},"hostedApiKey":"none"},"affinity_get_list_entry":{"id":"affinity_get_list_entry","name":"Affinity Get List Entry","description":"Read one row of a list with its entity. Field data is returned only for the Field IDs or Field Types asked for.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, list, or relationship-intelligence. Mutually exclusive with Field IDs"}},"hostedApiKey":"none"},"affinity_get_list_entry_field":{"id":"affinity_get_list_entry_field","name":"Affinity Get List Entry Field","description":"Read one field value on a list row.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to read"}},"hostedApiKey":"none"},"affinity_get_list_field_dropdown_option":{"id":"affinity_get_list_field_dropdown_option","name":"Affinity Get List Field Dropdown Option","description":"Read one dropdown option on a list field.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"dropdownOptionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown option ID"}},"hostedApiKey":"none"},"affinity_get_merge":{"id":"affinity_get_merge","name":"Affinity Get Merge","description":"Read the status of one company or person merge, including why it failed if it did.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merge to read: companies or persons"},"mergeId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The merge ID"}},"hostedApiKey":"none"},"affinity_get_merge_task":{"id":"affinity_get_merge_task","name":"Affinity Get Merge Task","description":"Read one merge task and how its merges are progressing. Poll this after starting a merge.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merge task to read: companies or persons"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The merge task ID"}},"hostedApiKey":"none"},"affinity_get_note":{"id":"affinity_get_note","name":"Affinity Get Note","description":"Read one note with its body, author, mentions, and attached records.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return, e.g. [\\"repliesCount\\",\\"personsPreview\\",\\"companiesPreview\\",\\"opportunitiesPreview\\"]. Those four fields are omitted unless requested here"}},"hostedApiKey":"none"},"affinity_get_opportunity":{"id":"affinity_get_opportunity","name":"Affinity Get Opportunity","description":"Read one opportunity and the list it belongs to. Its field data lives on the list entry.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"opportunityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The opportunity ID"}},"hostedApiKey":"none"},"affinity_get_person":{"id":"affinity_get_person","name":"Affinity Get Person","description":"Look up one person by ID. Field data is returned only for the Field IDs or Field Types asked for.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"personId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The person ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"}},"hostedApiKey":"none"},"affinity_get_saved_view":{"id":"affinity_get_saved_view","name":"Affinity Get Saved View","description":"Read one saved view — its name, kind, and creation date.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"viewId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The saved view ID"}},"hostedApiKey":"none"},"affinity_get_transcript":{"id":"affinity_get_transcript","name":"Affinity Get Transcript","description":"Read one transcript with its first 100 fragments. Page the fragments endpoint for a longer meeting.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"transcriptId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The transcript ID"}},"hostedApiKey":"none"},"affinity_get_user":{"id":"affinity_get_user","name":"Affinity Get User","description":"Read one internal user. A user and their person record share the same numeric ID, so a person ID works here.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"userId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The user ID, which is also their person ID"}},"hostedApiKey":"none"},"affinity_list_calls":{"id":"affinity_list_calls","name":"Affinity List Calls","description":"Page through logged calls and their participants. Only calls the API key holder can see are returned.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_chat_messages":{"id":"affinity_list_chat_messages","name":"Affinity List Chat Messages","description":"Page through logged chat messages and their participants. Only messages the API key holder can see are returned.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_companies":{"id":"affinity_list_companies","name":"Affinity List Companies","description":"Page through companies. Companies come back without field data unless Field IDs or Field Types asks for it.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the page to these company IDs, e.g. [1, 2, 3]"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_coworker_connections":{"id":"affinity_list_coworker_connections","name":"Affinity List Coworker Connections","description":"Find warm paths into a company through shared work history: who in your Affinity data once worked alongside the people you want to reach. Grouped by target, strongest first.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":true,"visibility":"user-or-llm","description":"Required scope. The only supported filter is target.currentCompany.id, e.g. \\"target.currentCompany.id=123\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of targets to return per page, 1-50. Defaults to 20"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_emails":{"id":"affinity_list_emails","name":"Affinity List Emails","description":"Page through email metadata — subject, participants, and timestamps. Affinity never exposes email bodies through the API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_entity_field_values":{"id":"affinity_list_entity_field_values","name":"Affinity List Entity Field Values","description":"Page through a company\'s or person\'s non-list field values. List fields are not returned here — read those through the list entry.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to read field values from: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field IDs. Mutually exclusive with Field Types"},"types":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field categories: enriched, global, relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_list_entity_list_entries":{"id":"affinity_list_entity_list_entries","name":"Affinity List Entity List Entries","description":"Page through a company\'s or person\'s rows across every list, each carrying that list\'s field values and when the entity was added.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to look up the rows of: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_entity_lists":{"id":"affinity_list_entity_lists","name":"Affinity List Entity Lists","description":"List every list a company or person appears on that the caller can view.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to look up the lists of: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_entity_notes":{"id":"affinity_list_entity_notes","name":"Affinity List Entity Notes","description":"List the notes relevant to one company, person, or opportunity — directly attached notes plus notes reaching it through its people and meetings.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity the notes hang off: companies, persons, or opportunities"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company, person, or opportunity"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_entity_relationships":{"id":"affinity_list_entity_relationships","name":"Affinity List Entity Relationships","description":"List who knows a company or person, scored 0.0 to 1.0 by how much the two actually interact. Strongest first by default.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to look up relationships for: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on interactionScore only, e.g. \\"interactionScore>=0.5\\""},"orderBy":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order: [\\"interactionScore\\"] for weakest first, [\\"-interactionScore\\"] for strongest first (the default)"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_field_dropdown_options":{"id":"affinity_list_field_dropdown_options","name":"Affinity List Field Dropdown Options","description":"List the selectable options on a dropdown or ranked-dropdown company or person field. Writing such a field needs the option ID, not its text.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which field family the field belongs to: companies or persons"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown or ranked-dropdown field ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_field_metadata":{"id":"affinity_list_field_metadata","name":"Affinity List Field Metadata","description":"List the non-list company or person fields, with the value type, filter operators, and sort support of each. Start here to find the Field IDs the read and write tools take.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which fields to describe: companies or persons"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return: [\\"filterability\\",\\"sortability\\"]. Both are omitted unless requested here"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on name only, e.g. \\"name=~Status\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_field_value_changes":{"id":"affinity_list_field_value_changes","name":"Affinity List Field Value Changes","description":"Page through field value changes across the whole workspace. Built for delta sync: follow nextCursor to the end of a run, then resume from the last cursor next time.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over field.id, listEntry.id, changer.id, changedAt, or actionType. Resume a sync with e.g. \\"changedAt>2026-06-01T12:00:00Z\\""},"orderBy":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order: [\\"changedAt\\"] for oldest first (the default), [\\"-changedAt\\"] for newest first"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_investor_executive_connections":{"id":"affinity_list_investor_executive_connections","name":"Affinity List Investor Executive Connections","description":"Find warm paths into a company through investment history: which investors in your Affinity data backed a company the people you want to reach once led. Grouped by target, strongest first.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":true,"visibility":"user-or-llm","description":"Required scope. The only supported filter is target.currentCompany.id, e.g. \\"target.currentCompany.id=123\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of targets to return per page, 1-50. Defaults to 20"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_list_entries":{"id":"affinity_list_list_entries","name":"Affinity List List Entries","description":"Page through the rows of a list. Rows come back without field data unless Field IDs or Field Types asks for it.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, list, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_list_entry_field_value_changes":{"id":"affinity_list_list_entry_field_value_changes","name":"Affinity List List Entry Field Value Changes","description":"Page through the history of one list row — who changed which field, when, and to what.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over field.id, changer.id, changedAt, or actionType, e.g. \\"field.id=field-1234\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_list_entry_fields":{"id":"affinity_list_list_entry_fields","name":"Affinity List List Entry Fields","description":"Page through every field value on one list row, including the list-specific columns. All fields are returned unless narrowed.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field IDs. Mutually exclusive with Field Types"},"types":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field categories: enriched, global, list, relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_list_list_field_dropdown_options":{"id":"affinity_list_list_field_dropdown_options","name":"Affinity List List Field Dropdown Options","description":"List the selectable options on a dropdown, ranked-dropdown, or status-dropdown field of a list.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_list_fields":{"id":"affinity_list_list_fields","name":"Affinity List List Fields","description":"List the fields available on one list, including its list-specific columns. Use these Field IDs when reading or writing list entries.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return: [\\"filterability\\",\\"sortability\\"]. Both are omitted unless requested here"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on name only, e.g. \\"name=~Stage\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_lists":{"id":"affinity_list_lists","name":"Affinity List Lists","description":"Page through the lists in the organization that the caller can view.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"term":{"type":"string","required":false,"visibility":"user-or-llm","description":"Case-insensitive substring match on the list name"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_meetings":{"id":"affinity_list_meetings","name":"Affinity List Meetings","description":"Page through past and upcoming meetings with their organizer and attendees.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_merge_tasks":{"id":"affinity_list_merge_tasks","name":"Affinity List Merge Tasks","description":"Page through merge tasks, each summarizing how many of its merges are in progress, succeeded, or failed.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merge tasks to list: companies or persons"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on status only, e.g. \\"status=in-progress\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_merges":{"id":"affinity_list_merges","name":"Affinity List Merges","description":"Page through the company or person merges the organization has run, with the status and the records involved in each.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merges to list: companies or persons"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over status or taskId, e.g. \\"status=failed\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_note_attached_companies":{"id":"affinity_list_note_attached_companies","name":"Affinity List Note Attached Companies","description":"List the companies directly attached to one note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_note_attached_opportunities":{"id":"affinity_list_note_attached_opportunities","name":"Affinity List Note Attached Opportunities","description":"List the opportunities directly attached to one note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_note_attached_persons":{"id":"affinity_list_note_attached_persons","name":"Affinity List Note Attached Persons","description":"List the persons directly attached to one note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_note_replies":{"id":"affinity_list_note_replies","name":"Affinity List Note Replies","description":"Page through the replies on one note, including AI Notetaker replies.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID whose replies to read"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_notes":{"id":"affinity_list_notes","name":"Affinity List Notes","description":"Page through every note the caller can see. Replies are excluded.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return, e.g. [\\"repliesCount\\",\\"personsPreview\\",\\"companiesPreview\\",\\"opportunitiesPreview\\"]. Those four fields are omitted unless requested here"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_opportunities":{"id":"affinity_list_opportunities","name":"Affinity List Opportunities","description":"Page through opportunities. Field data lives on the list entry, not here — read it through the list or saved view the opportunity belongs to.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the page to these opportunity IDs, e.g. [1, 2, 3]"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_persons":{"id":"affinity_list_persons","name":"Affinity List Persons","description":"Page through persons. Persons come back without field data unless Field IDs or Field Types asks for it.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the page to these person IDs, e.g. [1, 2, 3]"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_reminders":{"id":"affinity_list_reminders","name":"Affinity List Reminders","description":"Page through the reminders the caller can see. Filter by status to surface what is overdue.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_saved_view_entries":{"id":"affinity_list_saved_view_entries","name":"Affinity List Saved View Entries","description":"Page through the rows of a saved view. The view\'s own filters and columns decide which rows and which field data come back.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"viewId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The saved view ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_saved_views":{"id":"affinity_list_saved_views","name":"Affinity List Saved Views","description":"List the saved views on a list that the caller can view.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_transcript_fragments":{"id":"affinity_list_transcript_fragments","name":"Affinity List Transcript Fragments","description":"Page through everything said in a meeting, segment by segment with the speaker.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"transcriptId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The transcript ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_transcripts":{"id":"affinity_list_transcripts","name":"Affinity List Transcripts","description":"Page through meeting transcript metadata. Read one transcript to get what was actually said.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_users":{"id":"affinity_list_users","name":"Affinity List Users","description":"Page through the internal users in the organization. Email addresses and roles are returned only to callers with the \\"Manage Users\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"term":{"type":"string","required":false,"visibility":"user-or-llm","description":"Case-insensitive match across first name, last name, and primary email"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over id or status, e.g. \\"status=active\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_search_companies":{"id":"affinity_search_companies","name":"Affinity Search Companies","description":"Search companies by filters, sorts, and a free-text term. Requires the \\"Export All Organizations directory\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Filter group as {operator: \\"and\\"|\\"or\\", filters: [...]}, at most 50 leaves. Each leaf is {valueType, fieldId, operator, value}, and a leaf may itself be a nested group"},"searchTerm":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-text term matched against the searchable fields. At least 3 characters"},"searchFieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs the search term is matched against. Defaults to the searchable fields"},"sorts":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order as [{fieldId, direction: \\"asc\\"|\\"desc\\", attributeId?}], up to 5, applied in order"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_search_files":{"id":"affinity_search_files","name":"Affinity Search Files","description":"Search files by keyword, ordered by relevance. Narrow to specific files or to one company, or leave both unset to search the whole account.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"prompt":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to search for. Between 3 and 500 characters"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the search to these file IDs. Cannot be combined with Company ID"},"companyId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Restrict the search to one company\'s files. Cannot be combined with file IDs"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of files to return, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_search_list_entries":{"id":"affinity_search_list_entries","name":"Affinity Search List Entries","description":"Search the rows of one list by filters, sorts, and a free-text term. Requires the \\"Export data from Lists\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID to search"},"filters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Filter group as {operator: \\"and\\"|\\"or\\", filters: [...]}, at most 50 leaves. Each leaf is {valueType, fieldId, operator, value}, and a leaf may itself be a nested group"},"searchTerm":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-text term matched against the searchable fields. At least 3 characters"},"searchFieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs the search term is matched against. Defaults to the searchable fields"},"sorts":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order as [{fieldId, direction: \\"asc\\"|\\"desc\\", attributeId?}], up to 5, applied in order"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, list, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_search_notes":{"id":"affinity_search_notes","name":"Affinity Search Notes","description":"Search notes by keyword, ordered by relevance. Narrow to specific notes or to one company, or leave both unset to search the whole account.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"prompt":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to search for. Between 3 and 500 characters"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the search to these note IDs. Cannot be combined with Company ID"},"companyId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Restrict the search to one company\'s notes. Cannot be combined with note IDs"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of notes to return, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_search_persons":{"id":"affinity_search_persons","name":"Affinity Search Persons","description":"Search persons by filters, sorts, and a free-text term. Requires the \\"Export All People directory\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Filter group as {operator: \\"and\\"|\\"or\\", filters: [...]}, at most 50 leaves. Each leaf is {valueType, fieldId, operator, value}, and a leaf may itself be a nested group"},"searchTerm":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-text term matched against the searchable fields. At least 3 characters"},"searchFieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs the search term is matched against. Defaults to the searchable fields"},"sorts":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order as [{fieldId, direction: \\"asc\\"|\\"desc\\", attributeId?}], up to 5, applied in order"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_semantic_search":{"id":"affinity_semantic_search","name":"Affinity Semantic Search","description":"Find companies from a description in plain language — industry, technology, stage, or business model. Currently searches companies only.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"prompt":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to look for, in plain language, e.g. \\"climate tech companies in our pipeline\\". Up to 500 characters"},"listIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the search to companies on these lists, e.g. [1, 2]"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of companies to return, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_update_entity_field_value":{"id":"affinity_update_entity_field_value","name":"Affinity Update Entity Field Value","description":"Write one non-list field value on a company or person. The value type must match how the field is defined.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to write the field on: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to write"},"value":{"type":"json","required":true,"visibility":"user-or-llm","description":"The new value as {type, data}, where type matches the field\'s value type. Examples: {\\"type\\":\\"text\\",\\"data\\":\\"Series B\\"}, {\\"type\\":\\"number\\",\\"data\\":42}, {\\"type\\":\\"dropdown\\",\\"data\\":{\\"dropdownOptionId\\":7}}, {\\"type\\":\\"person\\",\\"data\\":{\\"id\\":123}}, {\\"type\\":\\"person-multi\\",\\"data\\":[{\\"id\\":123}]}. Pass data as null to clear the field"}},"hostedApiKey":"none"},"affinity_update_list_entry_field":{"id":"affinity_update_list_entry_field","name":"Affinity Update List Entry Field","description":"Write one field value on a list row. Requires the \\"Export data from Lists\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to write"},"value":{"type":"json","required":true,"visibility":"user-or-llm","description":"The new value as {type, data}, where type matches the field\'s value type. Examples: {\\"type\\":\\"text\\",\\"data\\":\\"Series B\\"}, {\\"type\\":\\"number\\",\\"data\\":42}, {\\"type\\":\\"dropdown\\",\\"data\\":{\\"dropdownOptionId\\":7}}, {\\"type\\":\\"person\\",\\"data\\":{\\"id\\":123}}, {\\"type\\":\\"person-multi\\",\\"data\\":[{\\"id\\":123}]}. Pass data as null to clear the field"}},"hostedApiKey":"none"},"affinity_update_list_field_dropdown_option":{"id":"affinity_update_list_field_dropdown_option","name":"Affinity Update List Field Dropdown Option","description":"Change a dropdown option on a list field. Every field is optional — supply only what should change, and only fields the option\'s kind actually has.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"dropdownOptionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown option ID to update"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Replacement option label. Supply at least one field to change"},"rank":{"type":"number","required":false,"visibility":"user-or-llm","description":"Sort order. Required on a ranked-dropdown or status-dropdown option"},"color":{"type":"string","required":false,"visibility":"user-or-llm","description":"Option color: white, gray, blue, green, purple, orange, or red. Required on a ranked-dropdown or status-dropdown option"},"statusCategory":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pipeline meaning of the option: open, won, lost, or on-hold. Status-dropdown options only"},"winRate":{"type":"number","required":false,"visibility":"user-or-llm","description":"Expected win rate of the status. Status-dropdown options only"}},"hostedApiKey":"none"},"affinity_update_note":{"id":"affinity_update_note","name":"Affinity Update Note","description":"Rewrite a note\'s body or replace which records it is attached to. Each list of IDs replaces that association wholesale, an empty list clears it, and omitting one leaves it untouched. A note\'s type cannot be changed.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID to update"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"Replacement note body as HTML"},"companyIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Replacement set of attached companies, e.g. [1, 2]. Send [] to detach every company; omit to leave them unchanged"},"personIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Replacement set of attached persons, e.g. [1, 2]. Send [] to detach every person; omit to leave them unchanged"},"opportunityIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Replacement set of attached opportunities, e.g. [1, 2]. Send [] to detach every opportunity; omit to leave them unchanged"}},"hostedApiKey":"none"},"agentmail_create_draft":{"id":"agentmail_create_draft","name":"Create Draft","description":"Create a new email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to create the draft in"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Draft subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text draft body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML draft body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"},"inReplyTo":{"type":"string","required":false,"visibility":"user-or-llm","description":"ID of message being replied to"},"sendAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to schedule sending"}},"hostedApiKey":"none"},"agentmail_create_inbox":{"id":"agentmail_create_inbox","name":"Create Inbox","description":"Create a new email inbox with AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"username":{"type":"string","required":false,"visibility":"user-or-llm","description":"Username for the inbox email address"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Domain for the inbox email address"},"displayName":{"type":"string","required":false,"visibility":"user-or-llm","description":"Display name for the inbox"}},"hostedApiKey":"none"},"agentmail_delete_draft":{"id":"agentmail_delete_draft","name":"Delete Draft","description":"Delete an email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to delete"}},"hostedApiKey":"none"},"agentmail_delete_inbox":{"id":"agentmail_delete_inbox","name":"Delete Inbox","description":"Delete an email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to delete"}},"hostedApiKey":"none"},"agentmail_delete_thread":{"id":"agentmail_delete_thread","name":"Delete Thread","description":"Delete an email thread in AgentMail (moves to trash, or permanently deletes if already in trash)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to delete"},"permanent":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Force permanent deletion instead of moving to trash"}},"hostedApiKey":"none"},"agentmail_forward_message":{"id":"agentmail_forward_message","name":"Forward Message","description":"Forward an email message to new recipients in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to forward"},"to":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Override subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Additional plain text to prepend"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"Additional HTML to prepend"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"}},"hostedApiKey":"none"},"agentmail_get_draft":{"id":"agentmail_get_draft","name":"Get Draft","description":"Get details of a specific email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox the draft belongs to"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to retrieve"}},"hostedApiKey":"none"},"agentmail_get_inbox":{"id":"agentmail_get_inbox","name":"Get Inbox","description":"Get details of a specific email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to retrieve"}},"hostedApiKey":"none"},"agentmail_get_message":{"id":"agentmail_get_message","name":"Get Message","description":"Get details of a specific email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to retrieve"}},"hostedApiKey":"none"},"agentmail_get_thread":{"id":"agentmail_get_thread","name":"Get Thread","description":"Get details of a specific email thread including messages in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to retrieve"}},"hostedApiKey":"none"},"agentmail_list_drafts":{"id":"agentmail_list_drafts","name":"List Drafts","description":"List email drafts in an inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list drafts from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of drafts to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}},"hostedApiKey":"none"},"agentmail_list_inboxes":{"id":"agentmail_list_inboxes","name":"List Inboxes","description":"List all email inboxes in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of inboxes to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}},"hostedApiKey":"none"},"agentmail_list_messages":{"id":"agentmail_list_messages","name":"List Messages","description":"List messages in an inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list messages from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of messages to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}},"hostedApiKey":"none"},"agentmail_list_threads":{"id":"agentmail_list_threads","name":"List Threads","description":"List email threads in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list threads from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of threads to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"},"labels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to filter threads by"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter threads before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter threads after this ISO 8601 timestamp"}},"hostedApiKey":"none"},"agentmail_reply_message":{"id":"agentmail_reply_message","name":"Reply to Message","description":"Reply to an existing email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to reply from"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to reply to"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text reply body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML reply body"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Override recipient email addresses (comma-separated)"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC email addresses (comma-separated)"},"replyAll":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Reply to all recipients of the original message"}},"hostedApiKey":"none"},"agentmail_send_draft":{"id":"agentmail_send_draft","name":"Send Draft","description":"Send an existing email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to send"}},"hostedApiKey":"none"},"agentmail_send_message":{"id":"agentmail_send_message","name":"Send Message","description":"Send an email message from an AgentMail inbox","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to send from"},"to":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient email address (comma-separated for multiple)"},"subject":{"type":"string","required":true,"visibility":"user-or-llm","description":"Email subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text email body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML email body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"}},"hostedApiKey":"none"},"agentmail_update_draft":{"id":"agentmail_update_draft","name":"Update Draft","description":"Update an existing email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to update"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Draft subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text draft body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML draft body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"},"sendAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to schedule sending"}},"hostedApiKey":"none"},"agentmail_update_inbox":{"id":"agentmail_update_inbox","name":"Update Inbox","description":"Update the display name of an email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to update"},"displayName":{"type":"string","required":true,"visibility":"user-or-llm","description":"New display name for the inbox"}},"hostedApiKey":"none"},"agentmail_update_message":{"id":"agentmail_update_message","name":"Update Message","description":"Add or remove labels on an email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to update"},"addLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to add to the message"},"removeLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to remove from the message"}},"hostedApiKey":"none"},"agentmail_update_thread":{"id":"agentmail_update_thread","name":"Update Thread Labels","description":"Add or remove labels on an email thread in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to update"},"addLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to add to the thread"},"removeLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to remove from the thread"}},"hostedApiKey":"none"},"agentphone_create_call":{"id":"agentphone_create_call","name":"Create Outbound Call","description":"Initiate an outbound voice call from an AgentPhone agent","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"agentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Agent that will handle the call"},"toNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Phone number to call in E.164 format (e.g. +14155551234)"},"fromNumberId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Phone number ID to use as caller ID. Must belong to the agent. If omitted, the agent\'s first assigned number is used."},"initialGreeting":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optional greeting spoken when the recipient answers"},"voice":{"type":"string","required":false,"visibility":"user-or-llm","description":"Voice ID override for this call (defaults to the agent\'s configured voice)"},"systemPrompt":{"type":"string","required":false,"visibility":"user-or-llm","description":"When provided, uses a built-in LLM for the conversation instead of forwarding to your webhook"}},"hostedApiKey":"none"},"agentphone_create_contact":{"id":"agentphone_create_contact","name":"Create Contact","description":"Create a new contact in AgentPhone","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"phoneNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Phone number in E.164 format (e.g. +14155551234)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact\'s full name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Contact\'s email address"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Freeform notes stored on the contact"}},"hostedApiKey":"none"},"agentphone_create_number":{"id":"agentphone_create_number","name":"Create Phone Number","description":"Provision a new SMS- and voice-enabled phone number","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Two-letter country code (e.g. US, CA). Defaults to US."},"areaCode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Preferred area code (US/CA only, e.g. \\"415\\"). Best-effort — may be ignored if unavailable."},"agentId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optionally attach the number to an agent immediately"}},"hostedApiKey":"none"},"agentphone_delete_contact":{"id":"agentphone_delete_contact","name":"Delete Contact","description":"Delete a contact by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"}},"hostedApiKey":"none"},"agentphone_get_call":{"id":"agentphone_get_call","name":"Get Call","description":"Fetch a call and its full transcript","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"callId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the call to retrieve"}},"hostedApiKey":"none"},"agentphone_get_call_transcript":{"id":"agentphone_get_call_transcript","name":"Get Call Transcript","description":"Get the full ordered transcript for a call","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"callId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the call to retrieve the transcript for"}},"hostedApiKey":"none"},"agentphone_get_contact":{"id":"agentphone_get_contact","name":"Get Contact","description":"Fetch a single contact by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"}},"hostedApiKey":"none"},"agentphone_get_conversation":{"id":"agentphone_get_conversation","name":"Get Conversation","description":"Get a conversation along with its recent messages","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"messageLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of recent messages to include (default 50, max 100)"}},"hostedApiKey":"none"},"agentphone_get_conversation_messages":{"id":"agentphone_get_conversation_messages","name":"Get Conversation Messages","description":"Get paginated messages for a conversation","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of messages to return (default 50, max 200)"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received after this ISO 8601 timestamp"}},"hostedApiKey":"none"},"agentphone_get_number_messages":{"id":"agentphone_get_number_messages","name":"Get Phone Number Messages","description":"Fetch messages received on a specific phone number","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"numberId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the phone number"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of messages to return (default 50, max 200)"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received after this ISO 8601 timestamp"}},"hostedApiKey":"none"},"agentphone_get_usage":{"id":"agentphone_get_usage","name":"Get Usage","description":"Retrieve current usage statistics for the AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"}},"hostedApiKey":"none"},"agentphone_get_usage_daily":{"id":"agentphone_get_usage_daily","name":"Get Daily Usage","description":"Get a daily breakdown of usage (messages, calls, webhooks) for the last N days","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"days":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of days to return (1-365, default 30)"}},"hostedApiKey":"none"},"agentphone_get_usage_monthly":{"id":"agentphone_get_usage_monthly","name":"Get Monthly Usage","description":"Get monthly usage aggregation (messages, calls, webhooks) for the last N months","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"months":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of months to return (1-24, default 6)"}},"hostedApiKey":"none"},"agentphone_list_calls":{"id":"agentphone_list_calls","name":"List Calls","description":"List voice calls for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"},"status":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by status (completed, in-progress, failed)"},"direction":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by direction (inbound, outbound)"},"type":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by call type (pstn, web)"},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search by phone number (matches fromNumber or toNumber)"}},"hostedApiKey":"none"},"agentphone_list_contacts":{"id":"agentphone_list_contacts","name":"List Contacts","description":"List contacts for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by name or phone number (case-insensitive contains)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 50, max 200)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}},"hostedApiKey":"none"},"agentphone_list_conversations":{"id":"agentphone_list_conversations","name":"List Conversations","description":"List conversations (message threads) for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}},"hostedApiKey":"none"},"agentphone_list_numbers":{"id":"agentphone_list_numbers","name":"List Phone Numbers","description":"List all phone numbers provisioned for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}},"hostedApiKey":"none"},"agentphone_react_to_message":{"id":"agentphone_react_to_message","name":"React to Message","description":"Send an iMessage tapback reaction to a message (iMessage only)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to react to"},"reaction":{"type":"string","required":true,"visibility":"user-or-llm","description":"Reaction type: love, like, dislike, laugh, emphasize, or question"}},"hostedApiKey":"none"},"agentphone_release_number":{"id":"agentphone_release_number","name":"Release Phone Number","description":"Release (delete) a phone number. This action is irreversible.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"numberId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the phone number to release"}},"hostedApiKey":"none"},"agentphone_send_message":{"id":"agentphone_send_message","name":"Send Message","description":"Send an outbound SMS or iMessage from an AgentPhone agent","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"agentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Agent sending the message"},"toNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient phone number in E.164 format (e.g. +14155551234)"},"body":{"type":"string","required":true,"visibility":"user-or-llm","description":"Message text to send"},"mediaUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optional URL of an image, video, or file to attach"},"numberId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Phone number ID to send from. If omitted, the agent\'s first assigned number is used."}},"hostedApiKey":"none"},"agentphone_update_contact":{"id":"agentphone_update_contact","name":"Update Contact","description":"Update a contact\'s fields","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"},"phoneNumber":{"type":"string","required":false,"visibility":"user-or-llm","description":"New phone number in E.164 format"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New contact name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"New email address"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"New freeform notes"}},"hostedApiKey":"none"},"agentphone_update_conversation":{"id":"agentphone_update_conversation","name":"Update Conversation","description":"Update conversation metadata (stored state). Pass null to clear existing metadata.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"metadata":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom key-value metadata to store on the conversation. Pass null to clear existing metadata."}},"hostedApiKey":"none"},"agiloft_async_status":{"id":"agiloft_async_status","name":"Agiloft Async Status","description":"Check whether an asynchronous Agiloft call, such as a run action button, has completed.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table the asynchronous call was made against"},"callbackId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Callback ID returned by the asynchronous call, e.g. from Run Action Button"}},"hostedApiKey":"none"},"agiloft_attach_file":{"id":"agiloft_attach_file","name":"Agiloft Attach File","description":"Attach a file to a field in an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to attach the file to"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"file":{"type":"file","required":true,"visibility":"user-or-llm","description":"File to attach"},"fileName":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name to assign to the file (defaults to original file name)"},"overwrite":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Replace the contents of the field instead of adding another file to it"}},"hostedApiKey":"none"},"agiloft_attachment_info":{"id":"agiloft_attachment_info","name":"Agiloft Attachment Info","description":"Get information about file attachments on a record field.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to check attachments on"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field to inspect"}},"hostedApiKey":"none"},"agiloft_create_record":{"id":"agiloft_create_record","name":"Agiloft Create Record","description":"Create a new record in an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record field values as a JSON object (e.g., {\\"first_name\\": \\"John\\", \\"status\\": \\"Active\\"})"}},"hostedApiKey":"none"},"agiloft_delete_record":{"id":"agiloft_delete_record","name":"Agiloft Delete Record","description":"Delete a record from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to delete"},"substituteIds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated IDs of records that adopt the dependants of the deleted record. Read only when the delete rule is REPLACE_WITH_ANOTHER."},"deleteRule":{"type":"string","required":false,"visibility":"user-or-llm","description":"How to treat records that depend on this one: ERROR_IF_DEPENDANTS (default — fails rather than cascading), APPLY_DELETE_WHERE_POSSIBLE, DELETE_WHERE_POSSIBLE_OTHERWISE_UNLINK, APPLY_UNLINK, UNLINK_WHERE_POSSIBLE_OTHERWISE_DELETE, or REPLACE_WITH_ANOTHER"}},"hostedApiKey":"none"},"agiloft_get_choice_line_id":{"id":"agiloft_get_choice_line_id","name":"Agiloft Get Choice Line ID","description":"Resolve the internal numeric ID of a choice-list value, for use in EWSelect WHERE clauses against choice fields.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"case\\", \\"contracts\\")"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Choice field name (e.g., \\"priority\\", \\"status\\")"},"value":{"type":"string","required":true,"visibility":"user-or-llm","description":"Choice display value to resolve (e.g., \\"High\\", \\"Active\\")"}},"hostedApiKey":"none"},"agiloft_list_tables":{"id":"agiloft_list_tables","name":"Agiloft List Tables","description":"List the tables and fields in an Agiloft knowledge base, to discover the logical names other operations need.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":false,"visibility":"user-or-llm","description":"Logical name of a single table to describe (e.g., \\"contacts\\"). Leave empty to list every table in the knowledge base."},"includeLinkedInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the source table and column behind each linked field"},"skipColumnsInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Return table names only, omitting field details, for a much smaller response"}},"hostedApiKey":"none"},"agiloft_lock_record":{"id":"agiloft_lock_record","name":"Agiloft Lock Record","description":"Lock, unlock, or check the lock status of an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to lock, unlock, or check"},"lockAction":{"type":"string","required":true,"visibility":"user-or-llm","description":"Action to perform: \\"lock\\", \\"unlock\\", or \\"check\\""},"force":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Unlock only: release a lock held by another user."}},"hostedApiKey":"none"},"agiloft_nlp_search":{"id":"agiloft_nlp_search","name":"Agiloft Natural Language Search","description":"Search Agiloft records by describing what you want in plain language, such as \\"active NDAs submitted last month\\".","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"nlpQuery":{"type":"string","required":true,"visibility":"user-or-llm","description":"The request in plain language, e.g. \\"Show me open, high-priority contracts\\". Structured field filters are not accepted — use Search Records for those."},"fields":{"type":"string","required":true,"visibility":"user-or-llm","description":"Comma-separated field names to return, e.g. \\"id, contract_title1, company_name\\""},"page":{"type":"string","required":false,"visibility":"user-or-llm","description":"Page number, starting from 0"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Records per page"}},"hostedApiKey":"none"},"agiloft_read_record":{"id":"agiloft_read_record","name":"Agiloft Read Record","description":"Read a record by ID from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to read"},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of field names to include in the response"}},"hostedApiKey":"none"},"agiloft_remove_attachment":{"id":"agiloft_remove_attachment","name":"Agiloft Remove Attachment","description":"Remove an attached file from a field in an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record containing the attachment"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"position":{"type":"string","required":true,"visibility":"user-or-llm","description":"Position index of the file to remove (starting from 0)"}},"hostedApiKey":"none"},"agiloft_retrieve_attachment":{"id":"agiloft_retrieve_attachment","name":"Agiloft Retrieve Attachment","description":"Download an attached file from an Agiloft record field.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record containing the attachment"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"position":{"type":"string","required":true,"visibility":"user-or-llm","description":"Position index of the file in the field (starting from 0)"}},"hostedApiKey":"none"},"agiloft_run_action_button":{"id":"agiloft_run_action_button","name":"Agiloft Run Action Button","description":"Run an action button on an Agiloft record, such as an approval or send-for-signature step.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"case\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to run the action button on"},"actionButtonField":{"type":"string","required":true,"visibility":"user-or-llm","description":"Logical name of the field holding the action button (e.g., \\"ab_field\\")"}},"hostedApiKey":"none"},"agiloft_saved_search":{"id":"agiloft_saved_search","name":"Agiloft Saved Search","description":"List the saved searches defined for an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Logical table name to list saved searches for (e.g., \\"contract\\")"}},"hostedApiKey":"none"},"agiloft_search_records":{"id":"agiloft_search_records","name":"Agiloft Search Records","description":"Search for records in an Agiloft table using a query.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name to search in (e.g., \\"contracts\\", \\"contacts.employees\\")"},"query":{"type":"string","required":false,"visibility":"user-or-llm","description":"Ad hoc EWSearch query. Combine conditions with && (and) or || (or) and quote every value — e.g. \\"summary~=\'test\'&&priority=\'High\'\\". Required unless a saved search is given."},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Label of a saved search defined on the table (e.g., \\"C: Status is Closed\\"). Can be combined with a query to narrow it further."},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of field names to include in the results"},"page":{"type":"string","required":false,"visibility":"user-or-llm","description":"Page number for paginated results (starting from 0)"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of records to return per page. Agiloft treats 0 as \\"all records\\", so leave it unset or use a positive value to keep result sizes bounded."}},"hostedApiKey":"none"},"agiloft_select_records":{"id":"agiloft_select_records","name":"Agiloft Select Records","description":"Select record IDs matching a SQL WHERE clause from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"where":{"type":"string","required":true,"visibility":"user-or-llm","description":"SQL WHERE clause using database column names (e.g., \\"summary like \'%new%\'\\" or \\"assigned_person=\'John Doe\'\\"). EWSelect has no page size and returns every matching ID, so append a database limit such as \\"limit 0,200\\" to bound the result."}},"hostedApiKey":"none"},"agiloft_update_record":{"id":"agiloft_update_record","name":"Agiloft Update Record","description":"Update an existing record in an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to update"},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Updated field values as a JSON object (e.g., {\\"status\\": \\"Active\\", \\"priority\\": \\"High\\"})"}},"hostedApiKey":"none"},"agiloft_upsert_record":{"id":"agiloft_upsert_record","name":"Agiloft Upsert Record","description":"Create an Agiloft record, or update it when a record already matches the given fields.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"match":{"type":"string","required":true,"visibility":"user-or-llm","description":"Field used to find an existing record (e.g., \\"ext_id\\"). Pick something that identifies a record uniquely — if more than one record matches, Agiloft writes nothing and returns a conflict."},"async":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Queue the write instead of waiting for it. Returns a callback ID instead of a record ID; pass that to Async Status to poll the result."},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Field values as a JSON object. On create these populate the new record; on update only the supplied fields change."}},"hostedApiKey":"none"},"ahrefs_anchors":{"id":"ahrefs_anchors","name":"Ahrefs Anchors","description":"Get the anchor text distribution for a target domain or URL\'s backlinks, showing how many links and referring domains use each anchor text.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live), \\"all_time\\" (default, includes lost backlinks), or \\"since:YYYY-MM-DD\\" (backlinks found since a date)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_backlinks":{"id":"ahrefs_backlinks","name":"Ahrefs Backlinks","description":"Get a list of backlinks pointing to a target domain or URL. Returns details about each backlink including source URL, anchor text, and domain rating.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live backlinks), \\"all_time\\" (default, includes lost backlinks), or \\"since:YYYY-MM-DD\\" (backlinks found since a date)."},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_backlinks_stats":{"id":"ahrefs_backlinks_stats","name":"Ahrefs Backlinks Stats","description":"Get backlink and referring domain totals for a target domain or URL, both currently live and across all time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_batch_analysis":{"id":"ahrefs_batch_analysis","name":"Ahrefs Batch Analysis","description":"Get bulk SEO metrics (Domain Rating, backlinks, referring domains, organic traffic, and more) for multiple domains or URLs in a single request. Useful for comparing many competitors at once.","version":"1.0.0","params":{"targets":{"type":"string","required":true,"visibility":"user-or-llm","description":"Comma-separated list of domains or URLs to analyze. Example: \\"example.com,competitor.com\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode applied to every target: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"protocol":{"type":"string","required":false,"visibility":"user-or-llm","description":"Protocol applied to every target: \\"both\\" (default), \\"http\\", or \\"https\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_broken_backlinks":{"id":"ahrefs_broken_backlinks","name":"Ahrefs Broken Backlinks","description":"Get a list of broken backlinks pointing to a target domain or URL. Useful for identifying link reclamation opportunities.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_domain_rating":{"id":"ahrefs_domain_rating","name":"Ahrefs Domain Rating","description":"Get the Domain Rating (DR) and Ahrefs Rank for a target domain. Domain Rating shows the strength of a website\'s backlink profile on a scale from 0 to 100.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain to analyze (e.g., example.com)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date for historical data in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_domain_rating_history":{"id":"ahrefs_domain_rating_history","name":"Ahrefs Domain Rating History","description":"Get the historical Domain Rating (DR) trend for a target domain or URL over a date range, grouped daily, weekly, or monthly.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_keyword_overview":{"id":"ahrefs_keyword_overview","name":"Ahrefs Keyword Overview","description":"Get detailed metrics for a keyword including search volume, keyword difficulty, CPC, clicks, and traffic potential.","version":"1.0.0","params":{"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The keyword to analyze"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for keyword data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_keywords_history":{"id":"ahrefs_keywords_history","name":"Ahrefs Keywords History","description":"Get the historical organic keyword ranking distribution for a target domain or URL over a date range: how many keywords rank in each position bucket at each point in time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_metrics":{"id":"ahrefs_metrics","name":"Ahrefs Metrics","description":"Get a one-call organic and paid search overview for a target domain or URL: organic traffic, organic keywords, paid traffic, paid keywords, and estimated traffic cost.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_metrics_history":{"id":"ahrefs_metrics_history","name":"Ahrefs Metrics History","description":"Get the historical organic and paid traffic trend for a target domain or URL over a date range: organic traffic/cost and paid traffic/cost at each point in time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_organic_competitors":{"id":"ahrefs_organic_competitors","name":"Ahrefs Organic Competitors","description":"Get domains that compete with a target domain or URL for the same organic keywords, ranked by keyword overlap.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_organic_keywords":{"id":"ahrefs_organic_keywords","name":"Ahrefs Organic Keywords","description":"Get organic keywords that a target domain or URL ranks for in Google search results. Returns keyword details including search volume, ranking position, and estimated traffic.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_paid_pages":{"id":"ahrefs_paid_pages","name":"Ahrefs Paid Pages","description":"Get a target domain\'s pages that receive paid search traffic, sorted by estimated paid traffic. Returns page URLs with their paid traffic, keyword counts, and estimated spend.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_competitors_overview":{"id":"ahrefs_rank_tracker_competitors_overview","name":"Ahrefs Rank Tracker Competitors Overview","description":"Get competitor rankings for the keywords tracked in an Ahrefs Rank Tracker project: each tracked keyword\'s volume and difficulty alongside every competitor\'s position, traffic, and traffic value. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report rankings for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"dateCompared":{"type":"string","required":false,"visibility":"user-only","description":"Comparison date in YYYY-MM-DD format, to compute position/traffic deltas"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_competitors_stats":{"id":"ahrefs_rank_tracker_competitors_stats","name":"Ahrefs Rank Tracker Competitors Stats","description":"Get aggregate competitor stats for an Ahrefs Rank Tracker project: each competitor\'s traffic, traffic value, average position, and share of voice across all tracked keywords. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report metrics for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_overview":{"id":"ahrefs_rank_tracker_overview","name":"Ahrefs Rank Tracker Overview","description":"Get ranking overview metrics for the keywords tracked in an Ahrefs Rank Tracker project: position, search volume, keyword difficulty, and estimated traffic. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report rankings for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"dateCompared":{"type":"string","required":false,"visibility":"user-only","description":"Comparison date in YYYY-MM-DD format, to compute position/traffic deltas"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_serp_overview":{"id":"ahrefs_rank_tracker_serp_overview","name":"Ahrefs Rank Tracker SERP Overview","description":"Get the full SERP (search engine results page) for a keyword tracked in an Ahrefs Rank Tracker project, including every ranking URL with its position, title, and authority metrics. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The tracked keyword to retrieve SERP data for"},"country":{"type":"string","required":true,"visibility":"user-or-llm","description":"Country code for the tracked keyword. Example: \\"us\\", \\"gb\\", \\"de\\""},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"topPositions":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of top organic positions to return (defaults to all available)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Timestamp to return the last available SERP Overview at, in YYYY-MM-DDThh:mm:ss format"},"locationId":{"type":"number","required":false,"visibility":"user-or-llm","description":"Location ID of the tracked keyword, if tracked at a specific location"},"languageCode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Language code of the tracked keyword"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_refdomains_history":{"id":"ahrefs_refdomains_history","name":"Ahrefs Referring Domains History","description":"Get the historical referring domains trend for a target domain or URL over a date range, grouped daily, weekly, or monthly.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_referring_domains":{"id":"ahrefs_referring_domains","name":"Ahrefs Referring Domains","description":"Get a list of domains that link to a target domain or URL. Returns unique referring domains with their domain rating, backlink counts, and discovery dates.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live), \\"all_time\\" (default, includes lost domains), or \\"since:YYYY-MM-DD\\" (domains found since a date)."},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_related_terms":{"id":"ahrefs_related_terms","name":"Ahrefs Related Terms","description":"Get keyword ideas related to a seed keyword: terms the same top-ranking pages also rank for (\\"also rank for\\") or also discuss (\\"also talk about\\"), with volume, difficulty, and CPC.","version":"1.0.0","params":{"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The seed keyword to find related terms for"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for keyword data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"terms":{"type":"string","required":false,"visibility":"user-or-llm","description":"Type of related keywords to return: \\"also_rank_for\\", \\"also_talk_about\\", or \\"all\\" (default: \\"all\\")"},"viewFor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Whether to derive related terms from the top 10 or top 100 ranking pages (default: \\"top_10\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_site_audit_page_explorer":{"id":"ahrefs_site_audit_page_explorer","name":"Ahrefs Site Audit Page Explorer","description":"Get crawled pages from an Ahrefs Site Audit project with health and SEO metrics: HTTP status, title, link counts, backlinks, indexability, and traffic. Optionally filter to pages affected by a specific issue.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Site Audit project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Crawl date in YYYY-MM-DDThh:mm:ss format (defaults to the most recent crawl)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip, for pagination"},"issueId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Only return pages affected by this issue ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_top_pages":{"id":"ahrefs_top_pages","name":"Ahrefs Top Pages","description":"Get the top pages of a target domain sorted by organic traffic. Returns page URLs with their traffic, keyword counts, and estimated traffic value.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"airtable_create_records":{"id":"airtable_create_records","name":"Airtable Create Records","description":"Write new records to an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to create, each with a `fields` object"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_delete_records":{"id":"airtable_delete_records","name":"Airtable Delete Records","description":"Delete one or more records from an Airtable table by ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordIds":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of record IDs to delete (each starts with \\"rec\\", e.g., [\\"recXXXXXXXXXXXXXX\\"]). Pass a single-element array to delete one record."}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_get_base_schema":{"id":"airtable_get_base_schema","name":"Airtable Get Base Schema","description":"Get the schema of all tables, fields, and views in an Airtable base","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_get_record":{"id":"airtable_get_record","name":"Airtable Get Record","description":"Retrieve a single record from an Airtable table by its ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record ID to retrieve (starts with \\"rec\\", e.g., \\"recXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_list_bases":{"id":"airtable_list_bases","name":"Airtable List Bases","description":"List all bases the authenticated user has access to","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"offset":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination offset for retrieving additional bases"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_list_records":{"id":"airtable_list_records","name":"Airtable List Records","description":"Read records from an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"maxRecords":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of records to return (default: all records)"},"filterFormula":{"type":"string","required":false,"visibility":"user-or-llm","description":"Formula to filter records (e.g., \\"({Field Name} = \'Value\')\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_list_tables":{"id":"airtable_list_tables","name":"Airtable List Tables","description":"List all tables and their schema in an Airtable base","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_update_multiple_records":{"id":"airtable_update_multiple_records","name":"Airtable Update Multiple Records","description":"Update multiple existing records in an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to update, each with an `id` and a `fields` object"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_update_record":{"id":"airtable_update_record","name":"Airtable Update Record","description":"Update an existing record in an Airtable table by ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record ID to update (starts with \\"rec\\", e.g., \\"recXXXXXXXXXXXXXX\\")"},"fields":{"type":"json","required":true,"visibility":"user-or-llm","description":"An object containing the field names and their new values"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_upsert_records":{"id":"airtable_upsert_records","name":"Airtable Upsert Records","description":"Update existing records or create new ones in an Airtable table, matching on the specified merge fields","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to upsert, each with a `fields` object"},"fieldsToMergeOn":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of field names used to match existing records (max 3). A record is updated when all merge fields match, otherwise it is created. Example: [\\"Name\\"]"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airweave_search":{"id":"airweave_search","name":"Airweave Search","description":"Search your synced data collections using Airweave. Supports semantic search with hybrid, neural, or keyword retrieval strategies. Optionally generate AI-powered answers from search results.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Airweave API Key for authentication"},"collectionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The readable ID of the collection to search"},"query":{"type":"string","required":true,"visibility":"user-or-llm","description":"The search query text"},"limit":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 100)"},"retrievalStrategy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Retrieval strategy: hybrid (default), neural, or keyword"},"expandQuery":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Generate query variations to improve recall"},"rerank":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Reorder results for improved relevance using LLM"},"generateAnswer":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Generate a natural-language answer to the query"}},"hostedApiKey":"none"},"algolia_add_record":{"id":"algolia_add_record","name":"Algolia Add Record","description":"Add or replace a record in an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":false,"visibility":"user-or-llm","description":"Object ID for the record (auto-generated if not provided)"},"record":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object representing the record to add"}},"hostedApiKey":"none"},"algolia_batch_operations":{"id":"algolia_batch_operations","name":"Algolia Batch Operations","description":"Perform batch add, update, partial update, or delete operations on records in an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"requests":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of batch operations. Each item has \\"action\\" (addObject, updateObject, partialUpdateObject, partialUpdateObjectNoCreate, deleteObject, delete, clear) and \\"body\\" (the record data; must include objectID for update/delete; use an empty object {} for the index-level delete/clear actions)"}},"hostedApiKey":"none"},"algolia_browse_records":{"id":"algolia_browse_records","name":"Algolia Browse Records","description":"Browse and iterate over all records in an Algolia index using cursor pagination","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key (must have browse ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to browse"},"query":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search query to filter browsed records"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter string to narrow down results"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of hits per page (default: 1000, max: 1000)"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous browse response for pagination"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search"}},"hostedApiKey":"none"},"algolia_clear_records":{"id":"algolia_clear_records","name":"Algolia Clear Records","description":"Clear all records from an Algolia index while keeping settings, synonyms, and rules","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to clear"}},"hostedApiKey":"none"},"algolia_copy_move_index":{"id":"algolia_copy_move_index","name":"Algolia Copy/Move Index","description":"Copy or move an Algolia index to a new destination","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the source index"},"operation":{"type":"string","required":true,"visibility":"user-or-llm","description":"Operation to perform: \\"copy\\" or \\"move\\""},"destination":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the destination index"},"scope":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of scopes to copy (only for \\"copy\\" operation): [\\"settings\\", \\"synonyms\\", \\"rules\\"]. Omit to copy everything including records."}},"hostedApiKey":"none"},"algolia_delete_by_filter":{"id":"algolia_delete_by_filter","name":"Algolia Delete By Filter","description":"Delete all records matching a filter from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter expression to match records for deletion (e.g., \\"category:outdated\\")"},"facetFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of facet filters (e.g., [\\"brand:Acme\\"])"},"numericFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of numeric filters (e.g., [\\"price > 100\\"])"},"tagFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of tag filters using the _tags attribute (e.g., [\\"published\\"])"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search filter (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search filter"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search filter"}},"hostedApiKey":"none"},"algolia_delete_index":{"id":"algolia_delete_index","name":"Algolia Delete Index","description":"Delete an entire Algolia index and all its records","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to delete"}},"hostedApiKey":"none"},"algolia_delete_record":{"id":"algolia_delete_record","name":"Algolia Delete Record","description":"Delete a record by objectID from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to delete"}},"hostedApiKey":"none"},"algolia_get_record":{"id":"algolia_get_record","name":"Algolia Get Record","description":"Get a record by objectID from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to retrieve"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"}},"hostedApiKey":"none"},"algolia_get_records":{"id":"algolia_get_records","name":"Algolia Get Records","description":"Retrieve multiple records by objectID from one or more Algolia indices","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Default index name for all requests"},"requests":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of objects specifying records to retrieve. Each must have \\"objectID\\" and optionally \\"indexName\\" and \\"attributesToRetrieve\\"."}},"hostedApiKey":"none"},"algolia_get_settings":{"id":"algolia_get_settings","name":"Algolia Get Settings","description":"Retrieve the settings of an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"}},"hostedApiKey":"none"},"algolia_get_task_status":{"id":"algolia_get_task_status","name":"Algolia Get Task Status","description":"Check whether an Algolia indexing task has finished publishing","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index the task ran against"},"taskID":{"type":"number","required":true,"visibility":"user-or-llm","description":"The taskID returned by a previous write operation"}},"hostedApiKey":"none"},"algolia_list_indices":{"id":"algolia_list_indices","name":"Algolia List Indices","description":"List all indices in an Algolia application","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for paginating indices (default: not paginated)"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of indices per page (default: 100)"}},"hostedApiKey":"none"},"algolia_partial_update_record":{"id":"algolia_partial_update_record","name":"Algolia Partial Update Record","description":"Partially update a record in an Algolia index without replacing it entirely","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to update"},"attributes":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object with attributes to update. Supports built-in operations like {\\"stock\\": {\\"_operation\\": \\"Decrement\\", \\"value\\": 1}}"},"createIfNotExists":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to create the record if it does not exist (default: true)"}},"hostedApiKey":"none"},"algolia_search":{"id":"algolia_search","name":"Algolia Search","description":"Search an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to search"},"query":{"type":"string","required":true,"visibility":"user-or-llm","description":"Search query text"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of hits per page (default: 20)"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number to retrieve (default: 0)"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter string (e.g., \\"category:electronics AND price < 100\\")"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"},"facets":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of facet attribute names to retrieve counts for (use \\"*\\" for all)"},"getRankingInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to include detailed ranking information in each hit"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search"}},"hostedApiKey":"none"},"algolia_update_settings":{"id":"algolia_update_settings","name":"Algolia Update Settings","description":"Update the settings of an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have editSettings ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"settings":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object with settings to update (e.g., {\\"searchableAttributes\\": [\\"name\\", \\"description\\"], \\"customRanking\\": [\\"desc(popularity)\\"]})"},"forwardToReplicas":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to apply changes to replica indices (default: false)"}},"hostedApiKey":"none"},"amplitude_event_segmentation":{"id":"amplitude_event_segmentation","name":"Amplitude Event Segmentation","description":"Query event analytics data with segmentation. Get event counts, uniques, averages, and more.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"eventType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Event type name to analyze"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric type: uniques, totals, pct_dau, average, histogram, sums, value_avg, or formula (default: uniques)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by (prefix custom user properties with \\"gp:\\")"},"groupBy2":{"type":"string","required":false,"visibility":"user-or-llm","description":"Second property name to group by (prefix custom user properties with \\"gp:\\")"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of group-by values (max 1000)"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON array of filter objects applied to the event, e.g. [{\\"subprop_type\\":\\"event\\",\\"subprop_key\\":\\"city\\",\\"subprop_op\\":\\"is\\",\\"subprop_value\\":[\\"San Francisco\\"]}]"},"formula":{"type":"string","required":false,"visibility":"user-or-llm","description":"Required when metric is \\"formula\\", e.g. \\"UNIQUES(A)/UNIQUES(B)\\""},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_funnels":{"id":"amplitude_funnels","name":"Amplitude Funnels","description":"Analyze conversion rates and drop-off between a sequence of events.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"events":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON array of event objects, one per funnel step in order, e.g. [{\\"event_type\\":\\"signup\\"},{\\"event_type\\":\\"purchase\\"}]"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Funnel ordering: \\"ordered\\", \\"unordered\\", or \\"sequential\\" (default: ordered)"},"userType":{"type":"string","required":false,"visibility":"user-or-llm","description":"User type: \\"new\\" or \\"active\\" (default: active)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: -300000 (real-time), -3600000 (hourly), 1 (daily), 7 (weekly), or 30 (monthly)"},"conversionWindowSeconds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Conversion window in seconds (default: 2592000, i.e. 30 days)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property to group by (limit: one; prefix custom properties with \\"gp:\\")"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of group-by values (default: 100, max: 1000)"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_get_active_users":{"id":"amplitude_get_active_users","name":"Amplitude Get Active Users","description":"Get active or new user counts over a date range from the Dashboard REST API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric type: \\"active\\" or \\"new\\" (default: active)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_get_revenue":{"id":"amplitude_get_revenue","name":"Amplitude Get Revenue","description":"Get revenue LTV data including ARPU, ARPPU, total revenue, and paying user counts.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric: 0 (ARPU), 1 (ARPPU), 2 (Total Revenue), 3 (Paying Users)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by (limit: one)"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_group_identify":{"id":"amplitude_group_identify","name":"Amplitude Group Identify","description":"Set group-level properties in Amplitude. Supports $set, $setOnce, $add, $append, $unset operations.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"groupType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Group classification (e.g., \\"company\\", \\"org_id\\")"},"groupValue":{"type":"string","required":true,"visibility":"user-or-llm","description":"Specific group identifier (e.g., \\"Acme Corp\\")"},"groupProperties":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON object of group properties. Use operations like $set, $setOnce, $add, $append, $unset."},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_identify_user":{"id":"amplitude_identify_user","name":"Amplitude Identify User","description":"Set user properties in Amplitude using the Identify API. Supports $set, $setOnce, $add, $append, $unset operations.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"User ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"userProperties":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON object of user properties. Use operations like $set, $setOnce, $add, $append, $unset."},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_list_events":{"id":"amplitude_list_events","name":"Amplitude List Events","description":"List all event types in the Amplitude project with their weekly totals and unique counts.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_realtime_active_users":{"id":"amplitude_realtime_active_users","name":"Amplitude Real-time Active Users","description":"Get real-time active user counts at 5-minute granularity for the last 2 days.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_retention":{"id":"amplitude_retention","name":"Amplitude Retention","description":"Measure how many users return to perform an action after a starting action.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"startEvent":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON starting event object, e.g. {\\"event_type\\":\\"_new\\"} or {\\"event_type\\":\\"_active\\"}"},"returnEvent":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON returning event object, e.g. {\\"event_type\\":\\"_all\\"} or {\\"event_type\\":\\"_active\\"}"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"retentionMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Retention type: \\"bracket\\", \\"rolling\\", or \\"n-day\\" (default: n-day)"},"retentionBrackets":{"type":"string","required":false,"visibility":"user-or-llm","description":"Required when Retention Mode is \\"bracket\\". Day ranges, e.g. [[0,4]]"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property to group by (limit: one; prefix custom properties with \\"gp:\\")"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_send_event":{"id":"amplitude_send_event","name":"Amplitude Send Event","description":"Track an event in Amplitude using the HTTP V2 API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"User ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"eventType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the event (e.g., \\"page_view\\", \\"purchase\\")"},"eventProperties":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON object of custom event properties"},"userProperties":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON object of user properties to set (supports $set, $setOnce, $add, $append, $unset)"},"time":{"type":"string","required":false,"visibility":"user-or-llm","description":"Event timestamp in milliseconds since epoch"},"sessionId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Session start time in milliseconds since epoch"},"insertId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Unique ID for deduplication (within 7-day window)"},"appVersion":{"type":"string","required":false,"visibility":"user-or-llm","description":"Application version string"},"platform":{"type":"string","required":false,"visibility":"user-or-llm","description":"Platform (e.g., \\"Web\\", \\"iOS\\", \\"Android\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Two-letter country code"},"language":{"type":"string","required":false,"visibility":"user-or-llm","description":"Language code (e.g., \\"en\\")"},"ip":{"type":"string","required":false,"visibility":"user-or-llm","description":"IP address for geo-location"},"price":{"type":"string","required":false,"visibility":"user-or-llm","description":"Price of the item purchased"},"quantity":{"type":"string","required":false,"visibility":"user-or-llm","description":"Quantity of items purchased"},"revenue":{"type":"string","required":false,"visibility":"user-or-llm","description":"Revenue amount"},"productId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Product identifier"},"revenueType":{"type":"string","required":false,"visibility":"user-or-llm","description":"Revenue type (e.g., \\"purchase\\", \\"refund\\")"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_user_activity":{"id":"amplitude_user_activity","name":"Amplitude User Activity","description":"Get the event stream for a specific user by their Amplitude ID.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"amplitudeId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Amplitude internal user ID"},"offset":{"type":"string","required":false,"visibility":"user-or-llm","description":"Offset for pagination (default 0)"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of events to return (default 1000, max 1000)"},"direction":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort direction: \\"latest\\" or \\"earliest\\" (default: latest)"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_user_profile":{"id":"amplitude_user_profile","name":"Amplitude User Profile","description":"Get a user profile including properties, cohort memberships, and computed properties. Not available for EU data-residency projects.","version":"1.0.0","params":{"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"External user ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"getAmpProps":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include Amplitude user properties (true/false, default: false)"},"getCohortIds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include cohort IDs the user belongs to (true/false, default: false)"},"getComputations":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include computed user properties (true/false, default: false)"}},"hostedApiKey":"none"},"amplitude_user_search":{"id":"amplitude_user_search","name":"Amplitude User Search","description":"Search for a user by User ID, Device ID, or Amplitude ID using the Dashboard REST API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"user":{"type":"string","required":true,"visibility":"user-or-llm","description":"User ID, Device ID, or Amplitude ID to search for"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"apify_get_dataset_items":{"id":"apify_get_dataset_items","name":"APIFY Get Dataset Items","description":"Retrieve items stored in an APIFY dataset","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"datasetId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Dataset ID to read items from. Example: \\"9RnD3Pql2vGZkc5H5\\""},"itemLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Max items to return (1-250000). Default: all items. Example: 500"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to skip at the start. Default: 0"},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of fields to include. Example: \\"title,url,price\\""}},"hostedApiKey":"none"},"apify_get_run":{"id":"apify_get_run","name":"APIFY Get Run","description":"Get the status and details of an APIFY actor run","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"runId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor run ID to fetch. Example: \\"HG7ML7M8z78YcAPEB\\""}},"hostedApiKey":"none"},"apify_run_actor_async":{"id":"apify_run_actor_async","name":"APIFY Run Actor (Async)","description":"Run an APIFY actor asynchronously with polling for long-running tasks","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"actorId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor ID or username/actor-name. Examples: \\"apify/web-scraper\\", \\"janedoe/my-actor\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor input as JSON string. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}], \\"maxPages\\": 10}"},"waitForFinish":{"type":"number","required":false,"visibility":"user-or-llm","description":"Initial wait time in seconds (0-60) before polling starts. Example: 30"},"itemLimit":{"type":"number","required":false,"default":100,"visibility":"user-or-llm","description":"Max dataset items to fetch (1-250000). Default: 100. Example: 500"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the actor run (128-32768). Example: 1024 for 1GB, 2048 for 2GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the actor run. Example: 300 for 5 minutes, 3600 for 1 hour"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\", \\"build-tag-name\\""}},"hostedApiKey":"none"},"apify_run_actor_sync":{"id":"apify_run_actor_sync","name":"APIFY Run Actor (Sync)","description":"Run an APIFY actor synchronously and get results (max 5 minutes)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"actorId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor ID or username/actor-name. Examples: \\"apify/web-scraper\\", \\"janedoe/my-actor\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor input as JSON string. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}], \\"maxPages\\": 10}"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the actor run (128-32768). Example: 1024 for 1GB, 2048 for 2GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the actor run. Example: 300 for 5 minutes, 3600 for 1 hour"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\", \\"build-tag-name\\""}},"hostedApiKey":"none"},"apify_run_task":{"id":"apify_run_task","name":"APIFY Run Task","description":"Run a saved APIFY actor task synchronously and get dataset items (max 5 minutes)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task ID or username/task-name. Examples: \\"janedoe/my-task\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON string that overrides the task\'s saved input. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}]}"},"itemLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Max dataset items to return (1-250000). Example: 500"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the run (128-32768). Example: 1024 for 1GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the run. Example: 300 for 5 minutes"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\""}},"hostedApiKey":"none"},"apollo_account_bulk_create":{"id":"apollo_account_bulk_create","name":"Apollo Bulk Create Accounts","description":"Create up to 100 accounts at once in your Apollo database. Set run_dedupe=true to deduplicate by domain, organization_id, and name. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"accounts":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of accounts to create (max 100). Each account should include a name, and may optionally include domain, phone, phone_status_cd, raw_address, owner_id, linkedin_url, facebook_url, twitter_url, salesforce_id, and hubspot_id."},"append_label_names":{"type":"array","required":false,"visibility":"user-only","description":"Array of label names to add to ALL accounts in this request"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, performs aggressive deduplication by domain, organization_id, and name (defaults to false)"}},"hostedApiKey":"none"},"apollo_account_bulk_update":{"id":"apollo_account_bulk_update","name":"Apollo Bulk Update Accounts","description":"Update up to 1000 existing accounts at once in your Apollo database (higher limit than contacts!). Each account must include an id field. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"account_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of account IDs to update with the same values (max 1000). Use with name/owner_id for uniform updates. Use either this OR account_attributes."},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this name to all accounts"},"owner_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this owner to all accounts"},"account_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this account stage to all accounts"},"account_attributes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of account objects with individual updates (each must include id). Example: [{\\"id\\": \\"acc1\\", \\"name\\": \\"Acme\\", \\"owner_id\\": \\"u1\\", \\"account_stage_id\\": \\"s1\\", \\"typed_custom_fields\\": {\\"field_id\\": \\"value\\"}}]"},"async":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, processes the update asynchronously. Only supported when using account_ids; returns 422 if used with account_attributes."}},"hostedApiKey":"none"},"apollo_account_create":{"id":"apollo_account_create","name":"Apollo Create Account","description":"Create a new account (company) in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Company name (e.g., \\"Acme Corporation\\")"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain without www. prefix (e.g., \\"acme.com\\")"},"phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number for the account"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo user ID of the account owner"},"account_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo ID for the account stage to assign this account to"},"raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate location (e.g., \\"San Francisco, CA, USA\\")"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_account_search":{"id":"apollo_account_search","name":"Apollo Search Accounts","description":"Search your team\'s accounts in Apollo. Display limit: 50,000 records (100 records per page, 500 pages max). Use filters to narrow results. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"q_organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter accounts by organization name (partial-match search)"},"account_stage_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by account stage IDs"},"account_label_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by account label IDs"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"account_last_activity_date\\", \\"account_created_at\\", or \\"account_updated_at\\""},"sort_ascending":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Sort ascending when true. Defaults to descending."},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_account_update":{"id":"apollo_account_update","name":"Apollo Update Account","description":"Update an existing account in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"account_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the account to update (e.g., \\"acc_abc123\\")"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company name (e.g., \\"Acme Corporation\\")"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain (e.g., \\"acme.com\\")"},"phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company phone number"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo user ID of the account owner"},"account_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo ID for the account stage to assign this account to"},"raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate location (e.g., \\"San Francisco, CA, USA\\")"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_contact_bulk_create":{"id":"apollo_contact_bulk_create","name":"Apollo Bulk Create Contacts","description":"Create up to 100 contacts at once in your Apollo database. Supports deduplication to prevent creating duplicate contacts. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"contacts":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of contacts to create (max 100). Each contact may include first_name, last_name, email, title, organization_name, account_id, owner_id, contact_stage_id, linkedin_url, phone (single string) or phone_numbers (array of {raw_number, position}), contact_emails, typed_custom_fields, and CRM IDs (salesforce_contact_id, hubspot_id, team_id) for cross-system matching"},"append_label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Label names to add to all contacts in this request (e.g., [\\"Hot Lead\\"])"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-only","description":"Enable deduplication to prevent creating duplicate contacts. When true, existing contacts are returned without modification"}},"hostedApiKey":"none"},"apollo_contact_bulk_update":{"id":"apollo_contact_bulk_update","name":"Apollo Bulk Update Contacts","description":"Update up to 100 existing contacts at once in your Apollo database. Each contact must include an id field. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"contact_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of contact IDs to update. Must be paired with an object-form contact_attributes specifying the fields to apply uniformly to all listed contacts."},"contact_attributes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Required. Either an array of per-contact updates (each with id) — used standalone — or a single object of attributes to apply to all contact_ids. Supported fields: owner_id, email, organization_name, title, first_name, last_name, account_id, present_raw_address, linkedin_url, typed_custom_fields"},"async":{"type":"boolean","required":false,"visibility":"user-only","description":"Force asynchronous processing. Automatically enabled for >100 contacts"}},"hostedApiKey":"none"},"apollo_contact_create":{"id":"apollo_contact_create","name":"Apollo Create Contact","description":"Create a new contact in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"first_name":{"type":"string","required":true,"visibility":"user-or-llm","description":"First name of the contact"},"last_name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Last name of the contact"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address of the contact"},"title":{"type":"string","required":false,"visibility":"user-or-llm","description":"Job title (e.g., \\"VP of Sales\\", \\"Software Engineer\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo account ID to associate with (e.g., \\"acc_abc123\\")"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the contact owner (accepted by Apollo but not officially documented for POST /contacts)"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the contact\'s employer (e.g., \\"Apollo\\")"},"website_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate website URL (e.g., \\"https://www.apollo.io/\\")"},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Lists/labels to add the contact to (e.g., [\\"Prospects\\"])"},"contact_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the contact stage"},"present_raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal location for the contact (e.g., \\"Atlanta, United States\\")"},"direct_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number"},"corporate_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Work/office phone number"},"mobile_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Mobile phone number"},"home_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Home phone number"},"other_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Alternative phone number"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom field values keyed by custom field ID"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, Apollo deduplicates against existing contacts"}},"hostedApiKey":"none"},"apollo_contact_search":{"id":"apollo_contact_search","name":"Apollo Search Contacts","description":"Search your team\'s contacts in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"q_keywords":{"type":"string","required":false,"visibility":"user-or-llm","description":"Keywords to search for"},"contact_stage_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by contact stage IDs"},"contact_label_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by Apollo label IDs (lists)"},"sort_by_field":{"type":"string","required":false,"visibility":"user-only","description":"Sort field: contact_last_activity_date, contact_email_last_opened_at, contact_email_last_clicked_at, contact_created_at, or contact_updated_at"},"sort_ascending":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, sort ascending. Must be used together with sort_by_field"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_contact_update":{"id":"apollo_contact_update","name":"Apollo Update Contact","description":"Update an existing contact in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"contact_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the contact to update (e.g., \\"con_abc123\\")"},"first_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"First name of the contact"},"last_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Last name of the contact"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address"},"title":{"type":"string","required":false,"visibility":"user-or-llm","description":"Job title (e.g., \\"VP of Sales\\", \\"Software Engineer\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo account ID (e.g., \\"acc_abc123\\")"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the contact owner (accepted by Apollo but not officially documented for PATCH /contacts/{id})"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the contact\'s employer (e.g., \\"Apollo\\")"},"website_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate website URL (e.g., \\"https://www.apollo.io/\\")"},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Lists/labels to add the contact to (e.g., [\\"Prospects\\"])"},"contact_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the contact stage"},"present_raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal location for the contact (e.g., \\"Atlanta, United States\\")"},"direct_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number"},"corporate_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Work/office phone number"},"mobile_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Mobile phone number"},"home_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Home phone number"},"other_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Alternative phone number"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom field values keyed by custom field ID"}},"hostedApiKey":"none"},"apollo_email_accounts":{"id":"apollo_email_accounts","name":"Apollo Get Email Accounts","description":"Get list of team\'s linked email accounts in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"}},"hostedApiKey":"none"},"apollo_opportunity_create":{"id":"apollo_opportunity_create","name":"Apollo Create Opportunity","description":"Create a new deal for an account in your Apollo database (master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the opportunity/deal (e.g., \\"Enterprise License - Q1\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"ID of the account this opportunity belongs to (e.g., \\"acc_abc123\\")"},"amount":{"type":"string","required":false,"visibility":"user-or-llm","description":"Monetary value as a plain number string with no commas or currency symbols"},"opportunity_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the opportunity stage"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the opportunity owner"},"closed_date":{"type":"string","required":false,"visibility":"user-or-llm","description":"Expected close date in YYYY-MM-DD format"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_opportunity_get":{"id":"apollo_opportunity_get","name":"Apollo Get Opportunity","description":"Retrieve complete details of a specific deal/opportunity by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"opportunity_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the opportunity to retrieve (e.g., \\"opp_abc123\\")"}},"hostedApiKey":"none"},"apollo_opportunity_search":{"id":"apollo_opportunity_search","name":"Apollo Search Opportunities","description":"Search and list all deals/opportunities in your team\'s Apollo account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"amount\\", \\"is_closed\\", or \\"is_won\\""},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_opportunity_update":{"id":"apollo_opportunity_update","name":"Apollo Update Opportunity","description":"Update an existing deal/opportunity in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"opportunity_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the opportunity to update (e.g., \\"opp_abc123\\")"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the opportunity/deal (e.g., \\"Enterprise License - Q1\\")"},"amount":{"type":"string","required":false,"visibility":"user-or-llm","description":"Monetary value as a plain number string with no commas or currency symbols"},"opportunity_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the opportunity stage"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the opportunity owner"},"closed_date":{"type":"string","required":false,"visibility":"user-or-llm","description":"Expected close date in YYYY-MM-DD format"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_organization_bulk_enrich":{"id":"apollo_organization_bulk_enrich","name":"Apollo Bulk Organization Enrichment","description":"Enrich data for up to 10 organizations at once using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"domains":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of company domains to enrich (max 10, no www. or @, e.g., [\\"apollo.io\\", \\"stripe.com\\"])"}},"hostedApiKey":"none"},"apollo_organization_enrich":{"id":"apollo_organization_enrich","name":"Apollo Organization Enrichment","description":"Enrich data for a single organization using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"domain":{"type":"string","required":true,"visibility":"user-or-llm","description":"Company domain (e.g., \\"apollo.io\\", \\"acme.com\\")"}},"hostedApiKey":"none"},"apollo_organization_search":{"id":"apollo_organization_search","name":"Apollo Organization Search","description":"Search Apollo\'s database for companies using filters","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"organization_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Company HQ locations (cities, US states, or countries)"},"organization_not_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Exclude companies whose HQ is in these locations"},"organization_num_employees_ranges":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employee count ranges as \\"min,max\\" strings (e.g., [\\"1,10\\", \\"250,500\\", \\"10000,20000\\"])"},"q_organization_keyword_tags":{"type":"array","required":false,"visibility":"user-or-llm","description":"Industry or keyword tags"},"q_organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Organization name to search for (e.g., \\"Acme\\", \\"TechCorp\\")"},"organization_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Apollo organization IDs to include (e.g., [\\"5e66b6381e05b4008c8331b8\\"])"},"q_organization_domains_list":{"type":"array","required":false,"visibility":"user-or-llm","description":"Domain names to filter by (no www. or @, up to 1,000)"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_people_bulk_enrich":{"id":"apollo_people_bulk_enrich","name":"Apollo Bulk People Enrichment","description":"Enrich data for up to 10 people at once using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"people":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of people to enrich (max 10)"},"reveal_personal_emails":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal personal email addresses (uses credits)"},"reveal_phone_number":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal phone numbers (uses credits, requires webhook_url)"},"webhook_url":{"type":"string","required":false,"visibility":"user-only","description":"Webhook URL for async phone number delivery (required when reveal_phone_number is true)"}},"hostedApiKey":"none"},"apollo_people_enrich":{"id":"apollo_people_enrich","name":"Apollo People Enrichment","description":"Enrich data for a single person using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"first_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"First name of the person"},"last_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Last name of the person"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Full name of the person (alternative to first_name/last_name)"},"id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the person"},"hashed_email":{"type":"string","required":false,"visibility":"user-or-llm","description":"MD5 or SHA-256 hashed email"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address of the person"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company name where the person works"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain (e.g., \\"apollo.io\\", \\"acme.com\\")"},"linkedin_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"LinkedIn profile URL"},"reveal_personal_emails":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal personal email addresses (uses credits)"},"reveal_phone_number":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal phone numbers (uses credits, requires webhook_url)"},"webhook_url":{"type":"string","required":false,"visibility":"user-only","description":"Webhook URL for async phone number delivery (required when reveal_phone_number is true)"}},"hostedApiKey":"none"},"apollo_people_search":{"id":"apollo_people_search","name":"Apollo People Search","description":"Search Apollo\'s database for people using demographic filters","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"person_titles":{"type":"array","required":false,"visibility":"user-or-llm","description":"Job titles to search for (e.g., [\\"CEO\\", \\"VP of Sales\\"])"},"include_similar_titles":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to return people with job titles similar to person_titles"},"person_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Locations to search in (e.g., [\\"San Francisco, CA\\", \\"New York, NY\\"])"},"person_seniorities":{"type":"array","required":false,"visibility":"user-or-llm","description":"Seniority levels (one of: owner, founder, c_suite, partner, vp, head, director, manager, senior, entry, intern)"},"organization_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Apollo organization IDs to filter by (e.g., [\\"5e66b6381e05b4008c8331b8\\"])"},"organization_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Company names to search within (legacy filter)"},"organization_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Headquarters locations of the people\'s current employer (e.g., [\'texas\', \'tokyo\', \'spain\'])"},"q_organization_domains_list":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employer domain names (e.g., [\\"apollo.io\\", \\"microsoft.com\\"]) — up to 1,000, no www. or @"},"organization_num_employees_ranges":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employee count ranges for the person\'s current employer. Each entry is \\"min,max\\" (e.g., [\\"1,10\\", \\"250,500\\", \\"10000,20000\\"])"},"contact_email_status":{"type":"array","required":false,"visibility":"user-or-llm","description":"Email statuses to filter by: \\"verified\\", \\"unverified\\", \\"likely to engage\\", \\"unavailable\\""},"q_keywords":{"type":"string","required":false,"visibility":"user-or-llm","description":"Keywords to search for"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination, default 1 (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, default 25, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_sequence_add_contacts":{"id":"apollo_sequence_add_contacts","name":"Apollo Add Contacts to Sequence","description":"Add contacts to an Apollo sequence","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"sequence_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the sequence to add contacts to (e.g., \\"seq_abc123\\")"},"contact_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of contact IDs to add to the sequence (e.g., [\\"con_abc123\\", \\"con_def456\\"]). Either contact_ids or label_names must be provided."},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of label names to identify contacts to add to the sequence. Either contact_ids or label_names must be provided."},"send_email_from_email_account_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the email account to send from. Use the Get Email Accounts operation to look this up."},"send_email_from_email_address":{"type":"string","required":false,"visibility":"user-only","description":"Specific email address to send from within the email account."},"sequence_no_email":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if they have no email address"},"sequence_unverified_email":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts with unverified email addresses"},"sequence_job_change":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts who recently changed jobs"},"sequence_active_in_other_campaigns":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts active in other campaigns"},"sequence_finished_in_other_campaigns":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts who finished other campaigns"},"sequence_same_company_in_same_campaign":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if others from the same company are in the sequence"},"contacts_without_ownership_permission":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts without ownership permission"},"add_if_in_queue":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if they are in the queue"},"contact_verification_skipped":{"type":"boolean","required":false,"visibility":"user-only","description":"Skip contact verification when adding"},"user_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the user performing the action"},"status":{"type":"string","required":false,"visibility":"user-only","description":"Initial status for added contacts: \\"active\\" or \\"paused\\""},"auto_unpause_at":{"type":"string","required":false,"visibility":"user-only","description":"ISO 8601 datetime to automatically unpause contacts"}},"hostedApiKey":"none"},"apollo_sequence_search":{"id":"apollo_sequence_search","name":"Apollo Search Sequences","description":"Search for sequences/campaigns in your team\'s Apollo account (master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"q_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search sequences by name (e.g., \\"Outbound Q1\\", \\"Follow-up\\")"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_task_create":{"id":"apollo_task_create","name":"Apollo Create Task","description":"Create one or more tasks in Apollo (one task per contact_id, master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"user_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the Apollo user the task is assigned to"},"contact_ids":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of contact IDs. One task is created per contact."},"priority":{"type":"string","required":false,"visibility":"user-or-llm","description":"Task priority: \\"high\\", \\"medium\\", or \\"low\\" (defaults to \\"medium\\")"},"due_at":{"type":"string","required":true,"visibility":"user-or-llm","description":"Due date/time in ISO 8601 format (e.g., \\"2024-12-31T23:59:59Z\\")"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task type: \\"call\\", \\"outreach_manual_email\\", \\"linkedin_step_connect\\", \\"linkedin_step_message\\", \\"linkedin_step_view_profile\\", \\"linkedin_step_interact_post\\", or \\"action_item\\""},"status":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task status: \\"scheduled\\", \\"completed\\", or \\"skipped\\""},"note":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-form note providing context for the task"}},"hostedApiKey":"none"},"apollo_task_search":{"id":"apollo_task_search","name":"Apollo Search Tasks","description":"Search for tasks in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"task_due_at\\" or \\"task_priority\\""},"open_factor_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Filter by status. Common values: [\\"task_types\\"] for open tasks, [\\"task_completed_at\\"] for completed tasks."},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"appconfig_create_application":{"id":"appconfig_create_application","name":"AppConfig Create Application","description":"Create an application in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the application to create"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the application"}},"hostedApiKey":"none"},"appconfig_create_configuration_profile":{"id":"appconfig_create_configuration_profile","name":"AppConfig Create Configuration Profile","description":"Create a configuration profile in an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to create the configuration profile in"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the configuration profile"},"locationUri":{"type":"string","required":true,"visibility":"user-or-llm","description":"Where the configuration is stored. Use \\"hosted\\" for AppConfig-hosted configurations, or an SSM/S3 URI"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the configuration profile"},"retrievalRoleArn":{"type":"string","required":false,"visibility":"user-or-llm","description":"ARN of an IAM role to retrieve the configuration (required for non-hosted URIs)"},"type":{"type":"string","required":false,"visibility":"user-or-llm","description":"Profile type: AWS.Freeform (default) or AWS.AppConfig.FeatureFlags"}},"hostedApiKey":"none"},"appconfig_create_environment":{"id":"appconfig_create_environment","name":"AppConfig Create Environment","description":"Create an environment for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to create the environment in"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the environment to create"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the environment"}},"hostedApiKey":"none"},"appconfig_create_hosted_configuration_version":{"id":"appconfig_create_hosted_configuration_version","name":"AppConfig Create Hosted Configuration Version","description":"Create a new hosted configuration version for an AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to add the version to"},"content":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration content (e.g., a JSON or YAML document)"},"contentType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Content type of the configuration (e.g., application/json, text/plain)"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the configuration version"},"latestVersionNumber":{"type":"number","required":false,"visibility":"user-or-llm","description":"The version number of the latest version, used for optimistic concurrency"},"versionLabel":{"type":"string","required":false,"visibility":"user-or-llm","description":"A user-defined label for the configuration version"}},"hostedApiKey":"none"},"appconfig_delete_application":{"id":"appconfig_delete_application","name":"AppConfig Delete Application","description":"Delete an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to delete"}},"hostedApiKey":"none"},"appconfig_delete_configuration_profile":{"id":"appconfig_delete_configuration_profile","name":"AppConfig Delete Configuration Profile","description":"Delete an AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to delete"}},"hostedApiKey":"none"},"appconfig_delete_environment":{"id":"appconfig_delete_environment","name":"AppConfig Delete Environment","description":"Delete an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to delete"}},"hostedApiKey":"none"},"appconfig_delete_hosted_configuration_version":{"id":"appconfig_delete_hosted_configuration_version","name":"AppConfig Delete Hosted Configuration Version","description":"Delete a specific hosted configuration version from an AppConfig profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID that owns the version"},"versionNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The version number to delete"}},"hostedApiKey":"none"},"appconfig_get_application":{"id":"appconfig_get_application","name":"AppConfig Get Application","description":"Get details about a single AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to retrieve"}},"hostedApiKey":"none"},"appconfig_get_configuration":{"id":"appconfig_get_configuration","name":"AppConfig Get Configuration","description":"Retrieve the latest deployed configuration for an AppConfig application, environment, and profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID or name to retrieve configuration for"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID or name to retrieve configuration for"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID or name to retrieve"}},"hostedApiKey":"none"},"appconfig_get_configuration_profile":{"id":"appconfig_get_configuration_profile","name":"AppConfig Get Configuration Profile","description":"Get details about a single AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to retrieve"}},"hostedApiKey":"none"},"appconfig_get_deployment":{"id":"appconfig_get_deployment","name":"AppConfig Get Deployment","description":"Get details about a specific AWS AppConfig deployment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployment"},"deploymentNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The sequence number of the deployment"}},"hostedApiKey":"none"},"appconfig_get_environment":{"id":"appconfig_get_environment","name":"AppConfig Get Environment","description":"Get details about a single AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to retrieve"}},"hostedApiKey":"none"},"appconfig_get_hosted_configuration_version":{"id":"appconfig_get_hosted_configuration_version","name":"AppConfig Get Hosted Configuration Version","description":"Retrieve a specific hosted configuration version from an AppConfig profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to read the version from"},"versionNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The version number to retrieve"}},"hostedApiKey":"none"},"appconfig_list_applications":{"id":"appconfig_list_applications","name":"AppConfig List Applications","description":"List applications in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of applications to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_configuration_profiles":{"id":"appconfig_list_configuration_profiles","name":"AppConfig List Configuration Profiles","description":"List configuration profiles for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profiles"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of configuration profiles to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_deployment_strategies":{"id":"appconfig_list_deployment_strategies","name":"AppConfig List Deployment Strategies","description":"List deployment strategies available in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of deployment strategies to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_deployments":{"id":"appconfig_list_deployments","name":"AppConfig List Deployments","description":"List deployments for an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployments"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployments"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of deployments to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_environments":{"id":"appconfig_list_environments","name":"AppConfig List Environments","description":"List environments for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environments"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of environments to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_hosted_configuration_versions":{"id":"appconfig_list_hosted_configuration_versions","name":"AppConfig List Hosted Configuration Versions","description":"List hosted configuration versions for an AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to list versions for"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of versions to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_start_deployment":{"id":"appconfig_start_deployment","name":"AppConfig Start Deployment","description":"Start deploying a configuration version to an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to deploy in"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to deploy to"},"deploymentStrategyId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The deployment strategy ID to use"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to deploy"},"configurationVersion":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration version to deploy"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the deployment"}},"hostedApiKey":"none"},"appconfig_stop_deployment":{"id":"appconfig_stop_deployment","name":"AppConfig Stop Deployment","description":"Stop an in-progress AWS AppConfig deployment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployment"},"deploymentNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The sequence number of the deployment to stop"}},"hostedApiKey":"none"},"appconfig_update_application":{"id":"appconfig_update_application","name":"AppConfig Update Application","description":"Update the name or description of an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the application"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the application"}},"hostedApiKey":"none"},"appconfig_update_configuration_profile":{"id":"appconfig_update_configuration_profile","name":"AppConfig Update Configuration Profile","description":"Update the name, description, or retrieval role of an AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the configuration profile"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the configuration profile"},"retrievalRoleArn":{"type":"string","required":false,"visibility":"user-or-llm","description":"New ARN of the IAM role used to retrieve the configuration"}},"hostedApiKey":"none"},"appconfig_update_environment":{"id":"appconfig_update_environment","name":"AppConfig Update Environment","description":"Update the name or description of an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the environment"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the environment"}},"hostedApiKey":"none"},"arxiv_get_author_papers":{"id":"arxiv_get_author_papers","name":"ArXiv Get Author Papers","description":"Search for papers by a specific author on ArXiv.","version":"1.0.0","params":{"authorName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Author name to search for"},"maxResults":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 10, max: 2000)"}},"hostedApiKey":"none"},"arxiv_get_paper":{"id":"arxiv_get_paper","name":"ArXiv Get Paper","description":"Get detailed information about a specific ArXiv paper by its ID.","version":"1.0.0","params":{"paperId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ArXiv paper ID (e.g., \\"1706.03762\\")"}},"hostedApiKey":"none"},"arxiv_search":{"id":"arxiv_search","name":"ArXiv Search","description":"Search for academic papers on ArXiv by keywords, authors, titles, or other fields.","version":"1.0.0","params":{"searchQuery":{"type":"string","required":true,"visibility":"user-or-llm","description":"The search query to execute"},"searchField":{"type":"string","required":false,"visibility":"user-only","description":"Field to search in: all, ti (title), au (author), abs (abstract), co (comment), jr (journal), cat (category), rn (report number)"},"maxResults":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 10, max: 2000)"},"sortBy":{"type":"string","required":false,"visibility":"user-only","description":"Sort by: relevance, lastUpdatedDate, submittedDate (default: relevance)"},"sortOrder":{"type":"string","required":false,"visibility":"user-only","description":"Sort order: ascending, descending (default: descending)"}},"hostedApiKey":"none"},"asana_add_comment":{"id":"asana_add_comment","name":"Asana Add Comment","description":"Add a comment (story) to an Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana task GID (numeric string)"},"text":{"type":"string","required":true,"visibility":"user-or-llm","description":"The text content of the comment"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_add_followers":{"id":"asana_add_followers","name":"Asana Add Followers","description":"Add one or more followers to an Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana task (numeric string)"},"followers":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of user GIDs to add as followers to the task"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_project":{"id":"asana_create_project","name":"Asana Create Project","description":"Create a new project in an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) where the project will be created"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the project"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the project"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_section":{"id":"asana_create_section","name":"Asana Create Section","description":"Create a new section in an Asana project","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana project (numeric string) to add the section to"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the section"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_subtask":{"id":"asana_create_subtask","name":"Asana Create Subtask","description":"Create a subtask under an existing Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the parent Asana task (numeric string)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the subtask"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the subtask"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"User GID to assign the subtask to"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_task":{"id":"asana_create_task","name":"Asana Create Task","description":"Create a new task in Asana","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) where the task will be created"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the task"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the task"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"User GID to assign the task to"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_delete_task":{"id":"asana_delete_task","name":"Asana Delete Task","description":"Delete an Asana task by its GID (moves it to the trash)","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana task to delete (numeric string)"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_get_project":{"id":"asana_get_project","name":"Asana Get Project","description":"Retrieve a single Asana project by its GID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana project GID (numeric string) to retrieve"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_get_projects":{"id":"asana_get_projects","name":"Asana Get Projects","description":"Retrieve all projects from an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to retrieve projects from"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_get_task":{"id":"asana_get_task","name":"Asana Get Task","description":"Retrieve a single task by GID or get multiple tasks with filters","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":false,"visibility":"user-or-llm","description":"The globally unique identifier (GID) of the task. If not provided, will get multiple tasks."},"workspace":{"type":"string","required":false,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to filter tasks (required when not using taskGid)"},"project":{"type":"string","required":false,"visibility":"user-or-llm","description":"Asana project GID (numeric string) to filter tasks"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of tasks to return (default: 50)"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_list_sections":{"id":"asana_list_sections","name":"Asana List Sections","description":"List all sections in an Asana project","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana project (numeric string) to list sections from"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_list_workspaces":{"id":"asana_list_workspaces","name":"Asana List Workspaces","description":"List all Asana workspaces and organizations the authenticated user belongs to","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_search_tasks":{"id":"asana_search_tasks","name":"Asana Search Tasks","description":"Search for tasks in an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to search tasks in"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Text to search for in task names"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter tasks by assignee user GID"},"projects":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of Asana project GIDs (numeric strings) to filter tasks by"},"completed":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Filter by completion status"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_update_task":{"id":"asana_update_task","name":"Asana Update Task","description":"Update an existing task in Asana","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana task GID (numeric string) of the task to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated name for the task"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated notes or description for the task"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated assignee user GID"},"completed":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Mark task as completed or not completed"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"ashby_add_candidate_tag":{"id":"ashby_add_candidate_tag","name":"Ashby Add Candidate Tag","description":"Adds a tag to a candidate in Ashby and returns the updated candidate.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to add the tag to"},"tagId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the tag to add"}},"hostedApiKey":"none"},"ashby_anonymize_candidate":{"id":"ashby_anonymize_candidate","name":"Ashby Anonymize Candidate","description":"Strips personally identifiable information from a candidate in Ashby. This does not delete the candidate - the record and its applications remain, with the PII removed. Ashby exposes no candidate deletion endpoint; true deletion is UI-only, restricted by role, and limited to a 10-day window. Requires the candidatesWrite permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"UUID of the candidate to anonymize"}},"hostedApiKey":"none"},"ashby_change_application_source":{"id":"ashby_change_application_source","name":"Ashby Change Application Source","description":"Changes the source attributed to an existing application, so programmatically created applications report correctly on the recruiting side. Requires the candidatesWrite permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"UUID of the application whose source should change"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to attribute the application to, as returned by List Sources. Omit only when unsetSource is true."},"unsetSource":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Set true to deliberately clear the application source. Required to unset, so that a missing or empty sourceId cannot wipe attribution by accident."}},"hostedApiKey":"none"},"ashby_change_application_stage":{"id":"ashby_change_application_stage","name":"Ashby Change Application Stage","description":"Moves an application to a different interview stage. Requires an archive reason when moving to an Archived stage.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the application to update the stage of"},"interviewStageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the interview stage to move the application to"},"archiveReasonId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Archive reason UUID. Required when moving to an Archived stage, ignored otherwise"},"archiveEmail":{"type":"json","required":false,"visibility":"user-or-llm","description":"Archive email configuration with communicationTemplateId and optional sendAt ISO 8601 timestamp. Pass null or omit to send no archive email."}},"hostedApiKey":"none"},"ashby_create_application":{"id":"ashby_create_application","name":"Ashby Create Application","description":"Creates a new application for a candidate on a job. Optionally specify interview plan, stage, source, and credited user.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to consider for the job"},"jobId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the job to consider the candidate for"},"interviewPlanId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the interview plan to use (defaults to the job default plan)"},"interviewStageId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the interview stage to place the application in, or FirstPreInterviewScreen (defaults to the first Lead stage)"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to set on the application"},"creditedToUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the user the application is credited to"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to set as the application creation date (defaults to now)"},"applicationHistory":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional documented application history entries to create with the application"}},"hostedApiKey":"none"},"ashby_create_candidate":{"id":"ashby_create_candidate","name":"Ashby Create Candidate","description":"Creates a new candidate record in Ashby.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"The candidate full name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary email address for the candidate"},"phoneNumber":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number for the candidate"},"linkedInUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"LinkedIn profile URL"},"githubUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"GitHub profile URL"},"website":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal website URL"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to attribute the candidate to"},"creditedToUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the Ashby user to credit with sourcing this candidate"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"Backdated creation timestamp in ISO 8601 (e.g. 2024-01-01T00:00:00Z). Defaults to now."},"alternateEmailAddresses":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of additional email address strings to add to the candidate, e.g. [\\"a@x.com\\",\\"b@y.com\\"]"},"location":{"type":"json","required":false,"visibility":"user-or-llm","description":"Candidate location object with optional city, region, and country"}},"hostedApiKey":"none"},"ashby_create_note":{"id":"ashby_create_note","name":"Ashby Create Note","description":"Creates a note on a candidate in Ashby. Supports plain text and HTML content (bold, italic, underline, links, lists, code).","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to add the note to"},"note":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note content. If noteType is text/html, supports: , , ,
,

{access.accessMode === 'members' - ? 'Documents stay hidden until members connect and sync. Listing caps are cleared.' + ? 'Everyone in the workspace is invited to connect their account. Documents stay hidden until members connect and sync; listing caps are cleared.' : 'Every workspace member can read every synced document once the next sync completes.'}

diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts index d26e85dc9e3..38dda1edd69 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts @@ -1,3 +1,4 @@ +export { MemberConnectBanner } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner' export { ActionBar } from './action-bar' export { AddConnectorModal } from './add-connector-modal' export { AddDocumentsModal } from './add-documents-modal' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx new file mode 100644 index 00000000000..670f01e514b --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx @@ -0,0 +1,87 @@ +'use client' + +import { useState } from 'react' +import { Button } from '@sim/emcn' +import { createLogger } from '@sim/logger' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import { + type ConnectorData, + useStartConnectorMemberEnrollment, +} from '@/hooks/queries/kb/connectors' + +const logger = createLogger('MemberConnectBanner') + +interface MemberConnectBannerProps { + knowledgeBaseId: string + connectors: ConnectorData[] +} + +/** + * Asks the viewer to connect their own account for every per-member connector + * they have not connected yet. Connecting is the one thing a member has to do + * to see the documents shared with them; everything else happens on its own. + */ +export function MemberConnectBanner({ knowledgeBaseId, connectors }: MemberConnectBannerProps) { + const { mutate: startEnrollment, isPending } = useStartConnectorMemberEnrollment() + const [pendingConnectorId, setPendingConnectorId] = useState(null) + const [error, setError] = useState(null) + + const awaiting = connectors.filter( + (connector) => + connector.accessMode === 'members' && + connector.viewerMembership !== null && + connector.viewerMembership !== undefined && + connector.viewerMembership !== 'connected' + ) + if (awaiting.length === 0) return null + + const connect = (connectorId: string) => { + setError(null) + setPendingConnectorId(connectorId) + startEnrollment( + { knowledgeBaseId, connectorId }, + { + onSuccess: ({ url }) => { + window.location.assign(url) + }, + onError: (err) => { + logger.error('Failed to start member enrollment', { error: err.message }) + setError(err.message) + setPendingConnectorId(null) + }, + } + ) + } + + return ( +
+ {awaiting.map((connector) => { + const name = + CONNECTOR_META_REGISTRY[connector.connectorType]?.name ?? connector.connectorType + const reconnect = connector.viewerMembership === 'needs_reauth' + const busy = isPending && pendingConnectorId === connector.id + return ( +
+

+ {reconnect + ? `Reconnect your ${name} account to keep seeing the documents shared with you.` + : `Connect your ${name} account to see the documents shared with you.`} +

+ +
+ ) + })} + {error &&

{error}

} +
+ ) +} diff --git a/apps/sim/connectors/permission-scoped-listing.test.ts b/apps/sim/connectors/permission-scoped-listing.test.ts index 6f57101bb40..71490455975 100644 --- a/apps/sim/connectors/permission-scoped-listing.test.ts +++ b/apps/sim/connectors/permission-scoped-listing.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest' import { getManagedOAuthConnectorPolicy } from '@/lib/auth/connectors/managed-oauth' import { getCredentialGroupProviderService, - isCredentialGroupProvider, + getCredentialGroupStandardOAuthProviderFromProviderId, } from '@/lib/credential-groups/providers' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' @@ -35,11 +35,10 @@ describe('permission-scoped connector listings', () => { expect(policy).toBeDefined() if (!policy) return - const groupProvider = isCredentialGroupProvider(meta.auth.provider) - ? meta.auth.provider - : undefined + const groupProvider = getCredentialGroupStandardOAuthProviderFromProviderId( + meta.auth.provider + ) expect(groupProvider).toBeDefined() - if (!groupProvider) return const optionScopes = [ ...new Set([ diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 445fde1e2eb..7549b993eeb 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -19,16 +19,20 @@ import { listKnowledgeConnectorsContract, type MemberSyncLogData, patchKnowledgeConnectorDocumentsContract, + type StartKnowledgeConnectorMemberEnrollmentData, type SyncLogData, + startKnowledgeConnectorMemberEnrollmentContract, triggerKnowledgeConnectorSyncContract, type UpdateConnectorAccessBody, updateKnowledgeConnectorAccessContract, updateKnowledgeConnectorContract, + type ViewerConnectorMembership, } from '@/lib/api/contracts/knowledge' import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' export type { + ViewerConnectorMembership, ConnectorData, ConnectorDetailData, ConnectorMemberSummary, @@ -334,6 +338,26 @@ async function updateConnectorAccess({ * list and detail for the new mode and member state, and the document lists * whose rows may have become hidden or visible. */ +interface StartConnectorMemberEnrollmentParams { + knowledgeBaseId: string + connectorId: string +} + +async function startConnectorMemberEnrollment({ + knowledgeBaseId, + connectorId, +}: StartConnectorMemberEnrollmentParams): Promise { + const response = await requestJson(startKnowledgeConnectorMemberEnrollmentContract, { + params: { id: knowledgeBaseId, connectorId }, + }) + return response.data +} + +/** Mints the viewer's enrollment link for a per-member connector; the caller navigates to it. */ +export function useStartConnectorMemberEnrollment() { + return useMutation({ mutationFn: startConnectorMemberEnrollment }) +} + export function useUpdateConnectorAccess() { const queryClient = useQueryClient() diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts index 92052a6afe3..c0d8118f9ec 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts @@ -19,9 +19,10 @@ describe('connector access binding contracts', () => { expect(parsed.syncIntervalMinutes).toBe(1440) }) - it('requires both group ids for members mode and refuses a credential there', () => { + it('lets members mode omit the binding, refuses half a binding, and refuses a credential there', () => { + /** No binding named: the server provisions a credential group for the connector. */ expect(createConnectorBodySchema.safeParse({ ...base, accessMode: 'members' }).success).toBe( - false + true ) expect( createConnectorBodySchema.safeParse({ diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index 894ac16aabc..cd135a59202 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -41,18 +41,12 @@ function requireAccessBinding( ctx: z.RefinementCtx ): void { if (value.accessMode === 'members') { - if (!value.credentialGroupId) { + /** Both name one option, or neither and the server provisions one. */ + if (Boolean(value.credentialGroupId) !== Boolean(value.credentialGroupOptionId)) { ctx.addIssue({ code: 'custom', - path: ['credentialGroupId'], - message: 'credentialGroupId is required when accessMode is members', - }) - } - if (!value.credentialGroupOptionId) { - ctx.addIssue({ - code: 'custom', - path: ['credentialGroupOptionId'], - message: 'credentialGroupOptionId is required when accessMode is members', + path: [value.credentialGroupId ? 'credentialGroupOptionId' : 'credentialGroupId'], + message: 'credentialGroupId and credentialGroupOptionId go together', }) } return @@ -138,6 +132,15 @@ export const connectorDocumentsPatchBodySchema = z.object({ .max(MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS), }) +export const VIEWER_CONNECTOR_MEMBERSHIPS = [ + 'connected', + 'needs_reauth', + 'invited', + 'not_enrolled', +] as const +export const viewerConnectorMembershipSchema = z.enum(VIEWER_CONNECTOR_MEMBERSHIPS) +export type ViewerConnectorMembership = z.output + export const connectorDataSchema = z .object({ id: z.string(), @@ -155,6 +158,11 @@ export const connectorDataSchema = z nextSyncAt: z.string().nullable(), consecutiveFailures: z.number(), accessMode: connectorAccessModeSchema, + /** + * Where the viewer stands with a per-member connector: absent for a + * workspace-mode connector or a caller with no person behind it. + */ + viewerMembership: viewerConnectorMembershipSchema.nullable().optional(), credentialGroupId: z.string().nullable(), credentialGroupOptionId: z.string().nullable(), /** Members mode only; `idle` otherwise. */ @@ -317,6 +325,24 @@ export const updateKnowledgeConnectorAccessContract = defineRouteContract({ }, }) +export const startKnowledgeConnectorMemberEnrollmentDataSchema = z.object({ + /** The viewer's enrollment link; opening it connects their account. */ + url: z.string().url(), +}) +export type StartKnowledgeConnectorMemberEnrollmentData = z.output< + typeof startKnowledgeConnectorMemberEnrollmentDataSchema +> + +export const startKnowledgeConnectorMemberEnrollmentContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/[id]/connectors/[connectorId]/enroll', + params: knowledgeConnectorParamsSchema, + response: { + mode: 'json', + schema: successResponseSchema(startKnowledgeConnectorMemberEnrollmentDataSchema), + }, +}) + export const deleteKnowledgeConnectorContract = defineRouteContract({ method: 'DELETE', path: '/api/knowledge/[id]/connectors/[connectorId]', diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index 996cd12ad0a..915351ed761 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -1,8 +1,11 @@ import { AuditAction, AuditResourceType } from '@sim/audit' -import type { Principal } from '@sim/auth/principal' +import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeAttributedUserId, @@ -23,6 +26,57 @@ import { getKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' import type { KnowledgeOperationSource } from '@/lib/knowledge/orchestration/shared' import { getConnectorMeta } from '@/connectors/registry' +const logger = createLogger('KnowledgeConnectorAccessApplication') + +/** Provisioning pulls in the credential-group services; loaded only when a members-mode switch needs it. */ +async function loadMemberProvisioning() { + return import('@/lib/knowledge/connectors/member-provisioning') +} + +export interface StartKnowledgeConnectorMemberEnrollmentInput { + knowledgeBaseId: string + connectorId: string + assertedWorkspaceId?: string +} + +/** + * Hands a workspace member the link that connects their own account to a + * per-member connector, minted on demand so they never need the invitation + * email. Only widens what the member themselves can see. + */ +export const startKnowledgeConnectorMemberEnrollment = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.enrollConnectorMember, + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: StartKnowledgeConnectorMemberEnrollmentInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), + async execute({ principal, context }) { + const workspaceId = requireConnectorWorkspaceId(context) + const userId = resolvePrincipalSubjectUserId(principal) + if (!userId) throw new OrchestrationError('forbidden', 'Sign in to connect your account') + const connector = await getKnowledgeConnector(context.knowledgeBaseId, context.connectorId) + if (!connector) throw new OrchestrationError('not_found', 'Connector not found') + if (connector.accessMode !== 'members' || !connector.credentialGroupId) { + throw new OrchestrationError('validation', 'This connector does not sync per member') + } + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId }))) { + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) + } + const url = await (await loadMemberProvisioning()).createViewerConnectorEnrollmentLink({ + userId, + workspaceId, + credentialGroupId: connector.credentialGroupId, + }) + return { url } + }, +}) + export interface UpdateKnowledgeConnectorAccessInput { knowledgeBaseId: string connectorId: string @@ -63,6 +117,7 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ ) } + const subjectUserId = resolvePrincipalSubjectUserId(principal) const target = input.accessMode === 'members' ? { @@ -70,16 +125,19 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ binding: await resolveKnowledgeConnectorMembersBinding({ workspaceId, connectorMeta, - binding: { - credentialGroupId: requireBindingField( - input.credentialGroupId, - 'credentialGroupId' - ), - credentialGroupOptionId: requireBindingField( - input.credentialGroupOptionId, - 'credentialGroupOptionId' - ), - }, + binding: + input.credentialGroupId && input.credentialGroupOptionId + ? { + credentialGroupId: input.credentialGroupId, + credentialGroupOptionId: input.credentialGroupOptionId, + } + : await ( + await loadMemberProvisioning() + ).provisionKnowledgeConnectorMembersBinding({ + workspaceId, + connectorMeta, + userId: actingUserId, + }), sourceConfig: connector.sourceConfig as Record, }), } @@ -107,6 +165,22 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ request, }) requireSuccessfulOutcome(outcome, 'Knowledge connector access update failed') + if (target.accessMode === 'members' && outcome.changed) { + const provisioning = await loadMemberProvisioning() + await provisioning + .inviteWorkspaceMembersToCredentialGroup({ + workspaceId, + credentialGroupId: target.binding.credentialGroupId, + inviterUserId: subjectUserId ?? undefined, + limit: provisioning.MEMBER_PROVISION_INVITES_PER_REQUEST, + }) + .catch((error) => { + logger.warn('Failed to invite workspace members after switching to members mode', { + workspaceId, + error: getErrorMessage(error), + }) + }) + } return { connector: outcome.connector, changed: outcome.changed, workspaceId } }, projectAudit: ({ input, context, result }) => @@ -133,11 +207,6 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ : [], }) -function requireBindingField(value: string | undefined, field: string): string { - if (!value) throw new OrchestrationError('validation', `${field} is required for members mode`) - return value -} - /** * Workspace mode needs a credential the caller may use, and one that yields a * token, since the connector syncs as it from then on. An API-key connector diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 48535e39079..fcf36ed6a7f 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -9,6 +9,8 @@ import { knowledgeConnectorMemberSyncLog, knowledgeConnectorSyncLog, } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { and, asc, count, desc, eq, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { decryptApiKey } from '@/lib/api-key/crypto' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' @@ -62,6 +64,13 @@ interface KnowledgeConnectorApplicationInput { source?: KnowledgeOperationSource } +const logger = createLogger('KnowledgeConnectorsApplication') + +/** Provisioning pulls in the credential-group services; loaded only when a members-mode connector needs it. */ +async function loadMemberProvisioning() { + return import('@/lib/knowledge/connectors/member-provisioning') +} + export interface ListKnowledgeConnectorsInput extends KnowledgeConnectorApplicationInput { knowledgeBaseId: string sortBy?: 'connectorType' | 'createdAt' | 'updatedAt' @@ -293,7 +302,7 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ principal: Principal input: ListKnowledgeConnectorsInput }) => resolveActiveKnowledgeResourceContext(input, principal), - async execute({ input, context }) { + async execute({ principal, input, context }) { const sortOrder = input.sortOrder === 'asc' ? asc : desc const sortColumn = input.sortBy === 'connectorType' @@ -319,8 +328,20 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ : await orderedQuery.limit(input.limit + 1).offset(offset) const hasMore = input.limit !== undefined && rows.length > input.limit const page = input.limit === undefined ? rows : rows.slice(0, input.limit) + const viewerUserId = resolvePrincipalSubjectUserId(principal) + const memberships = + viewerUserId && context.workspaceId + ? await (await loadMemberProvisioning()).resolveViewerConnectorMemberships({ + userId: viewerUserId, + workspaceId: context.workspaceId, + connectors: page, + }) + : new Map() return { - connectors: page.map(({ encryptedApiKey: _encryptedApiKey, ...rest }) => rest), + connectors: page.map(({ encryptedApiKey: _encryptedApiKey, ...rest }) => ({ + ...rest, + viewerMembership: memberships.get(rest.id) ?? null, + })), hasMore, offset, limit: input.limit ?? page.length, @@ -425,19 +446,21 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ if (!connectorMeta) { throw new OrchestrationError('validation', `Unknown connector type: ${input.connectorType}`) } - if (!input.credentialGroupId || !input.credentialGroupOptionId) { - throw new OrchestrationError( - 'validation', - 'credentialGroupId and credentialGroupOptionId are required for members mode' - ) - } + const named = + input.credentialGroupId && input.credentialGroupOptionId + ? { + credentialGroupId: input.credentialGroupId, + credentialGroupOptionId: input.credentialGroupOptionId, + } + : await (await loadMemberProvisioning()).provisionKnowledgeConnectorMembersBinding({ + workspaceId, + connectorMeta, + userId: subjectUserId, + }) membersBinding = await resolveKnowledgeConnectorMembersBinding({ workspaceId, connectorMeta, - binding: { - credentialGroupId: input.credentialGroupId, - credentialGroupOptionId: input.credentialGroupOptionId, - }, + binding: named, sourceConfig: input.sourceConfig, }) } @@ -467,6 +490,29 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ recordProductAnalytics: false, }) requireSuccessfulOutcome(outcome, 'Knowledge connector creation failed') + if (membersBinding && outcome.success) { + /** + * Everyone in the workspace is invited to connect, so the admin's only + * next step is to wait. Bounded here; the member run invites the rest. + */ + const provisioning = await loadMemberProvisioning() + await provisioning + .inviteWorkspaceMembersToCredentialGroup({ + workspaceId, + credentialGroupId: membersBinding.credentialGroupId, + inviterUserId: resolvePrincipalSubjectUserId(principal) ?? undefined, + limit: provisioning.MEMBER_PROVISION_INVITES_PER_REQUEST, + }) + .catch((error) => { + logger.warn( + 'Failed to invite workspace members after creating a members-mode connector', + { + workspaceId, + error: getErrorMessage(error), + } + ) + }) + } return { connector: outcome.connector, workspaceId } }, projectAudit: ({ input, context, result }) => ({ diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index d6fdb40895d..e8ccb213d12 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -56,6 +56,7 @@ describe('knowledge operation registry', () => { 'knowledge.connectors.create', 'knowledge.connectors.update', 'knowledge.connectors.access.update', + 'knowledge.connectors.members.enroll', 'knowledge.connectors.delete', 'knowledge.connectors.sync', 'knowledge.connectors.documents.list', diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index 05ee5ea1bd0..1b6aee03ab5 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -398,6 +398,17 @@ export const knowledgeOperations = { capability: 'knowledge.use', principalKinds: ['session'], }), + /** + * A workspace member joining a per-member connector: any reader may connect + * their own account, which only ever widens what they themselves see. + */ + enrollConnectorMember: defineWorkspaceOperation({ + id: 'knowledge.connectors.members.enroll', + minimumRole: 'read', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }), deleteConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.delete', minimumRole: 'write', diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts new file mode 100644 index 00000000000..4cea7cfdf6c --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts @@ -0,0 +1,65 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/credential-groups/enrollments', () => ({ + createCredentialGroupInvitationLink: vi.fn(), + inviteCredentialGroupEnrollment: vi.fn(), + loadCredentialGroupInviterIdentity: vi.fn(), +})) +vi.mock('@/lib/credential-groups/service', () => ({ + createCredentialGroup: vi.fn(), + listCredentialGroups: vi.fn(), +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ getUsersWithPermissions: vi.fn() })) + +import { + deriveViewerConnectorMembership, + pickProvisionedGroupName, +} from '@/lib/knowledge/connectors/member-provisioning' + +describe('pickProvisionedGroupName', () => { + it('names the group after the connector and steps past taken names', () => { + expect(pickProvisionedGroupName('Google Drive', [])).toBe('Google Drive access') + expect(pickProvisionedGroupName('Google Drive', ['google drive access'])).toBe( + 'Google Drive access 2' + ) + expect( + pickProvisionedGroupName('Google Drive', ['Google Drive access', 'Google Drive access 2']) + ).toBe('Google Drive access 3') + }) + + it('gives up with a pointer to Settings once every candidate is taken', () => { + const taken = [ + 'Google Drive access', + 'Google Drive access 2', + 'Google Drive access 3', + 'Google Drive access 4', + 'Google Drive access 5', + ] + expect(() => pickProvisionedGroupName('Google Drive', taken)).toThrow('Settings') + }) +}) + +describe('deriveViewerConnectorMembership', () => { + it.each([ + ['active', 'completed', 'connected'], + ['active', 'in_progress', 'connected'], + ['needs_reauth', 'completed', 'needs_reauth'], + [null, 'invited', 'invited'], + [null, 'delivery_failed', 'invited'], + [null, 'in_progress', 'invited'], + [null, 'completed', 'invited'], + ['revoked', 'completed', 'invited'], + [null, 'revoked', 'not_enrolled'], + [null, null, 'not_enrolled'], + ] as const)( + 'credential %s + enrollment %s → %s', + (managedOauthStatus, enrollmentStatus, expected) => { + expect(deriveViewerConnectorMembership({ managedOauthStatus, enrollmentStatus })).toBe( + expected + ) + } + ) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.ts new file mode 100644 index 00000000000..b20bf8ea667 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.ts @@ -0,0 +1,303 @@ +import { db } from '@sim/db' +import { credential, credentialGroupEnrollment, user } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { and, eq, inArray } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + createCredentialGroupInvitationLink, + inviteCredentialGroupEnrollment, + loadCredentialGroupInviterIdentity, +} from '@/lib/credential-groups/enrollments' +import { + getCredentialGroupProviderId, + getCredentialGroupStandardOAuthProviderFromProviderId, + isCredentialGroupProvider, +} from '@/lib/credential-groups/providers' +import { createCredentialGroup, listCredentialGroups } from '@/lib/credential-groups/service' +import { getUsersWithPermissions } from '@/lib/workspaces/permissions/utils' +import type { ConnectorMeta } from '@/connectors/types' + +const logger = createLogger('KnowledgeConnectorMemberProvisioning') + +/** Invitations one request sends before handing the rest to the member run. */ +export const MEMBER_PROVISION_INVITES_PER_REQUEST = 25 +/** Invitations one member run sends, so a large workspace is covered within a few runs. */ +export const MEMBER_PROVISION_INVITES_PER_RUN = 100 +/** Names tried for the group a connector provisions, in order. */ +const PROVISIONED_GROUP_NAME_ATTEMPTS = 5 + +export interface ProvisionedMembersBinding { + credentialGroupId: string + credentialGroupOptionId: string + /** Whether this call created the group rather than reusing one. */ + created: boolean +} + +/** + * The group name a connector provisions for its provider: the connector's + * name, suffixed until it is free of the workspace's existing group names. + */ +export function pickProvisionedGroupName( + connectorName: string, + takenNames: readonly string[] +): string { + const taken = new Set(takenNames.map((name) => name.trim().toLocaleLowerCase())) + const base = `${connectorName} access` + for (let attempt = 1; attempt <= PROVISIONED_GROUP_NAME_ATTEMPTS; attempt++) { + const candidate = attempt === 1 ? base : `${base} ${attempt}` + if (!taken.has(candidate.toLocaleLowerCase())) return candidate + } + throw new OrchestrationError( + 'conflict', + `Every name from "${base}" to "${base} ${PROVISIONED_GROUP_NAME_ATTEMPTS}" is taken; pick a Credential Group in Settings` + ) +} + +/** + * The Credential Group option a members-mode connector should crawl through + * when the caller named none: the workspace's one active option collecting + * the connector's accounts, or a group created for the purpose. Two or more + * candidate options is an ambiguity the caller has to resolve by naming one. + */ +export async function provisionKnowledgeConnectorMembersBinding(input: { + workspaceId: string + connectorMeta: Pick + userId: string +}): Promise { + const { connectorMeta } = input + if (connectorMeta.auth.mode !== 'oauth') { + throw new OrchestrationError('validation', 'Only an OAuth connector can sync per member') + } + const providerId = connectorMeta.auth.provider + let provider: ReturnType + try { + provider = getCredentialGroupStandardOAuthProviderFromProviderId(providerId) + } catch { + throw new OrchestrationError( + 'validation', + `${connectorMeta.name} accounts cannot be collected through a Credential Group yet` + ) + } + + const groups = await listCredentialGroups(input.workspaceId) + const candidates: ProvisionedMembersBinding[] = [] + for (const group of groups) { + if (group.status !== 'active') continue + for (const option of group.options) { + if (option.status !== 'active') continue + if (!isCredentialGroupProvider(option.provider)) continue + if (getCredentialGroupProviderId(option.provider) !== providerId) continue + candidates.push({ + credentialGroupId: group.id, + credentialGroupOptionId: option.id, + created: false, + }) + } + } + if (candidates.length === 1) return candidates[0] + if (candidates.length > 1) { + throw new OrchestrationError( + 'validation', + `Several Credential Groups collect ${connectorMeta.name} accounts; choose which one this connector syncs through` + ) + } + + const name = pickProvisionedGroupName( + connectorMeta.name, + groups.map((group) => group.name) + ) + const group = await createCredentialGroup(input.workspaceId, input.userId, { + name, + options: [{ provider, label: connectorMeta.name, required: true }], + }) + const option = group.options[0] + if (!option) throw new Error('Provisioned Credential Group has no option') + logger.info('Provisioned a Credential Group for a members-mode connector', { + workspaceId: input.workspaceId, + credentialGroupId: group.id, + provider, + }) + return { credentialGroupId: group.id, credentialGroupOptionId: option.id, created: true } +} + +export interface InviteWorkspaceMembersResult { + invited: number + failed: number + /** Members left uninvited because the limit was reached; the next call continues. */ + remaining: number +} + +/** + * Invites every workspace member who has no enrollment in the group yet, up + * to `limit`, so joining the workspace is all a person has to do before + * connecting their account. An enrollment an admin revoked is left alone. + * Failures are logged per person and never abort the caller. + */ +export async function inviteWorkspaceMembersToCredentialGroup(input: { + workspaceId: string + credentialGroupId: string + inviterUserId: string | undefined + limit: number +}): Promise { + const [members, enrolled, inviter] = await Promise.all([ + getUsersWithPermissions(input.workspaceId), + db + .select({ email: credentialGroupEnrollment.email }) + .from(credentialGroupEnrollment) + .where(eq(credentialGroupEnrollment.credentialGroupId, input.credentialGroupId)), + input.inviterUserId + ? loadCredentialGroupInviterIdentity(input.inviterUserId) + : Promise.resolve(null), + ]) + const enrolledEmails = new Set(enrolled.map((row) => row.email.trim().toLocaleLowerCase())) + const pending = members + .map((member) => member.email.trim().toLocaleLowerCase()) + .filter((email, index, all) => email && all.indexOf(email) === index) + .filter((email) => !enrolledEmails.has(email)) + const batch = pending.slice(0, input.limit) + const inviterName = inviter?.name ?? inviter?.email ?? undefined + + let invited = 0 + let failed = 0 + for (const email of batch) { + try { + await inviteCredentialGroupEnrollment( + input.workspaceId, + input.credentialGroupId, + input.inviterUserId, + inviterName, + email + ) + invited += 1 + } catch (error) { + failed += 1 + logger.warn('Failed to invite a workspace member to a connector credential group', { + workspaceId: input.workspaceId, + credentialGroupId: input.credentialGroupId, + error: getErrorMessage(error), + }) + } + } + return { invited, failed, remaining: pending.length - batch.length } +} + +export type ViewerConnectorMembership = 'connected' | 'needs_reauth' | 'invited' | 'not_enrolled' + +/** + * Where a viewer stands with a members-mode connector, from their enrollment + * and managed credential for the connector's option. + */ +export function deriveViewerConnectorMembership(input: { + enrollmentStatus: string | null + managedOauthStatus: string | null +}): ViewerConnectorMembership { + if (input.managedOauthStatus === 'active') return 'connected' + if (input.managedOauthStatus === 'needs_reauth') return 'needs_reauth' + if ( + input.enrollmentStatus === 'invited' || + input.enrollmentStatus === 'delivery_failed' || + input.enrollmentStatus === 'in_progress' || + input.enrollmentStatus === 'completed' + ) { + return 'invited' + } + return 'not_enrolled' +} + +/** + * The viewer's membership in each members-mode connector, keyed by connector + * id. Connectors that sync as the workspace are absent. + */ +export async function resolveViewerConnectorMemberships(input: { + userId: string + workspaceId: string + connectors: ReadonlyArray<{ + id: string + accessMode: string + credentialGroupId: string | null + credentialGroupOptionId: string | null + }> +}): Promise> { + const result = new Map() + const memberConnectors = input.connectors.filter( + (connector) => + connector.accessMode === 'members' && + connector.credentialGroupId && + connector.credentialGroupOptionId + ) + if (memberConnectors.length === 0) return result + + const [viewer] = await db + .select({ email: user.email }) + .from(user) + .where(eq(user.id, input.userId)) + .limit(1) + const email = viewer?.email.trim().toLocaleLowerCase() + const groupIds = [...new Set(memberConnectors.map((connector) => connector.credentialGroupId!))] + const rows = email + ? await db + .select({ + credentialGroupId: credentialGroupEnrollment.credentialGroupId, + enrollmentStatus: credentialGroupEnrollment.status, + credentialGroupOptionId: credential.credentialGroupOptionId, + managedOauthStatus: credential.managedOauthStatus, + }) + .from(credentialGroupEnrollment) + .leftJoin( + credential, + and( + eq(credential.credentialGroupEnrollmentId, credentialGroupEnrollment.id), + eq(credential.workspaceId, input.workspaceId), + eq(credential.type, 'managed_oauth') + ) + ) + .where( + and( + inArray(credentialGroupEnrollment.credentialGroupId, groupIds), + eq(credentialGroupEnrollment.email, email) + ) + ) + : [] + + for (const connector of memberConnectors) { + const enrollment = rows.find((row) => row.credentialGroupId === connector.credentialGroupId) + const forOption = rows.find( + (row) => + row.credentialGroupId === connector.credentialGroupId && + row.credentialGroupOptionId === connector.credentialGroupOptionId + ) + result.set( + connector.id, + deriveViewerConnectorMembership({ + enrollmentStatus: enrollment?.enrollmentStatus ?? null, + managedOauthStatus: forOption?.managedOauthStatus ?? null, + }) + ) + } + return result +} + +/** + * A fresh enrollment link for the viewer into the connector's group, created + * on demand so a workspace member never has to find the invitation email. + */ +export async function createViewerConnectorEnrollmentLink(input: { + userId: string + workspaceId: string + credentialGroupId: string +}): Promise { + const [viewer] = await db + .select({ email: user.email }) + .from(user) + .where(eq(user.id, input.userId)) + .limit(1) + if (!viewer) throw new OrchestrationError('not_found', 'User not found') + const { invitationLink } = await createCredentialGroupInvitationLink( + input.workspaceId, + input.credentialGroupId, + input.userId, + viewer.email + ) + return invitationLink +} diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 9897fc67ac4..6b7e241af4e 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -36,6 +36,10 @@ import { removeMemberObservationsForDocuments, removeUnseenMemberObservations, } from '@/lib/knowledge/connectors/member-observations' +import { + inviteWorkspaceMembersToCredentialGroup, + MEMBER_PROVISION_INVITES_PER_RUN, +} from '@/lib/knowledge/connectors/member-provisioning' import { CONNECTOR_AUTO_DISABLED_ERROR, CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES, @@ -1253,6 +1257,28 @@ export async function executeMemberSync( if (connector.accessRewritePending) await finishPendingAccessRewrite(run) const affectedDocumentIds = new Set() + /** + * Anyone who joined the workspace since the last run is invited now, so + * membership grows on its own; the invitation is the only thing they need. + */ + const invited = await inviteWorkspaceMembersToCredentialGroup({ + workspaceId: run.workspaceId, + credentialGroupId: connector.credentialGroupId, + inviterUserId: undefined, + limit: MEMBER_PROVISION_INVITES_PER_RUN, + }).catch((error) => { + logger.warn('Failed to invite new workspace members during a member run', { + connectorId, + error: getErrorMessage(error), + }) + return null + }) + if (invited && invited.invited > 0) { + logger.info('Invited new workspace members to the connector credential group', { + connectorId, + ...invited, + }) + } const membership = await reconcileMembership(run, binding) for (const documentId of membership.affectedDocumentIds) affectedDocumentIds.add(documentId) diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 70f836782c3..ad31f63d6b0 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -114,6 +114,7 @@ app: # Generate using: openssl rand -hex 32 CRON_SECRET: "" # OPTIONAL - required only if cronjobs.enabled=true, authenticates scheduled job requests TABLE_ROW_TTL: "" # Enable TTL columns and expired-row cleanup when AppConfig is unavailable + KNOWLEDGE_MEMBER_ACCESS: "" # Enable per-member knowledge connectors when AppConfig is unavailable # Optional: API Key Encryption (RECOMMENDED for production) # Generate with: openssl rand -hex 32 (produces the required 64-hex-char / 32-byte value). diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index 295695c36c5..1c6b6446ab4 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -6,66 +6,66 @@ }, "entries": { "app/api/v2/blocks/[blockId]/route.ts": { - "modules": 1609, + "modules": 1614, "gateways": { "apps/sim/triggers/index.ts": 485, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 448, - "apps/sim/lib/api/server/routes/index.ts": 393, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 349, - "apps/sim/lib/auth/index.ts": 336, + "apps/sim/lib/api/server/routes/index.ts": 398, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 354, + "apps/sim/lib/auth/index.ts": 341, "apps/sim/lib/webhooks/providers/index.ts": 110, "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/blocks/route.ts": { - "modules": 1608, + "modules": 1613, "gateways": { "apps/sim/triggers/index.ts": 485, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 448, - "apps/sim/lib/api/server/routes/index.ts": 386, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 351, - "apps/sim/lib/auth/index.ts": 338, + "apps/sim/lib/api/server/routes/index.ts": 391, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 356, + "apps/sim/lib/auth/index.ts": 343, "apps/sim/lib/webhooks/providers/index.ts": 110, "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/connector-types/route.ts": { - "modules": 1672, + "modules": 1677, "gateways": { "apps/sim/triggers/index.ts": 485, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 448, - "apps/sim/lib/api/server/routes/index.ts": 395, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 351, - "apps/sim/lib/auth/index.ts": 338, + "apps/sim/lib/api/server/routes/index.ts": 400, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 356, + "apps/sim/lib/auth/index.ts": 343, "apps/sim/lib/webhooks/providers/index.ts": 110, "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/tools/[toolId]/route.ts": { - "modules": 1606, + "modules": 1611, "gateways": { "apps/sim/triggers/index.ts": 485, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 448, - "apps/sim/lib/api/server/routes/index.ts": 393, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 349, - "apps/sim/lib/auth/index.ts": 336, + "apps/sim/lib/api/server/routes/index.ts": 398, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 354, + "apps/sim/lib/auth/index.ts": 341, "apps/sim/lib/webhooks/providers/index.ts": 110, "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/tools/route.ts": { - "modules": 1607, + "modules": 1612, "gateways": { "apps/sim/triggers/index.ts": 485, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 448, - "apps/sim/lib/api/server/routes/index.ts": 384, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 349, - "apps/sim/lib/auth/index.ts": 336, + "apps/sim/lib/api/server/routes/index.ts": 389, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 354, + "apps/sim/lib/auth/index.ts": 341, "apps/sim/lib/webhooks/providers/index.ts": 110, "apps/sim/lib/webhooks/providers/registry.ts": 108 } @@ -83,14 +83,14 @@ "gateways": {} }, "app/workspace/[workspaceId]/chat/[chatId]/page.tsx": { - "modules": 2872, + "modules": 2879, "gateways": { - "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1301, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 923, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 774, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 771, + "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1303, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 925, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 776, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 773, "apps/sim/triggers/registry.ts": 483, - "apps/sim/blocks/registry.ts": 325, + "apps/sim/blocks/registry.ts": 321, "apps/sim/lib/auth/index.ts": 248, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 199 } @@ -112,7 +112,7 @@ } }, "app/workspace/[workspaceId]/files/[fileId]/page.tsx": { - "modules": 1945, + "modules": 1950, "gateways": { "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 350, @@ -148,7 +148,7 @@ } }, "app/workspace/[workspaceId]/files/page.tsx": { - "modules": 1945, + "modules": 1950, "gateways": { "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 350, @@ -173,14 +173,14 @@ "gateways": {} }, "app/workspace/[workspaceId]/home/page.tsx": { - "modules": 2872, + "modules": 2879, "gateways": { - "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1301, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 923, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 774, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 771, + "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1303, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 925, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 776, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 773, "apps/sim/triggers/registry.ts": 483, - "apps/sim/blocks/registry.ts": 325, + "apps/sim/blocks/registry.ts": 321, "apps/sim/lib/auth/index.ts": 248, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 199 } @@ -241,15 +241,15 @@ } }, "app/workspace/[workspaceId]/knowledge/[id]/[documentId]/page.tsx": { - "modules": 1367, + "modules": 1370, "gateways": { - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx": 1218, + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx": 1221, "apps/sim/triggers/registry.ts": 519, - "apps/sim/blocks/registry.ts": 358, - "apps/sim/blocks/registry-maps.ts": 355, + "apps/sim/blocks/registry.ts": 354, + "apps/sim/blocks/registry-maps.ts": 351, "apps/sim/connectors/registry.ts": 66, "apps/sim/app/workspace/[workspaceId]/components/index.ts": 59, - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 50, + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 53, "apps/sim/lib/api/contracts/index.ts": 40 } }, @@ -270,15 +270,15 @@ } }, "app/workspace/[workspaceId]/knowledge/[id]/page.tsx": { - "modules": 1370, + "modules": 1373, "gateways": { - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx": 1220, + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx": 1223, "apps/sim/triggers/registry.ts": 519, - "apps/sim/blocks/registry.ts": 366, - "apps/sim/blocks/registry-maps.ts": 363, + "apps/sim/blocks/registry.ts": 362, + "apps/sim/blocks/registry-maps.ts": 359, "apps/sim/connectors/registry.ts": 66, "apps/sim/app/workspace/[workspaceId]/components/index.ts": 59, - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 43, + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 46, "apps/sim/lib/api/contracts/index.ts": 40 } }, @@ -299,25 +299,25 @@ } }, "app/workspace/[workspaceId]/knowledge/page.tsx": { - "modules": 2158, + "modules": 2205, "gateways": { "apps/sim/triggers/registry.ts": 483, - "apps/sim/blocks/registry.ts": 350, - "apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts": 305, - "apps/sim/lib/knowledge/application/knowledge-bases.ts": 249, - "apps/sim/lib/auth/index.ts": 204, - "apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx": 161, - "apps/sim/lib/knowledge/orchestration/index.ts": 148, - "apps/sim/lib/knowledge/orchestration/connectors.ts": 144 + "apps/sim/blocks/registry.ts": 346, + "apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts": 344, + "apps/sim/lib/knowledge/application/knowledge-bases.ts": 288, + "apps/sim/lib/auth/index.ts": 202, + "apps/sim/lib/knowledge/orchestration/index.ts": 187, + "apps/sim/lib/knowledge/orchestration/connectors.ts": 183, + "apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx": 164 } }, "app/workspace/[workspaceId]/layout.tsx": { - "modules": 2034, + "modules": 2039, "gateways": { "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 349, + "apps/sim/lib/auth/index.ts": 344, "apps/sim/app/workspace/[workspaceId]/components/workspace-chrome/index.ts": 340, - "apps/sim/lib/auth/index.ts": 340, "apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx": 335, "apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/index.ts": 229, "apps/sim/lib/webhooks/providers/index.ts": 110, @@ -341,11 +341,11 @@ } }, "app/workspace/[workspaceId]/logs/page.tsx": { - "modules": 1663, + "modules": 1662, "gateways": { - "apps/sim/app/workspace/[workspaceId]/logs/logs.tsx": 1516, + "apps/sim/app/workspace/[workspaceId]/logs/logs.tsx": 1515, "apps/sim/triggers/registry.ts": 519, - "apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/execution-snapshot/execution-snapshot.tsx": 410, + "apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/execution-snapshot/execution-snapshot.tsx": 409, "apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/index.ts": 361, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 355, "apps/sim/blocks/registry.ts": 345, @@ -382,12 +382,12 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/[section]/page.tsx": { - "modules": 2199, + "modules": 2206, "gateways": { "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 576, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 345, - "apps/sim/lib/auth/index.ts": 309, + "apps/sim/lib/auth/index.ts": 314, "apps/sim/lib/webhooks/providers/index.ts": 110, "apps/sim/lib/webhooks/providers/registry.ts": 108, "apps/sim/ee/access-control/components/access-control.tsx": 74, @@ -403,9 +403,9 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/billing/credit-usage/page.tsx": { - "modules": 1563, + "modules": 1568, "gateways": { - "apps/sim/lib/auth/index.ts": 1426, + "apps/sim/lib/auth/index.ts": 1430, "apps/sim/triggers/index.ts": 485, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 445, @@ -466,9 +466,9 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/usage/events/page.tsx": { - "modules": 1572, + "modules": 1577, "gateways": { - "apps/sim/lib/auth/index.ts": 1428, + "apps/sim/lib/auth/index.ts": 1432, "apps/sim/triggers/index.ts": 485, "apps/sim/triggers/registry.ts": 483, "apps/sim/blocks/registry.ts": 447, @@ -542,11 +542,11 @@ } }, "app/workspace/[workspaceId]/tables/[tableId]/page.tsx": { - "modules": 1768, + "modules": 1767, "gateways": { - "apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx": 1621, + "apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx": 1620, "apps/sim/triggers/registry.ts": 519, - "apps/sim/app/workspace/[workspaceId]/w/components/preview/index.ts": 342, + "apps/sim/app/workspace/[workspaceId]/w/components/preview/index.ts": 341, "apps/sim/blocks/registry.ts": 326, "apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/index.ts": 295, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 291, @@ -571,10 +571,10 @@ } }, "app/workspace/[workspaceId]/tables/page.tsx": { - "modules": 1778, + "modules": 1783, "gateways": { "apps/sim/triggers/registry.ts": 483, - "apps/sim/lib/auth/index.ts": 369, + "apps/sim/lib/auth/index.ts": 373, "apps/sim/blocks/registry.ts": 350, "apps/sim/app/workspace/[workspaceId]/tables/tables.tsx": 125, "apps/sim/lib/webhooks/providers/index.ts": 110, @@ -604,9 +604,9 @@ } }, "app/workspace/[workspaceId]/w/[workflowId]/page.tsx": { - "modules": 2018, + "modules": 2017, "gateways": { - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 2017, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 2016, "apps/sim/triggers/registry.ts": 519, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 351, "apps/sim/blocks/registry.ts": 345, @@ -617,13 +617,13 @@ } }, "app/workspace/[workspaceId]/w/page.tsx": { - "modules": 2003, + "modules": 2002, "gateways": { - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 848, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 554, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 847, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 553, "apps/sim/triggers/registry.ts": 519, "apps/sim/blocks/registry.ts": 345, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 317, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 316, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 157, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 150, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts": 144 From 64c3cc73133e227e5ffe2318b25af9a9a33544c9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 21:05:34 -0700 Subject: [PATCH 18/76] fix(knowledge): make per-member access self-serve end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Choosing per-member access provisions a group named after the connector and the first member run invites everyone in the workspace, under the run's lease; the request itself sends nothing, so nobody gets two links. A member connecting queues a member run for every connector on that option, and the knowledge base page shows what each viewer must do — connect, reconnect, verify their email, or nothing — opening enrollment in a new tab and refreshing until they are connected. Every gate reads one availability check (flag and Credential Groups). When it is off, readers get no member token, the engine waits instead of suspending anyone, and the field cannot re-choose per-member access. Member tokens go only to current workspace members with a live group and option. A connector that just entered members mode never tombstones its documents before a member has listed, the stale sweep leaves paused and manual connectors alone, a disabled member sync is re-enabled by re-applying its binding, leaving members mode forces a full content sync, and by-id bulk operations honour the caller's scope. Deleting a per-member connector always takes its documents. --- .../connectors/[connectorId]/enroll/route.ts | 3 +- .../add-connector-modal.tsx | 15 +- .../connector-access-field.tsx | 131 +++------ .../connectors-section/connectors-section.tsx | 44 ++- .../edit-connector-modal.tsx | 86 +++++- .../knowledge/[id]/components/index.ts | 2 +- .../member-connect-banner.tsx | 102 +++++-- .../use-connector-member-group-options.ts | 106 +++++++ .../lib/api/contracts/knowledge/connectors.ts | 9 +- apps/sim/lib/credential-groups/oauth.ts | 22 ++ apps/sim/lib/knowledge/access/availability.ts | 22 +- apps/sim/lib/knowledge/access/scope.test.ts | 30 +- apps/sim/lib/knowledge/access/scope.ts | 26 +- .../knowledge/application/connector-access.ts | 32 +- .../lib/knowledge/application/connectors.ts | 78 +++-- .../lib/knowledge/application/documents.ts | 1 + .../connectors/member-observations.ts | 40 ++- .../connectors/member-provisioning.test.ts | 60 ++-- .../connectors/member-provisioning.ts | 274 +++++++++++------- .../connectors/member-sync-engine.ts | 76 +++-- apps/sim/lib/knowledge/documents/service.ts | 4 +- .../orchestration/connector-access.test.ts | 63 +++- .../orchestration/connector-access.ts | 67 ++++- 23 files changed, 872 insertions(+), 421 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts index 8dee8d7d8bb..23a6c8f6f77 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts @@ -12,7 +12,8 @@ export const POST = defineInternalJsonRoute({ auth: internalKnowledgeSessionOrExecutorAuth, operation: knowledgeOperations.enrollConnectorMember, rateLimit: internalRateLimits.none({ - reason: 'A member connecting their own account by hand; the link is single-use', + reason: + 'A member connecting their own account by hand; each call only re-issues their own invitation', }), errorPolicy: internalKnowledgeErrorPolicies.connectors, mapInput: ({ params }) => ({ diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index bd7d662f591..9a28f548567 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -39,6 +39,7 @@ import { hasWorkspaceMaxConnectorAccess } from '@/app/workspace/[workspaceId]/kn import { SYNC_INTERVALS } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' +import { useConnectorMemberGroupOptions } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { getBlock } from '@/blocks' @@ -95,6 +96,14 @@ export function AddConnectorModal({ const connectorConfig = selectedType ? CONNECTOR_META_REGISTRY[selectedType] : null const isApiKeyMode = connectorConfig?.auth.mode === 'apiKey' const isMembersMode = access.accessMode === 'members' + const groupOptions = useConnectorMemberGroupOptions({ + workspaceId, + connectorConfig, + enabled: canAdmin && memberAccessAvailable, + }) + /** Several groups collect this provider's accounts: the admin has to say which. */ + const membersChoiceOpen = + isMembersMode && groupOptions.needsChoice && !access.credentialGroupOptionId /** Fields a per-member crawl refuses: a cap would hide part of a member's corpus. */ const memberCapFieldIds = useMemo( () => @@ -184,7 +193,9 @@ export function AddConnectorModal({ if (!connectorConfig) return false if (isApiKeyMode) { if (!isApiKeyOptional && !apiKeyValue.trim()) return false - } else if (!isMembersMode) { + } else if (isMembersMode) { + if (membersChoiceOpen) return false + } else { if (!effectiveCredentialId) return false } @@ -199,6 +210,7 @@ export function AddConnectorModal({ connectorConfig, isApiKeyMode, isMembersMode, + membersChoiceOpen, memberCapFieldIds, isApiKeyOptional, apiKeyValue, @@ -337,6 +349,7 @@ export function AddConnectorModal({ connectorConfig={connectorConfig} value={access} onChange={setAccess} + groupOptions={groupOptions} canAdmin={canAdmin} disabled={isCreating} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx index 3f9d8a51ee1..9bc2d8089ad 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx @@ -1,63 +1,30 @@ 'use client' -import { type ReactNode, useMemo } from 'react' -import { - ButtonGroup, - ButtonGroupItem, - ChipCombobox, - ChipModalField, - type ComboboxOption, -} from '@sim/emcn' +import type { ReactNode } from 'react' +import { ButtonGroup, ButtonGroupItem, ChipCombobox, ChipModalField } from '@sim/emcn' import Link from 'next/link' import { - type CredentialGroupStandardOAuthProvider, - getCredentialGroupProviderId, - getCredentialGroupStandardOAuthProviderFromProviderId, - isCredentialGroupProvider, -} from '@/lib/credential-groups/providers' + type ConnectorMemberGroupOptions, + decodeConnectorMemberGroupOption, + encodeConnectorMemberGroupOption, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import type { ConnectorMeta } from '@/connectors/types' -import { useCredentialGroups } from '@/hooks/queries/credential-groups' -/** What the caller chose; `members` needs the option the connector crawls with. */ +/** What the caller chose; `members` may name the option the connector crawls with. */ export interface ConnectorAccessSelection { accessMode: 'workspace' | 'members' credentialGroupId?: string credentialGroupOptionId?: string } -/** Encodes a group and option pair as one combobox value. */ -function optionValue(credentialGroupId: string, credentialGroupOptionId: string): string { - return `${credentialGroupId}:${credentialGroupOptionId}` -} - -function parseOptionValue(value: string): ConnectorAccessSelection | null { - const separator = value.indexOf(':') - if (separator <= 0) return null - return { - accessMode: 'members', - credentialGroupId: value.slice(0, separator), - credentialGroupOptionId: value.slice(separator + 1), - } -} - -/** The credential-group provider that collects accounts for this connector, if any. */ -function credentialGroupProviderFor( - connectorConfig: ConnectorMeta -): CredentialGroupStandardOAuthProvider | null { - if (connectorConfig.auth.mode !== 'oauth' || !connectorConfig.permissionScopedListing) return null - try { - return getCredentialGroupStandardOAuthProviderFromProviderId(connectorConfig.auth.provider) - } catch { - return null - } -} - interface ConnectorAccessFieldProps { workspaceId: string connectorConfig: ConnectorMeta value: ConnectorAccessSelection onChange: (value: ConnectorAccessSelection) => void + /** From `useConnectorMemberGroupOptions`; shared with the modal so both agree on what is required. */ + groupOptions: ConnectorMemberGroupOptions /** Only an admin may put a connector into members mode. */ canAdmin: boolean disabled?: boolean @@ -69,59 +36,27 @@ interface ConnectorAccessFieldProps { /** * The Access section of a connector's settings: sync as the workspace, or - * crawl once per Credential Group member so each person sees only what the - * source lets them read. Rendered only for connectors whose listing reflects - * who may read each document. An admin with no group collecting the - * connector's accounts can create one here; members are then invited from - * Settings. + * crawl once per member so each person sees only what the source lets them + * read. Per-member access needs nothing from the admin: a Credential Group is + * found or created for the connector's provider, everyone in the workspace is + * invited, and each person connects their own account. Only a workspace with + * several matching groups is asked which one to use. */ export function ConnectorAccessField({ workspaceId, connectorConfig, value, onChange, + groupOptions, canAdmin, disabled = false, allowMembers = true, footer, }: ConnectorAccessFieldProps) { const { features } = useWorkspaceHostContext() - const provider = credentialGroupProviderFor(connectorConfig) - const providerId = provider ? getCredentialGroupProviderId(provider) : null const credentialGroupsAvailable = features?.credentialGroups === true - const { - data: settings, - isLoading, - error: loadError, - } = useCredentialGroups( - canAdmin && provider && credentialGroupsAvailable ? workspaceId : undefined - ) - - const options = useMemo(() => { - if (!settings || !providerId) return [] - const entries: ComboboxOption[] = [] - for (const group of settings.credentialGroups) { - if (group.status !== 'active') continue - for (const option of group.options) { - if (option.status !== 'active') continue - if (!isCredentialGroupProvider(option.provider)) continue - if (getCredentialGroupProviderId(option.provider) !== providerId) continue - entries.push({ - label: `${group.name} · ${option.label}`, - value: optionValue(group.id, option.id), - }) - } - } - return entries - }, [settings, providerId]) - - if (!provider) return null - - const selectedValue = - value.accessMode === 'members' && value.credentialGroupId && value.credentialGroupOptionId - ? optionValue(value.credentialGroupId, value.credentialGroupOptionId) - : undefined + if (!groupOptions.supported) return null if (!canAdmin) { if (value.accessMode !== 'members') return null @@ -139,21 +74,26 @@ export function ConnectorAccessField({ ) } - const settingsHref = `/workspace/${workspaceId}/settings/credential-groups` - /** One existing group is reused on its own; several need the admin to say which. */ - const needsChoice = options.length > 1 + const selectedValue = + value.accessMode === 'members' && value.credentialGroupId && value.credentialGroupOptionId + ? encodeConnectorMemberGroupOption(value.credentialGroupId, value.credentialGroupOptionId) + : undefined + const { options, needsChoice, isLoading, error } = groupOptions + const membersHint = !credentialGroupsAvailable + ? 'Per-member access needs Credential Groups, which are not available on this plan.' + : !allowMembers + ? 'Per-member access is turned off for this workspace.' + : undefined return (
@@ -181,24 +121,25 @@ export function ConnectorAccessField({ options={options} value={selectedValue} onChange={(next) => { - const parsed = parseOptionValue(next) - if (parsed) onChange(parsed) + const decoded = decodeConnectorMemberGroupOption(next) + if (decoded) onChange({ accessMode: 'members', ...decoded }) }} placeholder='Choose which credential group members connect through' isLoading={isLoading} - disabled={disabled || Boolean(loadError)} + disabled={disabled || Boolean(error)} /> )}

{options.length === 1 ? `Members connect through ${options[0].label}. ` : options.length === 0 - ? `A credential group is created for ${connectorConfig.name}. ` + ? `A credential group named ${connectorConfig.name} is created. ` : ''} - Everyone in the workspace is invited to connect their own {connectorConfig.name}{' '} - account, and people who join later are invited automatically. Manage members in{' '} + Everyone in the workspace is invited by email to connect their own{' '} + {connectorConfig.name} account as the first sync starts, and people who join later are + invited automatically. Manage members in{' '} Settings diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index a4ddc1bfe72..e7f7f554282 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -168,10 +168,18 @@ export function ConnectorsSection({ ) } + const deletingMembersConnector = + connectors.find((connector) => connector.id === deleteTarget)?.accessMode === 'members' + const handleDeleteConnector = () => { if (!deleteTarget) return deleteConnector( - { knowledgeBaseId, connectorId: deleteTarget, deleteDocuments }, + { + knowledgeBaseId, + connectorId: deleteTarget, + /** Documents synced per member have no meaning without their members. */ + deleteDocuments: deleteDocuments || deletingMembersConnector, + }, { onSuccess: () => { setError(null) @@ -240,7 +248,11 @@ export function ConnectorsSection({ }} srTitle='Remove Connector' title='Remove Connector' - text='This will disconnect the source and stop future syncs. Documents already synced will remain in the knowledge base unless you choose to delete them.' + text={ + deletingMembersConnector + ? 'This will disconnect the source, stop future syncs, and delete the documents it synced per member.' + : 'This will disconnect the source and stop future syncs. Documents already synced will remain in the knowledge base unless you choose to delete them.' + } confirm={{ label: 'Remove', onClick: handleDeleteConnector, @@ -248,19 +260,21 @@ export function ConnectorsSection({ pendingLabel: 'Removing...', }} > -

- setDeleteDocuments(checked === true)} - /> - -
+ {!deletingMembersConnector && ( +
+ setDeleteDocuments(checked === true)} + /> + +
+ )}
) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index bc883470122..2a8355abed0 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -34,6 +34,7 @@ import type { ConfigFieldValue, } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' +import { useConnectorMemberGroupOptions } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { withBrandIcon } from '@/blocks/brand-icon' @@ -243,14 +244,22 @@ export function EditConnectorModal({ const persistedAccess = useMemo(() => currentAccess(connector), [connector]) const accessDirty = accessChanged(persistedAccess, access) + const groupOptions = useConnectorMemberGroupOptions({ + workspaceId, + connectorConfig, + enabled: canAdmin && memberAccessAvailable, + }) /** Leaving members mode needs the credential the connector syncs as from then on. */ const needsWorkspaceCredential = accessDirty && access.accessMode === 'workspace' && connector.accessMode === 'members' const accessComplete = !accessDirty || - access.accessMode === 'members' || - !needsWorkspaceCredential || - Boolean(workspaceCredentialId) + (access.accessMode === 'members' + ? !groupOptions.needsChoice || Boolean(access.credentialGroupOptionId) + : !needsWorkspaceCredential || Boolean(workspaceCredentialId)) + /** A disabled member sync is re-enabled by applying the current binding again. */ + const canReenableMemberSync = + !accessDirty && connector.accessMode === 'members' && connector.memberSyncStatus === 'disabled' const memberCapFieldIds = useMemo( () => new Set( @@ -407,6 +416,8 @@ export function EditConnectorModal({ canAdmin={canAdmin} showAccessField={showAccessField} allowMembers={memberAccessAvailable} + groupOptions={groupOptions} + canReenableMemberSync={canReenableMemberSync} accessDirty={accessDirty} accessComplete={accessComplete} isSwitchingAccess={isSwitchingAccess} @@ -428,7 +439,8 @@ export function EditConnectorModal({ primaryAction={{ label: isSaving ? 'Saving…' : 'Save', onClick: handleSave, - disabled: !hasChanges || isSaving, + /** An open access change is applied by its own control, never folded into Save. */ + disabled: !hasChanges || accessDirty || isSaving, }} /> )} @@ -455,6 +467,8 @@ interface SettingsTabProps { canAdmin: boolean showAccessField: boolean allowMembers: boolean + groupOptions: ReturnType + canReenableMemberSync: boolean accessDirty: boolean accessComplete: boolean isSwitchingAccess: boolean @@ -485,6 +499,8 @@ function SettingsTab({ canAdmin, showAccessField, allowMembers, + groupOptions, + canReenableMemberSync, accessDirty, accessComplete, isSwitchingAccess, @@ -499,10 +515,14 @@ function SettingsTab({ connectorConfig?.auth.mode === 'oauth' ? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider) : null + const syncsPerMember = access.accessMode === 'members' const { data: rawCredentials = [], isLoading: credentialsLoading } = useOAuthCredentials( providerId ?? undefined, - { enabled: needsWorkspaceCredential && Boolean(providerId), workspaceId } + { enabled: (needsWorkspaceCredential || syncsPerMember) && Boolean(providerId), workspaceId } ) + const [browseCredentialId, setBrowseCredentialId] = useState(null) + /** A per-member connector has no credential of its own; the admin's account browses the source. */ + const selectorCredentialId = syncsPerMember ? browseCredentialId : credentialId const credentialOptions = useMemo( () => rawCredentials @@ -524,19 +544,38 @@ function SettingsTab({ onChange={onAccessChange} canAdmin={canAdmin} allowMembers={allowMembers} + groupOptions={groupOptions} disabled={isSaving} footer={ - accessDirty ? ( + canReenableMemberSync ? ( +
+
+ +
+

+ Members and their documents are kept; the next sync restores their access. +

+
+ ) : accessDirty ? (
{needsWorkspaceCredential && ( - + <> + + {!credentialsLoading && credentialOptions.length === 0 && ( +

+ Connect a {connectorConfig.name} account in Integrations first. +

+ )} + )}
+ {canConnect && ( + + )}
) })} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts new file mode 100644 index 00000000000..d25d4387a9c --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts @@ -0,0 +1,106 @@ +'use client' + +import { useMemo } from 'react' +import type { ComboboxOption } from '@sim/emcn' +import { + type CredentialGroupStandardOAuthProvider, + getCredentialGroupProviderId, + getCredentialGroupStandardOAuthProviderFromProviderId, + isCredentialGroupProvider, +} from '@/lib/credential-groups/providers' +import type { ConnectorMeta } from '@/connectors/types' +import { useCredentialGroups } from '@/hooks/queries/credential-groups' + +/** Encodes a group and option pair as one combobox value. */ +export function encodeConnectorMemberGroupOption( + credentialGroupId: string, + credentialGroupOptionId: string +): string { + return `${credentialGroupId}:${credentialGroupOptionId}` +} + +export function decodeConnectorMemberGroupOption( + value: string +): { credentialGroupId: string; credentialGroupOptionId: string } | null { + const separator = value.indexOf(':') + if (separator <= 0) return null + return { + credentialGroupId: value.slice(0, separator), + credentialGroupOptionId: value.slice(separator + 1), + } +} + +/** The credential-group provider that collects accounts for this connector, if any. */ +export function connectorMemberGroupProvider( + connectorConfig: ConnectorMeta +): CredentialGroupStandardOAuthProvider | null { + if (connectorConfig.auth.mode !== 'oauth' || !connectorConfig.permissionScopedListing) return null + try { + return getCredentialGroupStandardOAuthProviderFromProviderId(connectorConfig.auth.provider) + } catch { + return null + } +} + +interface UseConnectorMemberGroupOptionsInput { + workspaceId: string + connectorConfig: ConnectorMeta | null + /** False leaves the query off and reports no options, for a viewer who cannot choose anyway. */ + enabled: boolean +} + +export interface ConnectorMemberGroupOptions { + /** Every active option in the workspace collecting the connector's accounts, as combobox entries. */ + options: ComboboxOption[] + /** Whether the connector's provider can be collected through a Credential Group at all. */ + supported: boolean + /** More than one candidate: the admin has to say which, or the server refuses the ambiguity. */ + needsChoice: boolean + isLoading: boolean + error: Error | null +} + +/** + * The Credential Group options a per-member connector could sync through. + * One source for the Access field, which renders them, and the modals, which + * must not submit while a choice between several is still open. + */ +export function useConnectorMemberGroupOptions({ + workspaceId, + connectorConfig, + enabled, +}: UseConnectorMemberGroupOptionsInput): ConnectorMemberGroupOptions { + const provider = connectorConfig ? connectorMemberGroupProvider(connectorConfig) : null + const providerId = provider ? getCredentialGroupProviderId(provider) : null + const { + data: settings, + isLoading, + error, + } = useCredentialGroups(enabled && provider ? workspaceId : undefined) + + const options = useMemo(() => { + if (!settings || !providerId) return [] + const entries: ComboboxOption[] = [] + for (const group of settings.credentialGroups) { + if (group.status !== 'active') continue + for (const option of group.options) { + if (option.status !== 'active') continue + if (!isCredentialGroupProvider(option.provider)) continue + if (getCredentialGroupProviderId(option.provider) !== providerId) continue + entries.push({ + label: `${group.name} · ${option.label}`, + value: encodeConnectorMemberGroupOption(group.id, option.id), + }) + } + } + return entries + }, [settings, providerId]) + + return { + options, + supported: provider !== null, + needsChoice: options.length > 1, + isLoading, + error: error ?? null, + } +} diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index cd135a59202..9ce7814eebb 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -137,6 +137,8 @@ export const VIEWER_CONNECTOR_MEMBERSHIPS = [ 'needs_reauth', 'invited', 'not_enrolled', + 'revoked', + 'unverified_email', ] as const export const viewerConnectorMembershipSchema = z.enum(VIEWER_CONNECTOR_MEMBERSHIPS) export type ViewerConnectorMembership = z.output @@ -159,10 +161,11 @@ export const connectorDataSchema = z consecutiveFailures: z.number(), accessMode: connectorAccessModeSchema, /** - * Where the viewer stands with a per-member connector: absent for a - * workspace-mode connector or a caller with no person behind it. + * Where the viewer stands with a per-member connector; null for a + * workspace-mode connector, a caller with no person behind it, or where + * per-member access is not available. */ - viewerMembership: viewerConnectorMembershipSchema.nullable().optional(), + viewerMembership: viewerConnectorMembershipSchema.nullable(), credentialGroupId: z.string().nullable(), credentialGroupOptionId: z.string().nullable(), /** Members mode only; `idle` otherwise. */ diff --git a/apps/sim/lib/credential-groups/oauth.ts b/apps/sim/lib/credential-groups/oauth.ts index 0aea1addc2a..665ea90ce95 100644 --- a/apps/sim/lib/credential-groups/oauth.ts +++ b/apps/sim/lib/credential-groups/oauth.ts @@ -1,5 +1,7 @@ import { db } from '@sim/db' import { credential, credentialGroup, credentialGroupEnrollment } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, eq, ne, sql } from 'drizzle-orm' import { @@ -84,6 +86,8 @@ async function assertCurrentPolicy( } /** Builds a provider authorization URL after persisting a provider-bound one-time attempt. */ +const logger = createLogger('CredentialGroupOAuth') + export async function startCredentialGroupOAuth( context: CredentialGroupOAuthContext, invitationToken: string @@ -269,6 +273,24 @@ async function persistGrant( throw new CredentialGroupInvitationUnavailableError() } }) + + /** + * Knowledge connectors crawling through this option pick the member up on + * their next run; queue one now so their documents arrive within minutes. + * Loaded lazily: credential groups do not otherwise depend on knowledge. + */ + const { dispatchMemberSyncsForCredentialOption } = await import( + '@/lib/knowledge/connectors/member-provisioning' + ) + await dispatchMemberSyncsForCredentialOption({ + workspaceId: context.workspaceId, + credentialGroupOptionId: context.option.id, + }).catch((error) => { + logger.warn('Failed to queue member syncs after an account connected', { + credentialGroupOptionId: context.option.id, + error: getErrorMessage(error), + }) + }) } /** Exchanges a single-use code through its provider adapter and persists a normalized grant. */ diff --git a/apps/sim/lib/knowledge/access/availability.ts b/apps/sim/lib/knowledge/access/availability.ts index 9af1410a7e1..f3d3f228823 100644 --- a/apps/sim/lib/knowledge/access/availability.ts +++ b/apps/sim/lib/knowledge/access/availability.ts @@ -1,4 +1,6 @@ +import { getWorkspaceOwnerSubscriptionAccess } from '@/lib/billing/core/workspace-access' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' +import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' /** * Who is asking. Members mode — creating, switching, syncing, and honouring @@ -14,16 +16,20 @@ export interface KnowledgeMemberAccessContext { } /** - * Whether permission-aware knowledge is on for this workspace: members-mode - * connectors, their per-member change feeds, and hybrid-by-default retrieval - * with the source-recency boost. Everything the feature adds checks this one - * gate. Turning the flag off hides every member-scoped document on the next - * read, disables each members-mode connector on its next run (members are - * suspended, nothing is deleted), and returns search to the semantic-only - * default; a disabled connector is re-enabled by switching its access again. + * Whether permission-aware knowledge is on for this workspace: the + * `knowledge-member-access` flag, and Credential Groups available to the + * workspace, which members mode enrolls people through. Every gate the + * feature has checks this one function — creating and switching connectors, + * the member engine, the member tokens a reader is granted, and the + * workspace host context the UI reads — so they can never disagree. When it + * turns off, member-scoped documents are hidden on the next read, members-mode + * connectors wait rather than change anything, and search returns to the + * semantic-only default; nothing is deleted. */ export async function isKnowledgeMemberAccessAvailable( context: KnowledgeMemberAccessContext ): Promise { - return isFeatureEnabled('knowledge-member-access', context) + if (!(await isFeatureEnabled('knowledge-member-access', context))) return false + const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(context.workspaceId) + return isCredentialGroupsAvailable({ workspaceId: context.workspaceId, ownerBilling }) } diff --git a/apps/sim/lib/knowledge/access/scope.test.ts b/apps/sim/lib/knowledge/access/scope.test.ts index 23a7a28f45d..759817878f3 100644 --- a/apps/sim/lib/knowledge/access/scope.test.ts +++ b/apps/sim/lib/knowledge/access/scope.test.ts @@ -5,13 +5,17 @@ import type { Principal } from '@sim/auth/principal' import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockMemberAccessAvailable } = vi.hoisted(() => ({ +const { mockMemberAccessAvailable, mockCheckWorkspaceAccess } = vi.hoisted(() => ({ mockMemberAccessAvailable: vi.fn(async () => true), + mockCheckWorkspaceAccess: vi.fn(async () => ({ hasAccess: true })), })) vi.mock('@/lib/knowledge/access/availability', () => ({ isKnowledgeMemberAccessAvailable: mockMemberAccessAvailable, })) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + checkWorkspaceAccess: mockCheckWorkspaceAccess, +})) import { createKnowledgeAccessProvider, @@ -46,7 +50,29 @@ describe('resolveKnowledgeAccessScope', () => { userId: 'user-1', tokens: ['pub', 's:confluence:-:557058:abc', 's:google-drive:acme.com:42', 'ws'], }) - expect(dbChainMockFns.leftJoin).toHaveBeenCalledTimes(2) + expect(dbChainMockFns.leftJoin).toHaveBeenCalledTimes(3) + }) + + it('grants no member token to someone who is no longer in the workspace', async () => { + mockCheckWorkspaceAccess.mockResolvedValueOnce({ hasAccess: false }) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 'ws'], + }) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('grants no member token where per-member access is off, whatever the person holds', async () => { + mockMemberAccessAvailable.mockResolvedValueOnce(false) + queueSubjects([ + { providerId: 'google-drive', providerTenantId: 'acme.com', providerSubjectId: '42' }, + ]) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 'ws'], + }) }) it('falls back to the workspace pair for a person with no credential, and for one who is unverified or unknown', async () => { diff --git a/apps/sim/lib/knowledge/access/scope.ts b/apps/sim/lib/knowledge/access/scope.ts index 477290440e3..7298e676754 100644 --- a/apps/sim/lib/knowledge/access/scope.ts +++ b/apps/sim/lib/knowledge/access/scope.ts @@ -1,6 +1,6 @@ import { type Principal, resolvePrincipalSubject } from '@sim/auth/principal' import { db } from '@sim/db' -import { credential, credentialGroupEnrollment, user } from '@sim/db/schema' +import { credential, credentialGroup, credentialGroupEnrollment, user } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { and, eq, inArray, sql } from 'drizzle-orm' @@ -13,6 +13,7 @@ import { WORKSPACE_ACCESS_TOKENS, type WorkspaceAccessScope, } from '@/lib/knowledge/access/types' +import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' const logger = createLogger('KnowledgeAccessScope') @@ -43,6 +44,14 @@ async function loadUserAccessTokens( ): Promise { if (!workspaceId) return [...WORKSPACE_ACCESS_TOKENS] + /** + * Member tokens belong to current workspace members. Resolved before any + * document is looked up, so someone who left the workspace but still holds + * a managed credential cannot learn which documents their old tokens match. + */ + const workspaceAccess = await checkWorkspaceAccess(workspaceId, userId) + if (!workspaceAccess.hasAccess) return [...WORKSPACE_ACCESS_TOKENS] + const rows = await db .select({ providerId: credential.providerId, @@ -60,13 +69,26 @@ async function loadUserAccessTokens( inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]) ) ) + .leftJoin( + credentialGroup, + and( + eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId), + eq(credentialGroup.status, 'active') + ) + ) .leftJoin( credential, and( eq(credential.credentialGroupEnrollmentId, credentialGroupEnrollment.id), eq(credential.workspaceId, workspaceId), eq(credential.type, 'managed_oauth'), - eq(credential.managedOauthStatus, 'active') + eq(credential.managedOauthStatus, 'active'), + /** The option must still be live, exactly as the member engine requires. */ + sql`EXISTS ( + SELECT 1 FROM jsonb_array_elements(${credentialGroup.options}) AS option + WHERE option->>'id' = ${credential.credentialGroupOptionId} + AND option->>'status' = 'active' + )` ) ) .where(and(eq(user.id, userId), eq(user.emailVerified, true))) diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index 915351ed761..10ee7defc95 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -1,7 +1,5 @@ import { AuditAction, AuditResourceType } from '@sim/audit' import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' @@ -26,9 +24,7 @@ import { getKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' import type { KnowledgeOperationSource } from '@/lib/knowledge/orchestration/shared' import { getConnectorMeta } from '@/connectors/registry' -const logger = createLogger('KnowledgeConnectorAccessApplication') - -/** Provisioning pulls in the credential-group services; loaded only when a members-mode switch needs it. */ +/** The enrollment link needs the credential-group services; loaded only when a member connects. */ async function loadMemberProvisioning() { return import('@/lib/knowledge/connectors/member-provisioning') } @@ -117,7 +113,6 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ ) } - const subjectUserId = resolvePrincipalSubjectUserId(principal) const target = input.accessMode === 'members' ? { @@ -131,13 +126,8 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ credentialGroupId: input.credentialGroupId, credentialGroupOptionId: input.credentialGroupOptionId, } - : await ( - await loadMemberProvisioning() - ).provisionKnowledgeConnectorMembersBinding({ - workspaceId, - connectorMeta, - userId: actingUserId, - }), + : null, + actingUserId, sourceConfig: connector.sourceConfig as Record, }), } @@ -165,22 +155,6 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ request, }) requireSuccessfulOutcome(outcome, 'Knowledge connector access update failed') - if (target.accessMode === 'members' && outcome.changed) { - const provisioning = await loadMemberProvisioning() - await provisioning - .inviteWorkspaceMembersToCredentialGroup({ - workspaceId, - credentialGroupId: target.binding.credentialGroupId, - inviterUserId: subjectUserId ?? undefined, - limit: provisioning.MEMBER_PROVISION_INVITES_PER_REQUEST, - }) - .catch((error) => { - logger.warn('Failed to invite workspace members after switching to members mode', { - workspaceId, - error: getErrorMessage(error), - }) - }) - } return { connector: outcome.connector, changed: outcome.changed, workspaceId } }, projectAudit: ({ input, context, result }) => diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index fcf36ed6a7f..f90afc51dd7 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -9,8 +9,6 @@ import { knowledgeConnectorMemberSyncLog, knowledgeConnectorSyncLog, } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { and, asc, count, desc, eq, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { decryptApiKey } from '@/lib/api-key/crypto' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' @@ -64,9 +62,7 @@ interface KnowledgeConnectorApplicationInput { source?: KnowledgeOperationSource } -const logger = createLogger('KnowledgeConnectorsApplication') - -/** Provisioning pulls in the credential-group services; loaded only when a members-mode connector needs it. */ +/** The viewer's standing with per-member connectors; loaded only when the list holds one. */ async function loadMemberProvisioning() { return import('@/lib/knowledge/connectors/member-provisioning') } @@ -358,7 +354,7 @@ export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ principal: Principal input: ReadKnowledgeConnectorInput }) => resolveActiveKnowledgeConnectorContext(input, principal), - async execute({ context }) { + async execute({ principal, context }) { const connector = await getKnowledgeConnector(context.knowledgeBaseId, context.connectorId) if (!connector) throw new OrchestrationError('not_found', 'Connector not found') const [syncLogs, memberSyncLogs, members] = await Promise.all([ @@ -377,7 +373,24 @@ export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ summarizeConnectorMembers(context.connectorId, connector.syncIntervalMinutes), ]) const { encryptedApiKey: _encryptedApiKey, ...connectorData } = connector - return { connector: { ...connectorData, syncLogs, memberSyncLogs, members } } + const viewerUserId = resolvePrincipalSubjectUserId(principal) + const memberships = + viewerUserId && context.workspaceId + ? await (await loadMemberProvisioning()).resolveViewerConnectorMemberships({ + userId: viewerUserId, + workspaceId: context.workspaceId, + connectors: [connector], + }) + : new Map() + return { + connector: { + ...connectorData, + viewerMembership: memberships.get(connector.id) ?? null, + syncLogs, + memberSyncLogs, + members, + }, + } }, }) @@ -435,7 +448,13 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ * which is an admin decision even though creating a connector is not. */ const subjectUserId = resolvePrincipalSubjectUserId(principal) - if (!subjectUserId || context.workspaceId === undefined) { + if (context.workspaceId === undefined) { + throw new OrchestrationError( + 'validation', + 'Per-member access needs a workspace knowledge base' + ) + } + if (!subjectUserId) { throw new OrchestrationError( 'forbidden', 'A members-mode connector needs a signed-in admin' @@ -446,21 +465,17 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ if (!connectorMeta) { throw new OrchestrationError('validation', `Unknown connector type: ${input.connectorType}`) } - const named = - input.credentialGroupId && input.credentialGroupOptionId - ? { - credentialGroupId: input.credentialGroupId, - credentialGroupOptionId: input.credentialGroupOptionId, - } - : await (await loadMemberProvisioning()).provisionKnowledgeConnectorMembersBinding({ - workspaceId, - connectorMeta, - userId: subjectUserId, - }) membersBinding = await resolveKnowledgeConnectorMembersBinding({ workspaceId, connectorMeta, - binding: named, + binding: + input.credentialGroupId && input.credentialGroupOptionId + ? { + credentialGroupId: input.credentialGroupId, + credentialGroupOptionId: input.credentialGroupOptionId, + } + : null, + actingUserId: subjectUserId, sourceConfig: input.sourceConfig, }) } @@ -490,29 +505,6 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ recordProductAnalytics: false, }) requireSuccessfulOutcome(outcome, 'Knowledge connector creation failed') - if (membersBinding && outcome.success) { - /** - * Everyone in the workspace is invited to connect, so the admin's only - * next step is to wait. Bounded here; the member run invites the rest. - */ - const provisioning = await loadMemberProvisioning() - await provisioning - .inviteWorkspaceMembersToCredentialGroup({ - workspaceId, - credentialGroupId: membersBinding.credentialGroupId, - inviterUserId: resolvePrincipalSubjectUserId(principal) ?? undefined, - limit: provisioning.MEMBER_PROVISION_INVITES_PER_REQUEST, - }) - .catch((error) => { - logger.warn( - 'Failed to invite workspace members after creating a members-mode connector', - { - workspaceId, - error: getErrorMessage(error), - } - ) - }) - } return { connector: outcome.connector, workspaceId } }, projectAudit: ({ input, context, result }) => ({ diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index 78a9ed8eba6..47a35371412 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -1010,6 +1010,7 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, input.operation, input.documentIds, + await context.access.get(), generateRequestId() ) : null diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index ecefbb838bf..d70b8a01dae 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -9,6 +9,7 @@ import { and, eq, exists, + gt, inArray, isNotNull, isNull, @@ -231,23 +232,31 @@ export async function applyMemberDocumentLifecycle(input: { lease: Pick /** External ids whose refresh did not land this run; withheld from resurrection. */ failedExternalIds: ReadonlySet + /** + * Whether absence of observers may hide or purge a document. False until at + * least one member has completed a listing: before that, nothing has been + * observed yet, so absence says nothing. + */ + allowRemoval: boolean }): Promise { const { connectorId, knowledgeBaseId, runId } = input const now = new Date() - const tombstoned = await db - .update(document) - .set({ deletedAt: now }) - .where( - and( - eq(document.connectorId, connectorId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - hasNoObservation() - ) - ) - .returning({ id: document.id }) + const tombstoned = !input.allowRemoval + ? [] + : await db + .update(document) + .set({ deletedAt: now }) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + hasNoObservation() + ) + ) + .returning({ id: document.id }) const resurrectionCandidates = await db .select({ id: document.id, externalId: document.externalId }) @@ -303,7 +312,7 @@ export async function applyMemberDocumentLifecycle(input: { } let purged = 0 const purgeIds = purgeCandidates.map((row) => row.id) - for (let offset = 0; offset < purgeIds.length; offset += PURGE_CHUNK_SIZE) { + for (let offset = 0; input.allowRemoval && offset < purgeIds.length; offset += PURGE_CHUNK_SIZE) { await input.lease.beatIfDue() purged += await hardDeleteDocuments( purgeIds.slice(offset, offset + PURGE_CHUNK_SIZE), @@ -354,6 +363,9 @@ export async function sweepStaleMemberObservations(now: Date): Promise ({ createCredentialGroupInvitationLink: vi.fn(), inviteCredentialGroupEnrollment: vi.fn(), - loadCredentialGroupInviterIdentity: vi.fn(), +})) +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: vi.fn(), +})) +vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ dispatchMemberSync: vi.fn() })) +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + resolveSystemBillingAttribution: vi.fn(), })) vi.mock('@/lib/credential-groups/service', () => ({ createCredentialGroup: vi.fn(), @@ -21,22 +27,20 @@ import { describe('pickProvisionedGroupName', () => { it('names the group after the connector and steps past taken names', () => { - expect(pickProvisionedGroupName('Google Drive', [])).toBe('Google Drive access') - expect(pickProvisionedGroupName('Google Drive', ['google drive access'])).toBe( - 'Google Drive access 2' + expect(pickProvisionedGroupName('Google Drive', [])).toBe('Google Drive') + expect(pickProvisionedGroupName('Google Drive', ['google drive'])).toBe('Google Drive 2') + expect(pickProvisionedGroupName('Google Drive', ['Google Drive', 'Google Drive 2'])).toBe( + 'Google Drive 3' ) - expect( - pickProvisionedGroupName('Google Drive', ['Google Drive access', 'Google Drive access 2']) - ).toBe('Google Drive access 3') }) it('gives up with a pointer to Settings once every candidate is taken', () => { const taken = [ - 'Google Drive access', - 'Google Drive access 2', - 'Google Drive access 3', - 'Google Drive access 4', - 'Google Drive access 5', + 'Google Drive', + 'Google Drive 2', + 'Google Drive 3', + 'Google Drive 4', + 'Google Drive 5', ] expect(() => pickProvisionedGroupName('Google Drive', taken)).toThrow('Settings') }) @@ -44,22 +48,24 @@ describe('pickProvisionedGroupName', () => { describe('deriveViewerConnectorMembership', () => { it.each([ - ['active', 'completed', 'connected'], - ['active', 'in_progress', 'connected'], - ['needs_reauth', 'completed', 'needs_reauth'], - [null, 'invited', 'invited'], - [null, 'delivery_failed', 'invited'], - [null, 'in_progress', 'invited'], - [null, 'completed', 'invited'], - ['revoked', 'completed', 'invited'], - [null, 'revoked', 'not_enrolled'], - [null, null, 'not_enrolled'], + [true, 'active', 'completed', 'connected'], + [true, 'active', 'in_progress', 'connected'], + [true, 'needs_reauth', 'completed', 'needs_reauth'], + [true, null, 'invited', 'invited'], + [true, null, 'delivery_failed', 'invited'], + [true, null, 'in_progress', 'invited'], + [true, null, 'completed', 'invited'], + [true, 'revoked', 'completed', 'invited'], + [true, 'active', 'revoked', 'revoked'], + [true, null, 'revoked', 'revoked'], + [true, null, null, 'not_enrolled'], + [false, 'active', 'completed', 'unverified_email'], ] as const)( - 'credential %s + enrollment %s → %s', - (managedOauthStatus, enrollmentStatus, expected) => { - expect(deriveViewerConnectorMembership({ managedOauthStatus, enrollmentStatus })).toBe( - expected - ) + 'verified %s + credential %s + enrollment %s → %s', + (emailVerified, managedOauthStatus, enrollmentStatus, expected) => { + expect( + deriveViewerConnectorMembership({ emailVerified, managedOauthStatus, enrollmentStatus }) + ).toBe(expected) } ) }) diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.ts index b20bf8ea667..dfcdce546d5 100644 --- a/apps/sim/lib/knowledge/connectors/member-provisioning.ts +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.ts @@ -1,13 +1,19 @@ import { db } from '@sim/db' -import { credential, credentialGroupEnrollment, user } from '@sim/db/schema' +import { + credential, + credentialGroupEnrollment, + knowledgeBase, + knowledgeConnector, + user, +} from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { and, eq, inArray } from 'drizzle-orm' +import { and, eq, inArray, isNull } from 'drizzle-orm' +import { resolveSystemBillingAttribution } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { createCredentialGroupInvitationLink, inviteCredentialGroupEnrollment, - loadCredentialGroupInviterIdentity, } from '@/lib/credential-groups/enrollments' import { getCredentialGroupProviderId, @@ -15,49 +21,47 @@ import { isCredentialGroupProvider, } from '@/lib/credential-groups/providers' import { createCredentialGroup, listCredentialGroups } from '@/lib/credential-groups/service' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { dispatchMemberSync } from '@/lib/knowledge/connectors/member-queue' import { getUsersWithPermissions } from '@/lib/workspaces/permissions/utils' import type { ConnectorMeta } from '@/connectors/types' const logger = createLogger('KnowledgeConnectorMemberProvisioning') -/** Invitations one request sends before handing the rest to the member run. */ -export const MEMBER_PROVISION_INVITES_PER_REQUEST = 25 -/** Invitations one member run sends, so a large workspace is covered within a few runs. */ -export const MEMBER_PROVISION_INVITES_PER_RUN = 100 +/** Invitations sent between two lease heartbeats of a member run. */ +const INVITATION_BATCH_SIZE = 25 /** Names tried for the group a connector provisions, in order. */ const PROVISIONED_GROUP_NAME_ATTEMPTS = 5 export interface ProvisionedMembersBinding { credentialGroupId: string credentialGroupOptionId: string - /** Whether this call created the group rather than reusing one. */ - created: boolean } /** - * The group name a connector provisions for its provider: the connector's - * name, suffixed until it is free of the workspace's existing group names. + * The name of the group a connector provisions: the connector's own name, + * which is what the invitation email and the enrollment page show, suffixed + * only when the workspace already uses it. */ export function pickProvisionedGroupName( connectorName: string, takenNames: readonly string[] ): string { const taken = new Set(takenNames.map((name) => name.trim().toLocaleLowerCase())) - const base = `${connectorName} access` for (let attempt = 1; attempt <= PROVISIONED_GROUP_NAME_ATTEMPTS; attempt++) { - const candidate = attempt === 1 ? base : `${base} ${attempt}` + const candidate = attempt === 1 ? connectorName : `${connectorName} ${attempt}` if (!taken.has(candidate.toLocaleLowerCase())) return candidate } throw new OrchestrationError( 'conflict', - `Every name from "${base}" to "${base} ${PROVISIONED_GROUP_NAME_ATTEMPTS}" is taken; pick a Credential Group in Settings` + `Every name from "${connectorName}" to "${connectorName} ${PROVISIONED_GROUP_NAME_ATTEMPTS}" is taken; pick a Credential Group in Settings` ) } /** - * The Credential Group option a members-mode connector should crawl through - * when the caller named none: the workspace's one active option collecting - * the connector's accounts, or a group created for the purpose. Two or more + * The Credential Group option a members-mode connector crawls through when + * the caller named none: the workspace's one active option collecting the + * connector's accounts, or a group created for the purpose. Two or more * candidate options is an ambiguity the caller has to resolve by naming one. */ export async function provisionKnowledgeConnectorMembersBinding(input: { @@ -88,11 +92,7 @@ export async function provisionKnowledgeConnectorMembersBinding(input: { if (option.status !== 'active') continue if (!isCredentialGroupProvider(option.provider)) continue if (getCredentialGroupProviderId(option.provider) !== providerId) continue - candidates.push({ - credentialGroupId: group.id, - credentialGroupOptionId: option.id, - created: false, - }) + candidates.push({ credentialGroupId: group.id, credentialGroupOptionId: option.id }) } } if (candidates.length === 1) return candidates[0] @@ -118,96 +118,93 @@ export async function provisionKnowledgeConnectorMembersBinding(input: { credentialGroupId: group.id, provider, }) - return { credentialGroupId: group.id, credentialGroupOptionId: option.id, created: true } + return { credentialGroupId: group.id, credentialGroupOptionId: option.id } } export interface InviteWorkspaceMembersResult { invited: number failed: number - /** Members left uninvited because the limit was reached; the next call continues. */ - remaining: number } /** - * Invites every workspace member who has no enrollment in the group yet, up - * to `limit`, so joining the workspace is all a person has to do before - * connecting their account. An enrollment an admin revoked is left alone. - * Failures are logged per person and never abort the caller. + * Invites every workspace member who has no enrollment in the group yet, so + * joining the workspace is all a person has to do before connecting their + * account. An enrollment an admin revoked is left alone. Runs inside a member + * run: `beforeBatch` beats the run's lease between batches, and failures are + * logged per person rather than aborting the run. */ export async function inviteWorkspaceMembersToCredentialGroup(input: { workspaceId: string credentialGroupId: string - inviterUserId: string | undefined - limit: number + beforeBatch: () => Promise }): Promise { - const [members, enrolled, inviter] = await Promise.all([ + const [members, enrolled] = await Promise.all([ getUsersWithPermissions(input.workspaceId), db .select({ email: credentialGroupEnrollment.email }) .from(credentialGroupEnrollment) .where(eq(credentialGroupEnrollment.credentialGroupId, input.credentialGroupId)), - input.inviterUserId - ? loadCredentialGroupInviterIdentity(input.inviterUserId) - : Promise.resolve(null), ]) const enrolledEmails = new Set(enrolled.map((row) => row.email.trim().toLocaleLowerCase())) - const pending = members - .map((member) => member.email.trim().toLocaleLowerCase()) - .filter((email, index, all) => email && all.indexOf(email) === index) - .filter((email) => !enrolledEmails.has(email)) - const batch = pending.slice(0, input.limit) - const inviterName = inviter?.name ?? inviter?.email ?? undefined + const pending = [ + ...new Set(members.map((member) => member.email.trim().toLocaleLowerCase())), + ].filter((email) => email && !enrolledEmails.has(email)) - let invited = 0 - let failed = 0 - for (const email of batch) { - try { - await inviteCredentialGroupEnrollment( - input.workspaceId, - input.credentialGroupId, - input.inviterUserId, - inviterName, - email - ) - invited += 1 - } catch (error) { - failed += 1 - logger.warn('Failed to invite a workspace member to a connector credential group', { - workspaceId: input.workspaceId, - credentialGroupId: input.credentialGroupId, - error: getErrorMessage(error), - }) + const result: InviteWorkspaceMembersResult = { invited: 0, failed: 0 } + for (let offset = 0; offset < pending.length; offset += INVITATION_BATCH_SIZE) { + await input.beforeBatch() + for (const email of pending.slice(offset, offset + INVITATION_BATCH_SIZE)) { + try { + await inviteCredentialGroupEnrollment( + input.workspaceId, + input.credentialGroupId, + undefined, + undefined, + email + ) + result.invited += 1 + } catch (error) { + result.failed += 1 + logger.warn('Failed to invite a workspace member to a connector credential group', { + workspaceId: input.workspaceId, + credentialGroupId: input.credentialGroupId, + error: getErrorMessage(error), + }) + } } } - return { invited, failed, remaining: pending.length - batch.length } + return result } -export type ViewerConnectorMembership = 'connected' | 'needs_reauth' | 'invited' | 'not_enrolled' - /** - * Where a viewer stands with a members-mode connector, from their enrollment - * and managed credential for the connector's option. + * Where a viewer stands with a members-mode connector, from their account + * and their enrollment in the connector's group. */ +export type ViewerConnectorMembership = + | 'connected' + | 'needs_reauth' + | 'invited' + | 'not_enrolled' + | 'revoked' + | 'unverified_email' + export function deriveViewerConnectorMembership(input: { + emailVerified: boolean enrollmentStatus: string | null managedOauthStatus: string | null }): ViewerConnectorMembership { + if (!input.emailVerified) return 'unverified_email' + if (input.enrollmentStatus === 'revoked') return 'revoked' if (input.managedOauthStatus === 'active') return 'connected' if (input.managedOauthStatus === 'needs_reauth') return 'needs_reauth' - if ( - input.enrollmentStatus === 'invited' || - input.enrollmentStatus === 'delivery_failed' || - input.enrollmentStatus === 'in_progress' || - input.enrollmentStatus === 'completed' - ) { - return 'invited' - } + if (input.enrollmentStatus) return 'invited' return 'not_enrolled' } /** * The viewer's membership in each members-mode connector, keyed by connector - * id. Connectors that sync as the workspace are absent. + * id. Connectors that sync as the workspace are absent, and so is everything + * where the feature is off: there is nothing the viewer could connect to. */ export async function resolveViewerConnectorMemberships(input: { userId: string @@ -227,38 +224,38 @@ export async function resolveViewerConnectorMemberships(input: { connector.credentialGroupOptionId ) if (memberConnectors.length === 0) return result + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId: input.workspaceId }))) return result const [viewer] = await db - .select({ email: user.email }) + .select({ email: user.email, emailVerified: user.emailVerified }) .from(user) .where(eq(user.id, input.userId)) .limit(1) - const email = viewer?.email.trim().toLocaleLowerCase() + if (!viewer) return result + const email = viewer.email.trim().toLocaleLowerCase() const groupIds = [...new Set(memberConnectors.map((connector) => connector.credentialGroupId!))] - const rows = email - ? await db - .select({ - credentialGroupId: credentialGroupEnrollment.credentialGroupId, - enrollmentStatus: credentialGroupEnrollment.status, - credentialGroupOptionId: credential.credentialGroupOptionId, - managedOauthStatus: credential.managedOauthStatus, - }) - .from(credentialGroupEnrollment) - .leftJoin( - credential, - and( - eq(credential.credentialGroupEnrollmentId, credentialGroupEnrollment.id), - eq(credential.workspaceId, input.workspaceId), - eq(credential.type, 'managed_oauth') - ) - ) - .where( - and( - inArray(credentialGroupEnrollment.credentialGroupId, groupIds), - eq(credentialGroupEnrollment.email, email) - ) - ) - : [] + const rows = await db + .select({ + credentialGroupId: credentialGroupEnrollment.credentialGroupId, + enrollmentStatus: credentialGroupEnrollment.status, + credentialGroupOptionId: credential.credentialGroupOptionId, + managedOauthStatus: credential.managedOauthStatus, + }) + .from(credentialGroupEnrollment) + .leftJoin( + credential, + and( + eq(credential.credentialGroupEnrollmentId, credentialGroupEnrollment.id), + eq(credential.workspaceId, input.workspaceId), + eq(credential.type, 'managed_oauth') + ) + ) + .where( + and( + inArray(credentialGroupEnrollment.credentialGroupId, groupIds), + eq(credentialGroupEnrollment.email, email) + ) + ) for (const connector of memberConnectors) { const enrollment = rows.find((row) => row.credentialGroupId === connector.credentialGroupId) @@ -270,6 +267,7 @@ export async function resolveViewerConnectorMemberships(input: { result.set( connector.id, deriveViewerConnectorMembership({ + emailVerified: viewer.emailVerified, enrollmentStatus: enrollment?.enrollmentStatus ?? null, managedOauthStatus: forOption?.managedOauthStatus ?? null, }) @@ -279,8 +277,11 @@ export async function resolveViewerConnectorMemberships(input: { } /** - * A fresh enrollment link for the viewer into the connector's group, created + * A fresh enrollment link for the viewer into the connector's group, minted * on demand so a workspace member never has to find the invitation email. + * Issued without an inviter — the person is inviting themselves — and refused + * for an enrollment an admin revoked or an account whose email is unverified, + * which could connect but would never be granted a token. */ export async function createViewerConnectorEnrollmentLink(input: { userId: string @@ -288,16 +289,81 @@ export async function createViewerConnectorEnrollmentLink(input: { credentialGroupId: string }): Promise { const [viewer] = await db - .select({ email: user.email }) + .select({ email: user.email, emailVerified: user.emailVerified }) .from(user) .where(eq(user.id, input.userId)) .limit(1) if (!viewer) throw new OrchestrationError('not_found', 'User not found') + if (!viewer.emailVerified) { + throw new OrchestrationError( + 'validation', + 'Verify your email address before connecting an account' + ) + } + const email = viewer.email.trim().toLocaleLowerCase() + const [enrollment] = await db + .select({ status: credentialGroupEnrollment.status }) + .from(credentialGroupEnrollment) + .where( + and( + eq(credentialGroupEnrollment.credentialGroupId, input.credentialGroupId), + eq(credentialGroupEnrollment.email, email) + ) + ) + .limit(1) + if (enrollment?.status === 'revoked') { + throw new OrchestrationError( + 'forbidden', + 'A workspace admin removed your access to this connector' + ) + } const { invitationLink } = await createCredentialGroupInvitationLink( input.workspaceId, input.credentialGroupId, - input.userId, - viewer.email + undefined, + email ) return invitationLink } + +/** + * Queues a member run for every connector that crawls through the option a + * member just connected, so their documents arrive within minutes rather + * than at the next scheduled run. Best effort: a refused dispatch is logged + * and the schedule catches up. + */ +export async function dispatchMemberSyncsForCredentialOption(input: { + workspaceId: string + credentialGroupOptionId: string + requestId?: string +}): Promise { + const connectors = await db + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, input.workspaceId), + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.accessMode, 'members'), + eq(knowledgeConnector.credentialGroupOptionId, input.credentialGroupOptionId), + eq(knowledgeConnector.status, 'active'), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + if (connectors.length === 0) return + const billingAttribution = await resolveSystemBillingAttribution(input.workspaceId) + for (const connector of connectors) { + const dispatch = await dispatchMemberSync(connector.id, { + billingAttribution, + requestId: input.requestId, + }) + if (!dispatch.queued) { + logger.info('Member sync after a member connected was not queued', { + connectorId: connector.id, + reason: dispatch.reason, + }) + } + } +} diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 6b7e241af4e..28281ed2c27 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -15,8 +15,6 @@ import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' -import { getWorkspaceOwnerSubscriptionAccess } from '@/lib/billing/core/workspace-access' -import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' import { type CredentialGroupOptionCredentialReference, loadCredentialGroupCredentialListContext, @@ -36,10 +34,7 @@ import { removeMemberObservationsForDocuments, removeUnseenMemberObservations, } from '@/lib/knowledge/connectors/member-observations' -import { - inviteWorkspaceMembersToCredentialGroup, - MEMBER_PROVISION_INVITES_PER_RUN, -} from '@/lib/knowledge/connectors/member-provisioning' +import { inviteWorkspaceMembersToCredentialGroup } from '@/lib/knowledge/connectors/member-provisioning' import { CONNECTOR_AUTO_DISABLED_ERROR, CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES, @@ -1071,10 +1066,35 @@ async function failMemberSyncLog(runId: string, result: MemberSyncResult, errorM } /** - * Disables member sync on a connector that can no longer run it — the - * workspace lost Credential Groups, or the group binding is gone — and - * suspends every member so their tokens leave every ACL. Nothing is purged: - * re-enabling restores access from the retained observations. + * Ends a run without doing anything because the feature is not available to + * the workspace right now. The connector keeps its members and their + * observations, records nothing as a failure, and is looked at again on its + * next schedule; a manual-only connector waits for the next manual sync. + */ +async function deferMemberSync(run: MemberSyncRun, syncIntervalMinutes: number): Promise { + const now = new Date() + await failMemberSyncLog(run.runId, run.result, 'Per-member access is not available; waiting') + await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'idle', + lastMemberSyncError: 'Per-member access is not available for this workspace', + nextMemberSyncAt: nextMemberSyncTime(now, syncIntervalMinutes, false), + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where(holdsMemberSyncLockToken(run.connectorId, run.runId)) + logger.info('Member sync deferred; per-member access is not available', { + connectorId: run.connectorId, + }) +} + +/** + * Disables member sync on a connector that can no longer run it because its + * group binding is gone, and suspends every member so their tokens leave + * every ACL. Nothing is purged: re-enabling restores access from the retained + * observations. */ async function disableMemberSync(run: MemberSyncRun, reason: string): Promise { const now = new Date() @@ -1223,19 +1243,16 @@ export async function executeMemberSync( await insertMemberSyncLog(runId, connectorId, runStartedAt) try { - const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(run.workspaceId) - if (!(await isCredentialGroupsAvailable({ workspaceId: run.workspaceId, ownerBilling }))) { - await disableMemberSync(run, 'Credential Groups are not available for this workspace') - return { - ...skipped(result, 'connector_not_syncable'), - error: 'Credential Groups are not available', - } - } + /** + * Where the feature is off — flag, plan, or a flag read that could not + * reach its source — nothing changes: readers already see no member-scoped + * document, and the run waits for the next schedule to look again. + */ if (!(await isKnowledgeMemberAccessAvailable({ workspaceId: run.workspaceId }))) { - await disableMemberSync(run, 'Per-member access is not enabled for this workspace') + await deferMemberSync(run, connector.syncIntervalMinutes) return { ...skipped(result, 'connector_not_syncable'), - error: 'Per-member access is not enabled', + error: 'Per-member access is not available for this workspace', } } if (!connector.credentialGroupId || !connector.credentialGroupOptionId) { @@ -1264,8 +1281,7 @@ export async function executeMemberSync( const invited = await inviteWorkspaceMembersToCredentialGroup({ workspaceId: run.workspaceId, credentialGroupId: connector.credentialGroupId, - inviterUserId: undefined, - limit: MEMBER_PROVISION_INVITES_PER_RUN, + beforeBatch: run.lease.beatIfDue, }).catch((error) => { logger.warn('Failed to invite new workspace members during a member run', { connectorId, @@ -1424,12 +1440,28 @@ export async function executeMemberSync( await materializeDocumentAcls(connectorId, affectedDocumentIds) + /** + * Nobody has completed a listing yet — a connector that just entered + * members mode, waiting for its first member to connect — so an + * unobserved document says nothing about access and must not be + * tombstoned, let alone purged a week later. + */ + const [listed] = await db + .select({ count: sql`count(*)::int` }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, connectorId), + sql`${knowledgeConnectorMember.lastCompleteListingAt} IS NOT NULL` + ) + ) const lifecycle = await applyMemberDocumentLifecycle({ connectorId, knowledgeBaseId: connector.knowledgeBaseId, runId, lease: run.lease, failedExternalIds: state.failedExternalIds, + allowRemoval: (listed?.count ?? 0) > 0, }) result.docsTombstoned = lifecycle.tombstoned result.docsResurrected = lifecycle.resurrected diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 5322726a991..d6e9140c18b 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -2946,6 +2946,7 @@ export async function bulkDocumentOperation( knowledgeBaseId: string, operation: 'enable' | 'disable' | 'delete', documentIds: string[], + access: KnowledgeAccessScope, requestId: string ): Promise<{ success: boolean @@ -2973,7 +2974,8 @@ export async function bulkDocumentOperation( inArray(document.id, documentIds), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + knowledgeAccessCondition(access) ) ) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts index b2eca33e8fa..4a4f7ba310b 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ dispatchSync: vi.fn(), dispatchMemberSync: vi.fn(), memberAccessAvailable: vi.fn(), + provision: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -42,6 +43,9 @@ vi.mock('@/lib/credential-groups/credentials', () => ({ vi.mock('@/lib/knowledge/access/availability', () => ({ isKnowledgeMemberAccessAvailable: mocks.memberAccessAvailable, })) +vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ + provisionKnowledgeConnectorMembersBinding: mocks.provision, +})) vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mocks.dispatchSync })) vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ dispatchMemberSync: mocks.dispatchMemberSync, @@ -99,18 +103,69 @@ function switchTo(target: Parameters { beforeEach(() => { vi.clearAllMocks() mocks.memberAccessAvailable.mockResolvedValue(true) }) + it('refuses a connector whose listing is not permission-scoped, before loading anything', async () => { + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: { name: 'Slack', auth: { mode: 'oauth' }, configFields: [] } as never, + actingUserId: 'admin-1', + binding: null, + sourceConfig: {}, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mocks.provision).not.toHaveBeenCalled() + expect(mocks.loadGroup).not.toHaveBeenCalled() + }) + + it('provisions the group when none is named, then validates it like a named one', async () => { + mocks.provision.mockResolvedValue({ + credentialGroupId: 'group-9', + credentialGroupOptionId: 'option-9', + }) + mocks.loadGroup.mockResolvedValue({ workspaceId: 'ws-1', status: 'active', options: [] }) + mocks.validateBinding.mockReturnValue({ ok: true, option: {} }) + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', + binding: null, + sourceConfig: {}, + }) + ).resolves.toEqual({ + credentialGroupId: 'group-9', + credentialGroupOptionId: 'option-9', + workspaceId: 'ws-1', + sourceConfig: {}, + }) + expect(mocks.provision).toHaveBeenCalledWith({ + workspaceId: 'ws-1', + connectorMeta: SCOPED_META, + userId: 'admin-1', + }) + expect(mocks.loadGroup).toHaveBeenCalledWith('group-9') + }) + it('refuses members mode where the feature is off, before loading anything', async () => { mocks.memberAccessAvailable.mockResolvedValue(false) await expect( resolveKnowledgeConnectorMembersBinding({ workspaceId: 'ws-1', - connectorMeta: {} as never, + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, sourceConfig: {}, }) @@ -124,7 +179,8 @@ describe('resolveKnowledgeConnectorMembersBinding', () => { await expect( resolveKnowledgeConnectorMembersBinding({ workspaceId: 'ws-1', - connectorMeta: {} as never, + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, sourceConfig: {}, }) @@ -138,7 +194,8 @@ describe('resolveKnowledgeConnectorMembersBinding', () => { await expect( resolveKnowledgeConnectorMembersBinding({ workspaceId: 'ws-1', - connectorMeta: {} as never, + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, sourceConfig: { maxFiles: '5' }, }) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index cae16325963..21e658698ce 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -74,7 +74,10 @@ export interface ResolvedMembersBinding extends KnowledgeConnectorMembersBinding export async function resolveKnowledgeConnectorMembersBinding(input: { workspaceId: string connectorMeta: Pick - binding: KnowledgeConnectorMembersBinding + /** The option the caller named, or null to sync through the workspace's group for the provider, created if need be. */ + binding: KnowledgeConnectorMembersBinding | null + /** The admin acting, recorded as the creator of a provisioned group. */ + actingUserId: string sourceConfig: Record }): Promise { /** @@ -87,19 +90,34 @@ export async function resolveKnowledgeConnectorMembersBinding(input: { 'Per-member access is not available for this workspace' ) } - const group = await loadCredentialGroupCredentialListContext(input.binding.credentialGroupId) + if (!input.connectorMeta.permissionScopedListing) { + throw new OrchestrationError( + 'validation', + `${input.connectorMeta.name} cannot sync per member: its listing does not reflect who may read each document` + ) + } + const sourceConfig = stripListingCapFields(input.connectorMeta, input.sourceConfig) + const binding = + input.binding ?? + (await ( + await import('@/lib/knowledge/connectors/member-provisioning') + ).provisionKnowledgeConnectorMembersBinding({ + workspaceId: input.workspaceId, + connectorMeta: input.connectorMeta, + userId: input.actingUserId, + })) + const group = await loadCredentialGroupCredentialListContext(binding.credentialGroupId) if (!group || group.workspaceId !== input.workspaceId) { throw new OrchestrationError('validation', 'Credential Group was not found in this workspace') } - const sourceConfig = stripListingCapFields(input.connectorMeta, input.sourceConfig) const validation = validateKnowledgeConnectorMembersBinding({ connectorMeta: input.connectorMeta, group, - credentialGroupOptionId: input.binding.credentialGroupOptionId, + credentialGroupOptionId: binding.credentialGroupOptionId, sourceConfig, }) if (!validation.ok) throw new OrchestrationError('validation', validation.message) - return { ...input.binding, workspaceId: input.workspaceId, sourceConfig } + return { ...binding, workspaceId: input.workspaceId, sourceConfig } } /** @@ -242,6 +260,38 @@ export async function performUpdateKnowledgeConnectorAccess( : target.binding.credentialGroupId === existing.credentialGroupId && target.binding.credentialGroupOptionId === existing.credentialGroupOptionId) if (unchanged) { + /** + * Re-applying the current binding on a connector whose member sync was + * disabled is how it is re-enabled: the next run reconciles members from + * the group again and restores access from the retained observations. + */ + if (target.accessMode === 'members' && existing.memberSyncStatus === 'disabled') { + const now = new Date() + const [updated] = await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'idle', + memberSyncConsecutiveFailures: 0, + lastMemberSyncError: null, + nextMemberSyncAt: now, + updatedAt: now, + }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.memberSyncStatus, 'disabled'), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() + if (!updated) return fail('Connector changed; retry the request', 'conflict') + logger.info(`[${requestId}] Re-enabled member sync on connector ${connectorId}`) + const { encryptedApiKey: _secret, ...connector } = updated + if (updated.status !== 'paused') { + await dispatchMemberSyncBestEffort(connectorId, params, requestId, now) + } + return { success: true, connector, changed: true } + } const { encryptedApiKey: _secret, ...connector } = existing return { success: true, connector, changed: false } } @@ -393,6 +443,13 @@ export async function performUpdateKnowledgeConnectorAccess( credentialGroupId: null, credentialGroupOptionId: null, accessRewritePending: true, + /** + * The next content sync must list everything and reconcile: the + * union of every member's documents may hold documents the + * workspace credential cannot see, and only a full listing removes + * them. + */ + lastSyncAt: null, memberSyncStatus: 'idle', memberSyncConsecutiveFailures: 0, lastMemberSyncError: null, From 83eb6e2c9c5c023ac7f4271290bb4750769bc2fd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 21:09:36 -0700 Subject: [PATCH 19/76] feat(knowledge): list a workspace's per-member connectors for the viewer One read returns every per-member connector in the workspace with where the viewer stands on each, so a surface outside the knowledge base can ask them to connect. The member sync status enum now has one home in lib/knowledge/types. --- .../api/knowledge/member-connectors/route.ts | 20 ++++++ apps/sim/hooks/queries/kb/connectors.ts | 33 +++++++++ .../lib/api/contracts/knowledge/connectors.ts | 24 ++++++- .../lib/knowledge/application/connectors.ts | 71 +++++++++++++++++++ .../knowledge/application/operations.test.ts | 1 + .../lib/knowledge/application/operations.ts | 8 +++ apps/sim/lib/knowledge/types.ts | 8 +++ 7 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 apps/sim/app/api/knowledge/member-connectors/route.ts diff --git a/apps/sim/app/api/knowledge/member-connectors/route.ts b/apps/sim/app/api/knowledge/member-connectors/route.ts new file mode 100644 index 00000000000..4467c3ec35e --- /dev/null +++ b/apps/sim/app/api/knowledge/member-connectors/route.ts @@ -0,0 +1,20 @@ +import { listWorkspaceMemberConnectorsContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { listWorkspaceMemberConnectors } from '@/lib/knowledge/application/connectors' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const GET = defineInternalJsonRoute({ + contract: listWorkspaceMemberConnectorsContract, + auth: internalSessionAuth, + operation: knowledgeOperations.listWorkspaceMemberConnectors, + rateLimit: internalRateLimits.none({ reason: 'One small read per visit to the Search tab' }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ query }) => ({ workspaceId: query.workspaceId }), + useCase: listWorkspaceMemberConnectors, + present: ({ connectors }) => ({ success: true as const, data: connectors }), +}) diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 7549b993eeb..1b395defd10 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -17,6 +17,7 @@ import { getKnowledgeConnectorContract, listKnowledgeConnectorDocumentsContract, listKnowledgeConnectorsContract, + listWorkspaceMemberConnectorsContract, type MemberSyncLogData, patchKnowledgeConnectorDocumentsContract, type StartKnowledgeConnectorMemberEnrollmentData, @@ -27,12 +28,14 @@ import { updateKnowledgeConnectorAccessContract, updateKnowledgeConnectorContract, type ViewerConnectorMembership, + type WorkspaceMemberConnector, } from '@/lib/api/contracts/knowledge' import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' export type { ViewerConnectorMembership, + WorkspaceMemberConnector, ConnectorData, ConnectorDetailData, ConnectorMemberSummary, @@ -353,6 +356,36 @@ async function startConnectorMemberEnrollment({ return response.data } +export const memberConnectorKeys = { + all: ['member-connectors'] as const, + lists: () => [...memberConnectorKeys.all, 'list'] as const, + list: (workspaceId?: string) => [...memberConnectorKeys.lists(), workspaceId ?? ''] as const, +} + +export const WORKSPACE_MEMBER_CONNECTORS_STALE_TIME = 30 * 1000 + +async function fetchWorkspaceMemberConnectors( + workspaceId: string, + signal?: AbortSignal +): Promise { + const response = await requestJson(listWorkspaceMemberConnectorsContract, { + query: { workspaceId }, + signal, + }) + return response.data +} + +/** Every per-member connector in the workspace and where the viewer stands with each. */ +export function useWorkspaceMemberConnectors(workspaceId?: string) { + return useQuery({ + queryKey: memberConnectorKeys.list(workspaceId), + queryFn: ({ signal }) => fetchWorkspaceMemberConnectors(workspaceId as string, signal), + enabled: Boolean(workspaceId), + staleTime: WORKSPACE_MEMBER_CONNECTORS_STALE_TIME, + placeholderData: keepPreviousData, + }) +} + /** Mints the viewer's enrollment link for a per-member connector; the caller navigates to it. */ export function useStartConnectorMemberEnrollment() { return useMutation({ mutationFn: startConnectorMemberEnrollment }) diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index 9ce7814eebb..86fb791a09a 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -11,6 +11,7 @@ import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, } from '@/lib/knowledge/constants' +import { MEMBER_SYNC_STATUSES } from '@/lib/knowledge/types' /** * How a connector derives document access. `workspace` syncs as one credential @@ -169,7 +170,7 @@ export const connectorDataSchema = z credentialGroupId: z.string().nullable(), credentialGroupOptionId: z.string().nullable(), /** Members mode only; `idle` otherwise. */ - memberSyncStatus: z.enum(['idle', 'pending', 'running', 'error', 'disabled']), + memberSyncStatus: z.enum(MEMBER_SYNC_STATUSES), lastMemberSyncAt: z.string().nullable(), nextMemberSyncAt: z.string().nullable(), lastMemberSyncError: z.string().nullable(), @@ -346,6 +347,27 @@ export const startKnowledgeConnectorMemberEnrollmentContract = defineRouteContra }, }) +/** A per-member connector as the viewer meets it across the workspace's knowledge bases. */ +export const workspaceMemberConnectorSchema = z.object({ + knowledgeBaseId: z.string(), + knowledgeBaseName: z.string(), + connectorId: z.string(), + connectorType: z.string(), + memberSyncStatus: z.enum(MEMBER_SYNC_STATUSES), + viewerMembership: viewerConnectorMembershipSchema, +}) +export type WorkspaceMemberConnector = z.output + +export const listWorkspaceMemberConnectorsContract = defineRouteContract({ + method: 'GET', + path: '/api/knowledge/member-connectors', + query: z.object({ workspaceId: z.string().min(1) }), + response: { + mode: 'json', + schema: successResponseSchema(z.array(workspaceMemberConnectorSchema)), + }, +}) + export const deleteKnowledgeConnectorContract = defineRouteContract({ method: 'DELETE', path: '/api/knowledge/[id]/connectors/[connectorId]', diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index f90afc51dd7..6cda94ad6a5 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -4,6 +4,7 @@ import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' import { document, + knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeConnectorMemberSyncLog, @@ -30,6 +31,7 @@ import { type ActiveKnowledgeResourceBaseContext, resolveActiveKnowledgeConnectorContext, resolveActiveKnowledgeResourceContext, + resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' @@ -52,6 +54,7 @@ import type { KnowledgeOperationSource, KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' +import { isMemberSyncStatus } from '@/lib/knowledge/types' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { CAPABILITY_RULES, refuseCapability } from '@/lib/permission-groups/capabilities' import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' @@ -345,6 +348,74 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ }, }) +export interface ListWorkspaceMemberConnectorsInput { + workspaceId: string +} + +/** + * Every per-member connector in the workspace and where the viewer stands + * with each, so a surface outside the knowledge base — Sim Search — can ask + * them to connect. Only connectors the viewer could actually read documents + * from are listed: the knowledge base must be live and in the workspace. + */ +export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listWorkspaceMemberConnectors, + resolveContext: ({ input }: { input: ListWorkspaceMemberConnectorsInput }) => + resolveKnowledgeWorkspaceContext(input), + async execute({ principal, context }) { + const viewerUserId = resolvePrincipalSubjectUserId(principal) + if (!viewerUserId) return { connectors: [] } + const rows = await db + .select({ + knowledgeBaseId: knowledgeConnector.knowledgeBaseId, + knowledgeBaseName: knowledgeBase.name, + id: knowledgeConnector.id, + connectorType: knowledgeConnector.connectorType, + accessMode: knowledgeConnector.accessMode, + memberSyncStatus: knowledgeConnector.memberSyncStatus, + credentialGroupId: knowledgeConnector.credentialGroupId, + credentialGroupOptionId: knowledgeConnector.credentialGroupOptionId, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, context.workspaceId), + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.accessMode, 'members'), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .orderBy(asc(knowledgeBase.name), asc(knowledgeConnector.createdAt)) + const memberships = await (await loadMemberProvisioning()).resolveViewerConnectorMemberships({ + userId: viewerUserId, + workspaceId: context.workspaceId, + connectors: rows, + }) + return { + connectors: rows.flatMap((row) => { + const viewerMembership = memberships.get(row.id) + if (!isMemberSyncStatus(row.memberSyncStatus)) { + throw new Error(`Unexpected member sync status ${row.memberSyncStatus}`) + } + return viewerMembership + ? [ + { + knowledgeBaseId: row.knowledgeBaseId, + knowledgeBaseName: row.knowledgeBaseName, + connectorId: row.id, + connectorType: row.connectorType, + memberSyncStatus: row.memberSyncStatus, + viewerMembership, + }, + ] + : [] + }), + } + }, +}) + export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readConnector, resolveContext: ({ diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index e8ccb213d12..747332b284e 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -56,6 +56,7 @@ describe('knowledge operation registry', () => { 'knowledge.connectors.create', 'knowledge.connectors.update', 'knowledge.connectors.access.update', + 'knowledge.connectors.members.list', 'knowledge.connectors.members.enroll', 'knowledge.connectors.delete', 'knowledge.connectors.sync', diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index 1b6aee03ab5..88bdb8a0e33 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -402,6 +402,14 @@ export const knowledgeOperations = { * A workspace member joining a per-member connector: any reader may connect * their own account, which only ever widens what they themselves see. */ + /** Every per-member connector in the workspace, with where the viewer stands on each. */ + listWorkspaceMemberConnectors: defineWorkspaceOperation({ + id: 'knowledge.connectors.members.list', + minimumRole: 'read', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }), enrollConnectorMember: defineWorkspaceOperation({ id: 'knowledge.connectors.members.enroll', minimumRole: 'read', diff --git a/apps/sim/lib/knowledge/types.ts b/apps/sim/lib/knowledge/types.ts index 6009ba86f0f..7a7d22666a4 100644 --- a/apps/sim/lib/knowledge/types.ts +++ b/apps/sim/lib/knowledge/types.ts @@ -207,3 +207,11 @@ interface DocumentsPagination { offset: number hasMore: boolean } + +/** The member engine's states, as stored on `knowledge_connector.member_sync_status`. */ +export const MEMBER_SYNC_STATUSES = ['idle', 'pending', 'running', 'error', 'disabled'] as const +export type MemberSyncStatus = (typeof MEMBER_SYNC_STATUSES)[number] + +export function isMemberSyncStatus(value: string): value is MemberSyncStatus { + return (MEMBER_SYNC_STATUSES as readonly string[]).includes(value) +} From 37db0a204bdb889f14f8a427173fac0e6c4e624d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 21:26:30 -0700 Subject: [PATCH 20/76] feat(search): show per-member knowledge connectors on the Search tab The Search tab lists the knowledge bases whose connectors sync per member under "Shared with you", with where the viewer stands on each and the same one-click connect the knowledge base page offers. One hook opens enrollment in a new tab and refreshes the surface until the viewer is connected; the knowledge base banner uses it too. --- .../member-connect-banner.tsx | 83 +++++------- .../member-connectors-section.tsx | 119 ++++++++++++++++++ .../workspace/[workspaceId]/search/search.tsx | 3 + apps/sim/hooks/use-member-enrollment.ts | 75 +++++++++++ 4 files changed, 228 insertions(+), 52 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx create mode 100644 apps/sim/hooks/use-member-enrollment.ts diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx index ee6f3136fcb..7094278b4fc 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx @@ -1,21 +1,15 @@ 'use client' -import { useEffect, useState } from 'react' +import { useMemo } from 'react' import { Button } from '@sim/emcn' import { Loader } from '@sim/emcn/icons' -import { createLogger } from '@sim/logger' -import { useQueryClient } from '@tanstack/react-query' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { type ConnectorData, connectorKeys, - useStartConnectorMemberEnrollment, + type ViewerConnectorMembership, } from '@/hooks/queries/kb/connectors' - -const logger = createLogger('MemberConnectBanner') - -/** How often the connector list is refreshed while a member connects in another tab. */ -const AWAITING_CONNECTION_POLL_MS = 4_000 +import { useMemberEnrollment } from '@/hooks/use-member-enrollment' interface MemberConnectBannerProps { knowledgeBaseId: string @@ -26,19 +20,18 @@ function connectorName(connector: ConnectorData): string { return CONNECTOR_META_REGISTRY[connector.connectorType]?.name ?? connector.connectorType } +/** Memberships the viewer can act on themselves. */ +const CONNECTABLE: ReadonlySet = new Set([ + 'needs_reauth', + 'invited', + 'not_enrolled', +]) + /** - * What the viewer must do for a per-member connector, if anything. Enrollment - * opens in a new tab — the enrollment page ends by telling the person to close - * it — and the list is polled meanwhile so the row updates on its own. + * What the viewer must do for each per-member connector, if anything, and + * what is happening for them once they have connected. */ export function MemberConnectBanner({ knowledgeBaseId, connectors }: MemberConnectBannerProps) { - const queryClient = useQueryClient() - const { mutate: startEnrollment, isPending } = useStartConnectorMemberEnrollment() - const [awaitingConnectorIds, setAwaitingConnectorIds] = useState>( - () => new Set() - ) - const [error, setError] = useState(null) - const rows = connectors.filter( (connector) => connector.accessMode === 'members' && @@ -47,44 +40,32 @@ export function MemberConnectBanner({ knowledgeBaseId, connectors }: MemberConne connector.memberSyncStatus === 'pending' || connector.memberSyncStatus === 'running') ) - - const awaiting = rows.some( - (connector) => - awaitingConnectorIds.has(connector.id) && connector.viewerMembership !== 'connected' + const connectedConnectorIds = useMemo( + () => + new Set( + connectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.id) + ), + [connectors] ) - useEffect(() => { - if (!awaiting) return - const timer = setInterval(() => { - void queryClient.invalidateQueries({ queryKey: connectorKeys.lists(knowledgeBaseId) }) - }, AWAITING_CONNECTION_POLL_MS) - return () => clearInterval(timer) - }, [awaiting, knowledgeBaseId, queryClient]) + const membershipQueryKeys = useMemo( + () => [connectorKeys.lists(knowledgeBaseId)], + [knowledgeBaseId] + ) + const { connect, isAwaiting, isPending, error } = useMemberEnrollment({ + membershipQueryKeys, + connectedConnectorIds, + }) if (rows.length === 0) return null - const connect = (connectorId: string) => { - setError(null) - startEnrollment( - { knowledgeBaseId, connectorId }, - { - onSuccess: ({ url }) => { - window.open(url, '_blank', 'noopener') - setAwaitingConnectorIds((current) => new Set([...current, connectorId])) - }, - onError: (err) => { - logger.error('Failed to start member enrollment', { error: err.message }) - setError(err.message) - }, - } - ) - } - return (
{rows.map((connector) => { const name = connectorName(connector) const membership = connector.viewerMembership - const waiting = awaitingConnectorIds.has(connector.id) && membership !== 'connected' + const waiting = isAwaiting(connector.id) const text = membership === 'connected' ? `Syncing the ${name} documents shared with you. They appear when the sync completes.` @@ -97,8 +78,6 @@ export function MemberConnectBanner({ knowledgeBaseId, connectors }: MemberConne : waiting ? `Finish connecting your ${name} account in the other tab.` : `Connect your ${name} account to see the documents shared with you.` - const canConnect = - membership === 'needs_reauth' || membership === 'invited' || membership === 'not_enrolled' return (
{text}

- {canConnect && ( + {membership && CONNECTABLE.has(membership) && ( + ) : undefined + } + /> + ) + })} + {error &&

{error}

} + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx index 34c629ce2a2..f19aee99b36 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -17,6 +17,7 @@ import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/c import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' import { CONNECTED_LABEL } from '@/app/workspace/[workspaceId]/integrations/search-params' +import { MemberConnectorsSection } from '@/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section' import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' import { connectorSearchParam, @@ -150,6 +151,8 @@ export function Search() { />
+ + {visibleCredentials.length > 0 && ( {visibleCredentials.map((credential) => ( diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts new file mode 100644 index 00000000000..816352b29ba --- /dev/null +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -0,0 +1,75 @@ +'use client' + +import { useCallback, useEffect, useState } from 'react' +import { createLogger } from '@sim/logger' +import { type QueryKey, useQueryClient } from '@tanstack/react-query' +import { useStartConnectorMemberEnrollment } from '@/hooks/queries/kb/connectors' + +const logger = createLogger('MemberEnrollment') + +/** How often the given queries are refreshed while a member connects in another tab. */ +const AWAITING_CONNECTION_POLL_MS = 4_000 + +interface UseMemberEnrollmentProps { + /** Queries carrying the viewer's membership, refreshed while a connection is awaited. */ + membershipQueryKeys: readonly QueryKey[] + /** Connector ids the viewer is now connected to; awaiting stops for them. */ + connectedConnectorIds: ReadonlySet +} + +/** + * Lets the viewer connect their own account to a per-member connector. + * Enrollment opens in a new tab — the enrollment page ends by telling the + * person to close it — and the membership queries are polled meanwhile so + * the surface that started it updates on its own once they are connected. + */ +export function useMemberEnrollment({ + membershipQueryKeys, + connectedConnectorIds, +}: UseMemberEnrollmentProps) { + const queryClient = useQueryClient() + const { mutate: startEnrollment, isPending } = useStartConnectorMemberEnrollment() + const [awaitingConnectorIds, setAwaitingConnectorIds] = useState>( + () => new Set() + ) + const [error, setError] = useState(null) + + const awaiting = [...awaitingConnectorIds].some((id) => !connectedConnectorIds.has(id)) + useEffect(() => { + if (!awaiting) return + const timer = setInterval(() => { + for (const queryKey of membershipQueryKeys) { + void queryClient.invalidateQueries({ queryKey }) + } + }, AWAITING_CONNECTION_POLL_MS) + return () => clearInterval(timer) + }, [awaiting, membershipQueryKeys, queryClient]) + + const connect = useCallback( + (knowledgeBaseId: string, connectorId: string) => { + setError(null) + startEnrollment( + { knowledgeBaseId, connectorId }, + { + onSuccess: ({ url }) => { + window.open(url, '_blank', 'noopener') + setAwaitingConnectorIds((current) => new Set([...current, connectorId])) + }, + onError: (err) => { + logger.error('Failed to start member enrollment', { error: err.message }) + setError(err.message) + }, + } + ) + }, + [startEnrollment] + ) + + const isAwaiting = useCallback( + (connectorId: string) => + awaitingConnectorIds.has(connectorId) && !connectedConnectorIds.has(connectorId), + [awaitingConnectorIds, connectedConnectorIds] + ) + + return { connect, isAwaiting, isPending, error } +} From 4724422e18afd6d425d6e90be14c00fa995437d9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 21:29:18 -0700 Subject: [PATCH 21/76] feat(knowledge): enable per-member access on every permission-scoped connector Google Slides, Docs, Forms, Calendar, Gmail, Notion, Jira, Jira Service Management, Box, Dropbox, monday, ClickUp, Asana, Salesforce, Linear, DocuSign, and Zoom list only what the caller's own account can read, so each now declares its listing caps and can sync per member. A shared listing-scope error lets a connector report a folder, space, board, or calendar the member cannot reach as a complete listing of nothing; Notion and Asana report it from their typed errors, Dropbox from its 409, and the Atlassian connectors also from an unreachable site. Google Docs and Forms join the credential-group providers so their accounts can be collected per person. --- apps/sim/connectors/asana/asana.ts | 2 ++ apps/sim/connectors/asana/meta.ts | 1 + apps/sim/connectors/box/meta.ts | 1 + apps/sim/connectors/clickup/clickup.ts | 10 ++++-- apps/sim/connectors/clickup/meta.ts | 1 + apps/sim/connectors/docusign/meta.ts | 1 + apps/sim/connectors/dropbox/dropbox.ts | 12 ++++++- apps/sim/connectors/dropbox/meta.ts | 1 + apps/sim/connectors/gmail/meta.ts | 1 + .../google-calendar/google-calendar.ts | 11 +++++-- apps/sim/connectors/google-calendar/meta.ts | 1 + apps/sim/connectors/google-docs/meta.ts | 1 + .../connectors/google-forms/google-forms.ts | 6 +++- apps/sim/connectors/google-forms/meta.ts | 1 + .../connectors/google-slides/google-slides.ts | 6 +++- apps/sim/connectors/google-slides/meta.ts | 1 + apps/sim/connectors/jira/jira.ts | 18 +++++++++-- apps/sim/connectors/jira/meta.ts | 1 + apps/sim/connectors/jsm/jsm.ts | 13 ++++++-- apps/sim/connectors/jsm/meta.ts | 1 + apps/sim/connectors/linear/meta.ts | 1 + apps/sim/connectors/monday/meta.ts | 1 + apps/sim/connectors/notion/meta.ts | 1 + apps/sim/connectors/notion/notion.ts | 3 ++ .../permission-scoped-listing.test.ts | 22 ++++++++++++- apps/sim/connectors/salesforce/meta.ts | 1 + apps/sim/connectors/utils.ts | 32 +++++++++++++++++++ apps/sim/connectors/zoom/meta.ts | 1 + apps/sim/lib/auth/connectors/managed-oauth.ts | 4 ++- .../credential-groups/provider-registry.ts | 2 ++ apps/sim/lib/credential-groups/providers.ts | 12 +++++++ 31 files changed, 156 insertions(+), 14 deletions(-) diff --git a/apps/sim/connectors/asana/asana.ts b/apps/sim/connectors/asana/asana.ts index 576921e2882..b507fade076 100644 --- a/apps/sim/connectors/asana/asana.ts +++ b/apps/sim/connectors/asana/asana.ts @@ -314,6 +314,8 @@ async function listWorkspaceProjects( export const asanaConnector: ConnectorConfig = { ...asanaConnectorMeta, + isListingScopeUnavailableError: (error) => error instanceof AsanaApiError && error.status === 404, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/asana/meta.ts b/apps/sim/connectors/asana/meta.ts index 075b14e0f2a..0d146487c82 100644 --- a/apps/sim/connectors/asana/meta.ts +++ b/apps/sim/connectors/asana/meta.ts @@ -10,6 +10,7 @@ export const asanaConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'asana', requiredScopes: ['default'] }, + permissionScopedListing: { capFieldIds: ['maxTasks'] }, configFields: [ { id: 'workspaceSelector', diff --git a/apps/sim/connectors/box/meta.ts b/apps/sim/connectors/box/meta.ts index a8c2643ef30..9670b0575f0 100644 --- a/apps/sim/connectors/box/meta.ts +++ b/apps/sim/connectors/box/meta.ts @@ -14,6 +14,7 @@ export const boxConnectorMeta: ConnectorMeta = { requiredScopes: ['root_readwrite'], }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'folderId', diff --git a/apps/sim/connectors/clickup/clickup.ts b/apps/sim/connectors/clickup/clickup.ts index 7d42855face..a86aaa27d20 100644 --- a/apps/sim/connectors/clickup/clickup.ts +++ b/apps/sim/connectors/clickup/clickup.ts @@ -4,7 +4,11 @@ import { isRecordLike } from '@sim/utils/object' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { clickupConnectorMeta } from '@/connectors/clickup/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseTagDate } from '@/connectors/utils' +import { + isListingScopeUnavailableError, + listingRequestError, + parseTagDate, +} from '@/connectors/utils' import { clickupAuthorizationHeader, extractClickUpErrorMessage } from '@/tools/clickup/shared' const logger = createLogger('ClickUpConnector') @@ -159,6 +163,8 @@ function getRequiredWorkspaceId(sourceConfig: Record): string { export const clickupConnector: ConnectorConfig = { ...clickupConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -181,7 +187,7 @@ export const clickupConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list ClickUp Docs', { status: response.status, error: errorText }) - throw new Error(`Failed to list ClickUp Docs: ${response.status}`) + throw listingRequestError('Failed to list ClickUp Docs', response.status) } const data = (await response.json()) as Record diff --git a/apps/sim/connectors/clickup/meta.ts b/apps/sim/connectors/clickup/meta.ts index 5879b5cf2ab..76004caeefd 100644 --- a/apps/sim/connectors/clickup/meta.ts +++ b/apps/sim/connectors/clickup/meta.ts @@ -13,6 +13,7 @@ export const clickupConnectorMeta: ConnectorMeta = { provider: 'clickup', }, + permissionScopedListing: { capFieldIds: ['maxDocs'] }, configFields: [ { id: 'workspaceSelector', diff --git a/apps/sim/connectors/docusign/meta.ts b/apps/sim/connectors/docusign/meta.ts index 08a67c77e47..86e59da2770 100644 --- a/apps/sim/connectors/docusign/meta.ts +++ b/apps/sim/connectors/docusign/meta.ts @@ -16,6 +16,7 @@ export const docusignConnectorMeta: ConnectorMeta = { supportsIncrementalSync: true, + permissionScopedListing: { capFieldIds: ['maxEnvelopes'] }, configFields: [ { id: 'lookback', diff --git a/apps/sim/connectors/dropbox/dropbox.ts b/apps/sim/connectors/dropbox/dropbox.ts index 5e44cfb9edd..60b1cda51ce 100644 --- a/apps/sim/connectors/dropbox/dropbox.ts +++ b/apps/sim/connectors/dropbox/dropbox.ts @@ -6,7 +6,9 @@ import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/ import { CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, htmlToPlainText, + isListingScopeUnavailableError, isSkippedDocument, markSkipped, parseTagDate, @@ -170,6 +172,8 @@ function fileToStub(entry: DropboxFileMetadata): ExternalDocument { export const dropboxConnector: ConnectorConfig = { ...dropboxConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -227,7 +231,13 @@ export const dropboxConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to list Dropbox folder: ${response.status}`) + /** Dropbox reports a path the caller cannot reach as 409 path/not_found. */ + throw response.status === 409 + ? new ConnectorListingScopeUnavailableError( + `Failed to list Dropbox folder: ${response.status}`, + response.status + ) + : new Error(`Failed to list Dropbox folder: ${response.status}`) } data = await response.json() diff --git a/apps/sim/connectors/dropbox/meta.ts b/apps/sim/connectors/dropbox/meta.ts index 9e294b6fe7f..655935dbc25 100644 --- a/apps/sim/connectors/dropbox/meta.ts +++ b/apps/sim/connectors/dropbox/meta.ts @@ -14,6 +14,7 @@ export const dropboxConnectorMeta: ConnectorMeta = { requiredScopes: ['files.metadata.read', 'files.content.read'], }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'folderPath', diff --git a/apps/sim/connectors/gmail/meta.ts b/apps/sim/connectors/gmail/meta.ts index a5f91fcab23..bdeb1a6371b 100644 --- a/apps/sim/connectors/gmail/meta.ts +++ b/apps/sim/connectors/gmail/meta.ts @@ -16,6 +16,7 @@ export const gmailConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/gmail.modify'], }, + permissionScopedListing: { capFieldIds: ['maxThreads'] }, configFields: [ { id: 'labelSelector', diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index 921907a82ec..620ee62906a 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -3,7 +3,12 @@ import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + isListingScopeUnavailableError, + listingRequestError, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('GoogleCalendarConnector') @@ -301,6 +306,8 @@ function eventToDocument( export const googleCalendarConnector: ConnectorConfig = { ...googleCalendarConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -397,7 +404,7 @@ export const googleCalendarConnector: ConnectorConfig = { calendarId, error: errorText, }) - throw new Error(`Failed to list Google Calendar events: ${response.status}`) + throw listingRequestError('Failed to list Google Calendar events', response.status) } const data = await response.json() diff --git a/apps/sim/connectors/google-calendar/meta.ts b/apps/sim/connectors/google-calendar/meta.ts index dda94336817..dc9249aa5e3 100644 --- a/apps/sim/connectors/google-calendar/meta.ts +++ b/apps/sim/connectors/google-calendar/meta.ts @@ -16,6 +16,7 @@ export const googleCalendarConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/calendar'], }, + permissionScopedListing: { capFieldIds: ['maxEvents'] }, configFields: [ { id: 'calendarSelector', diff --git a/apps/sim/connectors/google-docs/meta.ts b/apps/sim/connectors/google-docs/meta.ts index eaca1bd826b..c3ef01e56dd 100644 --- a/apps/sim/connectors/google-docs/meta.ts +++ b/apps/sim/connectors/google-docs/meta.ts @@ -14,6 +14,7 @@ export const googleDocsConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], }, + permissionScopedListing: { capFieldIds: ['maxDocs'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/google-forms/google-forms.ts b/apps/sim/connectors/google-forms/google-forms.ts index 443b394443a..957e039cb93 100644 --- a/apps/sim/connectors/google-forms/google-forms.ts +++ b/apps/sim/connectors/google-forms/google-forms.ts @@ -5,7 +5,9 @@ import { googleFormsConnectorMeta, MAX_RESPONSES_PER_FORM } from '@/connectors/g import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { buildDriveParentsClause, + isListingScopeUnavailableError, joinTagArray, + listingRequestError, parseMultiValue, parseTagDate, } from '@/connectors/utils' @@ -434,6 +436,8 @@ function buildDriveQuery(folderIds: string[]): string { export const googleFormsConnector: ConnectorConfig = { ...googleFormsConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -480,7 +484,7 @@ export const googleFormsConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list Google Forms', { status: response.status, error: errorText }) - throw new Error(`Failed to list Google Forms: ${response.status}`) + throw listingRequestError('Failed to list Google Forms', response.status) } const data = await response.json() diff --git a/apps/sim/connectors/google-forms/meta.ts b/apps/sim/connectors/google-forms/meta.ts index 3ea4b31fbdb..73f87cef9bf 100644 --- a/apps/sim/connectors/google-forms/meta.ts +++ b/apps/sim/connectors/google-forms/meta.ts @@ -24,6 +24,7 @@ export const googleFormsConnectorMeta: ConnectorMeta = { ], }, + permissionScopedListing: { capFieldIds: ['maxForms'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/google-slides/google-slides.ts b/apps/sim/connectors/google-slides/google-slides.ts index cd957564e99..1257cf60aa5 100644 --- a/apps/sim/connectors/google-slides/google-slides.ts +++ b/apps/sim/connectors/google-slides/google-slides.ts @@ -7,7 +7,9 @@ import { buildDriveParentsClause, CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + isListingScopeUnavailableError, joinTagArray, + listingRequestError, markSkipped, parseMultiValue, parseTagDate, @@ -286,6 +288,8 @@ function buildQuery(sourceConfig: Record, lastSyncAt?: Date): s export const googleSlidesConnector: ConnectorConfig = { ...googleSlidesConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -339,7 +343,7 @@ export const googleSlidesConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to list Google Slides presentations: ${response.status}`) + throw listingRequestError('Failed to list Google Slides presentations', response.status) } const data = await response.json() diff --git a/apps/sim/connectors/google-slides/meta.ts b/apps/sim/connectors/google-slides/meta.ts index a4cc9a17aef..72242deef12 100644 --- a/apps/sim/connectors/google-slides/meta.ts +++ b/apps/sim/connectors/google-slides/meta.ts @@ -19,6 +19,7 @@ export const googleSlidesConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], }, + permissionScopedListing: { capFieldIds: ['maxDocs'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index a957b3f7468..9d2d3af3a6f 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -1,10 +1,19 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' +import { + AtlassianSiteNotAccessibleError, + normalizeAtlassianSiteUrl, +} from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jiraConnectorMeta } from '@/connectors/jira/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + isListingScopeUnavailableError, + joinTagArray, + listingRequestError, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils' const logger = createLogger('JiraConnector') @@ -131,6 +140,9 @@ function issueToFullDocument(issue: Record, siteUrl: string): E export const jiraConnector: ConnectorConfig = { ...jiraConnectorMeta, + isListingScopeUnavailableError: (error) => + isListingScopeUnavailableError(error) || error instanceof AtlassianSiteNotAccessibleError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -219,7 +231,7 @@ export const jiraConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to search Jira issues: ${response.status}`) + throw listingRequestError('Failed to search Jira issues', response.status) } const data = await response.json() diff --git a/apps/sim/connectors/jira/meta.ts b/apps/sim/connectors/jira/meta.ts index 40c1749618a..d38ee66c672 100644 --- a/apps/sim/connectors/jira/meta.ts +++ b/apps/sim/connectors/jira/meta.ts @@ -10,6 +10,7 @@ export const jiraConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'jira', requiredScopes: ['read:jira-work', 'offline_access'] }, + permissionScopedListing: { capFieldIds: ['maxIssues'] }, configFields: [ { id: 'domain', diff --git a/apps/sim/connectors/jsm/jsm.ts b/apps/sim/connectors/jsm/jsm.ts index ee42607ed9a..7f7a493a061 100644 --- a/apps/sim/connectors/jsm/jsm.ts +++ b/apps/sim/connectors/jsm/jsm.ts @@ -1,9 +1,15 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { AtlassianSiteNotAccessibleError } from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jsmConnectorMeta } from '@/connectors/jsm/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + isListingScopeUnavailableError, + listingRequestError, + parseTagDate, +} from '@/connectors/utils' import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils' import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' @@ -421,6 +427,9 @@ async function fetchComments( export const jsmConnector: ConnectorConfig = { ...jsmConnectorMeta, + isListingScopeUnavailableError: (error) => + isListingScopeUnavailableError(error) || error instanceof AtlassianSiteNotAccessibleError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -498,7 +507,7 @@ export const jsmConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list JSM requests', { status: response.status, error: errorText }) - throw new Error(`Failed to list JSM requests: ${response.status}`) + throw listingRequestError('Failed to list JSM requests', response.status) } const data = (await response.json()) as JsmPage diff --git a/apps/sim/connectors/jsm/meta.ts b/apps/sim/connectors/jsm/meta.ts index 8b4f79003f8..22c9cd4b748 100644 --- a/apps/sim/connectors/jsm/meta.ts +++ b/apps/sim/connectors/jsm/meta.ts @@ -36,6 +36,7 @@ export const jsmConnectorMeta: ConnectorMeta = { ], }, + permissionScopedListing: { capFieldIds: ['maxRequests'] }, configFields: [ { id: 'domain', diff --git a/apps/sim/connectors/linear/meta.ts b/apps/sim/connectors/linear/meta.ts index 33f9e4604ff..070346a5e13 100644 --- a/apps/sim/connectors/linear/meta.ts +++ b/apps/sim/connectors/linear/meta.ts @@ -18,6 +18,7 @@ export const linearConnectorMeta: ConnectorMeta = { */ supportsIncrementalSync: true, + permissionScopedListing: { capFieldIds: ['maxIssues'] }, configFields: [ { id: 'teamSelector', diff --git a/apps/sim/connectors/monday/meta.ts b/apps/sim/connectors/monday/meta.ts index ab9ffc882e1..00d01149e2e 100644 --- a/apps/sim/connectors/monday/meta.ts +++ b/apps/sim/connectors/monday/meta.ts @@ -14,6 +14,7 @@ export const mondayConnectorMeta: ConnectorMeta = { requiredScopes: ['boards:read', 'updates:read', 'me:read'], }, + permissionScopedListing: { capFieldIds: ['maxItems'] }, configFields: [ { id: 'boardSelector', diff --git a/apps/sim/connectors/notion/meta.ts b/apps/sim/connectors/notion/meta.ts index ad4a8b13cab..2395cab412b 100644 --- a/apps/sim/connectors/notion/meta.ts +++ b/apps/sim/connectors/notion/meta.ts @@ -10,6 +10,7 @@ export const notionConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'notion', requiredScopes: [] }, + permissionScopedListing: { capFieldIds: ['maxPages'] }, configFields: [ { id: 'scope', diff --git a/apps/sim/connectors/notion/notion.ts b/apps/sim/connectors/notion/notion.ts index fea1c0f0e69..501331c425b 100644 --- a/apps/sim/connectors/notion/notion.ts +++ b/apps/sim/connectors/notion/notion.ts @@ -515,6 +515,9 @@ function pageToStub(page: Record): ExternalDocument { export const notionConnector: ConnectorConfig = { ...notionConnectorMeta, + isListingScopeUnavailableError: (error) => + error instanceof NotionApiError && error.status === 404, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/permission-scoped-listing.test.ts b/apps/sim/connectors/permission-scoped-listing.test.ts index 71490455975..d7798f500fd 100644 --- a/apps/sim/connectors/permission-scoped-listing.test.ts +++ b/apps/sim/connectors/permission-scoped-listing.test.ts @@ -22,7 +22,27 @@ const permissionScoped = Object.values(CONNECTOR_META_REGISTRY).filter( */ describe('permission-scoped connector listings', () => { it('covers the connectors that crawl per member', () => { - expect(permissionScoped.map((meta) => meta.id).sort()).toEqual(['confluence', 'google_drive']) + expect(permissionScoped.map((meta) => meta.id).sort()).toEqual([ + 'asana', + 'box', + 'clickup', + 'confluence', + 'docusign', + 'dropbox', + 'gmail', + 'google_calendar', + 'google_docs', + 'google_drive', + 'google_forms', + 'google_slides', + 'jira', + 'jsm', + 'linear', + 'monday', + 'notion', + 'salesforce', + 'zoom', + ]) }) it.each(permissionScoped.map((meta) => [meta.id, meta] as const))( diff --git a/apps/sim/connectors/salesforce/meta.ts b/apps/sim/connectors/salesforce/meta.ts index bf9b4e0fa97..648c23d443b 100644 --- a/apps/sim/connectors/salesforce/meta.ts +++ b/apps/sim/connectors/salesforce/meta.ts @@ -21,6 +21,7 @@ export const salesforceConnectorMeta: ConnectorMeta = { requiredScopes: ['api', 'refresh_token', 'openid'], }, + permissionScopedListing: { capFieldIds: ['maxRecords'] }, configFields: [ { id: 'objectType', diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 8672c1ecb62..b5ed92d2d0c 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -695,3 +695,35 @@ export class ConnectorFileTooLargeError extends Error { this.name = 'ConnectorFileTooLargeError' } } + +/** + * A listing failed because the caller cannot reach the configured scope — the + * folder, space, board, or calendar is not shared with them. A members-mode + * crawl treats that as a complete listing of nothing for that member, so + * their access is withdrawn rather than retried forever. + */ +export class ConnectorListingScopeUnavailableError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'ConnectorListingScopeUnavailableError' + } +} + +/** + * The error a listing request throws for a failed response: scope-unavailable + * when the source says the scope does not exist for this caller (404), a plain + * error for anything else, which the sync engines retry with backoff. + */ +export function listingRequestError(message: string, status: number): Error { + const described = `${message}: ${status}` + return status === 404 + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + +export function isListingScopeUnavailableError(error: unknown): boolean { + return error instanceof ConnectorListingScopeUnavailableError +} diff --git a/apps/sim/connectors/zoom/meta.ts b/apps/sim/connectors/zoom/meta.ts index 39be56551e4..b23e4d7627a 100644 --- a/apps/sim/connectors/zoom/meta.ts +++ b/apps/sim/connectors/zoom/meta.ts @@ -19,6 +19,7 @@ export const zoomConnectorMeta: ConnectorMeta = { supportsIncrementalSync: true, + permissionScopedListing: { capFieldIds: ['maxRecordings'] }, configFields: [ { id: 'lookback', diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index f0adce284af..fcf307c731e 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -1043,7 +1043,9 @@ function resolveManagedOAuthPolicy( if ( providerId === 'google-email' || providerId === 'google-calendar' || - providerId === 'google-drive' + providerId === 'google-drive' || + providerId === 'google-docs' || + providerId === 'google-forms' ) { return () => createGoogleManagedOAuthConnector(providerId) } diff --git a/apps/sim/lib/credential-groups/provider-registry.ts b/apps/sim/lib/credential-groups/provider-registry.ts index b8f9a178067..9fc55743e94 100644 --- a/apps/sim/lib/credential-groups/provider-registry.ts +++ b/apps/sim/lib/credential-groups/provider-registry.ts @@ -13,6 +13,8 @@ const CREDENTIAL_GROUP_PROVIDER_ADAPTERS: Record< gmail: createStandardOAuthCredentialGroupProviderAdapter('gmail'), 'google-calendar': createStandardOAuthCredentialGroupProviderAdapter('google-calendar'), 'google-drive': createStandardOAuthCredentialGroupProviderAdapter('google-drive'), + 'google-docs': createStandardOAuthCredentialGroupProviderAdapter('google-docs'), + 'google-forms': createStandardOAuthCredentialGroupProviderAdapter('google-forms'), confluence: createStandardOAuthCredentialGroupProviderAdapter('confluence'), jira: createStandardOAuthCredentialGroupProviderAdapter('jira'), airtable: createStandardOAuthCredentialGroupProviderAdapter('airtable'), diff --git a/apps/sim/lib/credential-groups/providers.ts b/apps/sim/lib/credential-groups/providers.ts index 6a27f436fe6..e56f4d8dbd8 100644 --- a/apps/sim/lib/credential-groups/providers.ts +++ b/apps/sim/lib/credential-groups/providers.ts @@ -5,6 +5,8 @@ export const CREDENTIAL_GROUP_STANDARD_OAUTH_PROVIDER_IDS = [ 'gmail', 'google-calendar', 'google-drive', + 'google-docs', + 'google-forms', 'confluence', 'jira', 'airtable', @@ -61,6 +63,16 @@ const CREDENTIAL_GROUP_PROVIDER_SUPPORT: Record< description: 'Let each person connect one Google Drive account', configuration: 'oauth', }, + 'google-docs': { + serviceId: 'google-docs', + description: 'Let each person connect one Google Docs account', + configuration: 'oauth', + }, + 'google-forms': { + serviceId: 'google-forms', + description: 'Let each person connect one Google Forms account', + configuration: 'oauth', + }, confluence: { serviceId: 'confluence', description: 'Let each person connect one Confluence account', From b4201fc5ad3bd59431021ba9d033d374c4000799 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 21:57:09 -0700 Subject: [PATCH 22/76] refactor(knowledge): tighten the per-member access surface - The credential-group policy canonicaliser carries knowledge-connector statements through instead of rejecting a third statement - Google Docs and Box report an unreachable scope so a member's access is withdrawn rather than retried forever; Notion leaves members mode because its page access is granted to the shared integration bot - Provisioning reuses a Credential Group only when other members-mode connectors already sync through it, never a group curated for something else; the fast dispatch on connect accepts the same statuses as the sweep - Enrollment opens its tab inside the click so popup blockers cannot swallow it, and the membership poll stops after ten minutes - Session-only routes use session auth; members mode stores the cap-stripped config; one liveness rule for members and managed bindings - Drop dead code: the access barrel, system.ts and rank.ts folded into their siblings, unused ACL helpers, the unread change_cursor_at column, stale documentation, and the engine/provisioning/queue import cycle --- .../connectors/[connectorId]/access/route.ts | 13 +- .../connectors/[connectorId]/enroll/route.ts | 11 +- .../app/api/knowledge/search/utils.test.ts | 4 +- apps/sim/app/api/knowledge/utils.ts | 83 ------------- .../app/api/v1/knowledge/search/route.test.ts | 2 +- apps/sim/app/api/v1/knowledge/search/route.ts | 3 +- .../connector-access-field.tsx | 2 +- .../connectors-section/connectors-section.tsx | 7 +- .../member-connectors-section.tsx | 74 +++++------ .../[workspaceId]/search/search.test.tsx | 13 ++ apps/sim/connectors/box/box.ts | 9 +- .../sim/connectors/google-docs/google-docs.ts | 6 +- apps/sim/connectors/notion/meta.ts | 1 - apps/sim/connectors/notion/notion.ts | 3 - .../permission-scoped-listing.test.ts | 1 - apps/sim/hooks/queries/kb/connectors.ts | 18 +-- apps/sim/hooks/use-member-enrollment.ts | 40 ++++-- apps/sim/lib/auth/connectors/managed-oauth.ts | 11 +- apps/sim/lib/core/config/feature-flags.ts | 10 -- apps/sim/lib/credential-groups/credentials.ts | 5 +- apps/sim/lib/credential-groups/oauth.ts | 2 +- apps/sim/lib/knowledge/access/availability.ts | 10 +- apps/sim/lib/knowledge/access/index.ts | 30 ----- .../lib/knowledge/access/predicate.test.ts | 2 +- apps/sim/lib/knowledge/access/predicate.ts | 3 +- apps/sim/lib/knowledge/access/system.ts | 16 --- apps/sim/lib/knowledge/access/tokens.test.ts | 20 +-- apps/sim/lib/knowledge/access/tokens.ts | 12 -- apps/sim/lib/knowledge/access/types.ts | 17 +++ .../knowledge/application/connector-access.ts | 12 +- .../lib/knowledge/application/connectors.ts | 22 ++-- .../lib/knowledge/application/operations.ts | 8 +- apps/sim/lib/knowledge/application/search.ts | 2 +- .../lib/knowledge/chunks/keyset-sql.test.ts | 2 +- .../connectors/member-observations.ts | 16 --- .../connectors/member-provisioning.test.ts | 19 +++ .../connectors/member-provisioning.ts | 115 +++++++++--------- .../lib/knowledge/connectors/member-queue.ts | 39 ++++++ .../connectors/member-sync-engine.ts | 26 ++-- .../processing-outbox-handler.test.ts | 2 +- .../documents/processing-outbox-handler.ts | 2 +- apps/sim/lib/knowledge/documents/service.ts | 3 +- .../orchestration/connector-access.test.ts | 1 - .../orchestration/connector-access.ts | 8 +- .../lib/knowledge/orchestration/connectors.ts | 7 +- apps/sim/lib/knowledge/search/queries.ts | 5 +- apps/sim/lib/knowledge/search/rank.ts | 7 -- apps/sim/lib/knowledge/search/recency.ts | 8 +- apps/sim/lib/workspaces/host-context.ts | 2 +- .../db/credential-group-resource-policies.ts | 60 +++++++-- .../0318_permission_aware_knowledge.sql | 1 - .../db/migrations/meta/0318_snapshot.json | 6 - packages/db/schema.ts | 1 - packages/testing/src/mocks/schema.mock.ts | 1 - 54 files changed, 378 insertions(+), 425 deletions(-) delete mode 100644 apps/sim/lib/knowledge/access/index.ts delete mode 100644 apps/sim/lib/knowledge/access/system.ts delete mode 100644 apps/sim/lib/knowledge/search/rank.ts diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts index 129ee5eea24..ab89571ab7e 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts @@ -1,19 +1,20 @@ import { updateKnowledgeConnectorAccessContract } from '@/lib/api/contracts/knowledge' -import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' import { resolveInternalKnowledgeBillingAttribution, toInternalKnowledgeConnector, } from '@/lib/knowledge/api/internal-route' -import { - internalKnowledgeErrorPolicies, - internalKnowledgeSessionOrExecutorAuth, -} from '@/lib/knowledge/api/route-policies' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { updateKnowledgeConnectorAccess } from '@/lib/knowledge/application/connector-access' import { knowledgeOperations } from '@/lib/knowledge/application/operations' export const PATCH = defineInternalJsonRoute({ contract: updateKnowledgeConnectorAccessContract, - auth: internalKnowledgeSessionOrExecutorAuth, + auth: internalSessionAuth, operation: knowledgeOperations.updateConnectorAccess, rateLimit: internalRateLimits.none({ reason: 'A settings action an admin performs by hand; the switch itself is bounded', diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts index 23a6c8f6f77..da3cc91c176 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts @@ -1,15 +1,16 @@ import { startKnowledgeConnectorMemberEnrollmentContract } from '@/lib/api/contracts/knowledge' -import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - internalKnowledgeErrorPolicies, - internalKnowledgeSessionOrExecutorAuth, -} from '@/lib/knowledge/api/route-policies' + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' import { knowledgeOperations } from '@/lib/knowledge/application/operations' export const POST = defineInternalJsonRoute({ contract: startKnowledgeConnectorMemberEnrollmentContract, - auth: internalKnowledgeSessionOrExecutorAuth, + auth: internalSessionAuth, operation: knowledgeOperations.enrollConnectorMember, rateLimit: internalRateLimits.none({ reason: diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 4e335621c35..f6c3c0f603c 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -48,18 +48,18 @@ afterEach(() => { }) import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' +import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' import { executeKeywordSearch, executeKnowledgeSearch, fuseByReciprocalRank, - generateSearchEmbedding, getQueryStrategy, handleTagAndVectorSearch, handleTagOnlySearch, handleVectorOnlySearch, type SearchResult, } from '@/lib/knowledge/search/queries' -import { RRF_K } from '@/lib/knowledge/search/rank' +import { RRF_K } from '@/lib/knowledge/search/recency' /** Minimal SearchResult builder — only the fields fusion and ordering read. */ function makeResult(id: string, distance = 0.1): SearchResult { diff --git a/apps/sim/app/api/knowledge/utils.ts b/apps/sim/app/api/knowledge/utils.ts index 12ca5ce728a..20a00d38cba 100644 --- a/apps/sim/app/api/knowledge/utils.ts +++ b/apps/sim/app/api/knowledge/utils.ts @@ -18,89 +18,6 @@ interface KnowledgeBaseData { updatedAt: Date } -interface DocumentData { - id: string - knowledgeBaseId: string - filename: string - fileUrl: string - fileSize: number - mimeType: string - chunkCount: number - tokenCount: number - characterCount: number - processingStatus: string - processingStartedAt?: Date | null - processingCompletedAt?: Date | null - processingError?: string | null - enabled: boolean - deletedAt?: Date | null - uploadedAt: Date - // Text tags - tag1?: string | null - tag2?: string | null - tag3?: string | null - tag4?: string | null - tag5?: string | null - tag6?: string | null - tag7?: string | null - // Number tags (5 slots) - number1?: number | null - number2?: number | null - number3?: number | null - number4?: number | null - number5?: number | null - // Date tags (2 slots) - date1?: Date | null - date2?: Date | null - // Boolean tags (3 slots) - boolean1?: boolean | null - boolean2?: boolean | null - boolean3?: boolean | null - // Connector fields - connectorId?: string | null - sourceUrl?: string | null - externalId?: string | null -} - -interface EmbeddingData { - id: string - knowledgeBaseId: string - documentId: string - chunkIndex: number - chunkHash: string - content: string - contentLength: number - tokenCount: number - embedding?: number[] | null - embeddingModel: string - startOffset: number - endOffset: number - // Text tags - tag1?: string | null - tag2?: string | null - tag3?: string | null - tag4?: string | null - tag5?: string | null - tag6?: string | null - tag7?: string | null - // Number tags (5 slots) - number1?: number | null - number2?: number | null - number3?: number | null - number4?: number | null - number5?: number | null - // Date tags (2 slots) - date1?: Date | null - date2?: Date | null - // Boolean tags (3 slots) - boolean1?: boolean | null - boolean2?: boolean | null - boolean3?: boolean | null - enabled: boolean - createdAt: Date - updatedAt: Date -} - export interface KnowledgeBaseAccessResult { hasAccess: true knowledgeBase: Pick< diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index cabb7b98df9..659739bd027 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -46,7 +46,6 @@ const SYSTEM_BILLING_ATTRIBUTION = { vi.mock('@/lib/knowledge/search/queries', () => ({ executeKnowledgeSearch: mockExecuteKnowledgeSearch, - generateSearchEmbedding: mockGenerateSearchEmbedding, getDocumentMetadataByIds: mockGetDocumentMetadataByIds, })) @@ -63,6 +62,7 @@ vi.mock('@/lib/billing/core/billing-attribution', () => ({ })) vi.mock('@/lib/knowledge/embeddings', () => ({ + generateSearchEmbedding: mockGenerateSearchEmbedding, recordSearchEmbeddingUsage: mockRecordSearchEmbeddingUsage, })) diff --git a/apps/sim/app/api/v1/knowledge/search/route.ts b/apps/sim/app/api/v1/knowledge/search/route.ts index 5cd1b364309..05037fb52f0 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.ts @@ -8,11 +8,10 @@ import { } from '@/lib/billing/core/billing-attribution' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' -import { recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' +import { generateSearchEmbedding, recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' import { executeKnowledgeSearch, - generateSearchEmbedding, getDocumentMetadataByIds, type SearchResult, } from '@/lib/knowledge/search/queries' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx index 9bc2d8089ad..081584039c5 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx @@ -62,7 +62,7 @@ export function ConnectorAccessField({ if (value.accessMode !== 'members') return null return ( - undefined}> + Workspace diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index e7f7f554282..5bb7b73f09b 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -35,6 +35,7 @@ import { CONNECTOR_SYNC_STALE_LOCK_TTL_MS, MEMBER_SYNC_STALE_LOCK_TTL_MS, } from '@/lib/knowledge/connectors/sync-limits' +import type { MemberSyncStatus } from '@/lib/knowledge/types' import { getCanonicalScopesForProvider, getProviderIdFromServiceId } from '@/lib/oauth' import { getMissingRequiredScopes } from '@/lib/oauth/utils' import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' @@ -99,7 +100,7 @@ const MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS = { running: 'syncing', error: 'error', disabled: 'disabled', -} as const satisfies Record +} as const satisfies Record const CONNECTOR_ACTION_BUTTON_CLASSES = 'size-7 rounded-lg p-0 text-[var(--text-muted)] hover-hover:bg-[var(--surface-active)] hover-hover:text-[var(--text-primary)]' @@ -316,9 +317,7 @@ function ConnectorCard({ */ const effectiveStatus = connector.accessMode === 'members' && connector.status === 'active' - ? (MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS[ - connector.memberSyncStatus as keyof typeof MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS - ] ?? 'active') + ? MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS[connector.memberSyncStatus] : connector.status const statusConfig = STATUS_CONFIG[effectiveStatus as keyof typeof STATUS_CONFIG] || STATUS_CONFIG.active diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx index 74f1f6e29be..3e405779ecb 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx @@ -78,42 +78,44 @@ export function MemberConnectorsSection({ workspaceId, search }: MemberConnector if (visible.length === 0) return null return ( - - {visible.map((connector) => { - const meta = CONNECTOR_META_REGISTRY[connector.connectorType] - const waiting = isAwaiting(connector.connectorId) - const connectable = CONNECTABLE.has(connector.viewerMembership) - return ( - - ) : undefined - } - title={meta?.name ?? connector.connectorType} - description={describe(connector, waiting)} - trailing={ - connectable ? ( - - ) : undefined - } - /> - ) - })} + <> + + {visible.map((connector) => { + const meta = CONNECTOR_META_REGISTRY[connector.connectorType] + const waiting = isAwaiting(connector.connectorId) + const connectable = CONNECTABLE.has(connector.viewerMembership) + return ( + + ) : undefined + } + title={meta?.name ?? connector.connectorType} + description={describe(connector, waiting)} + trailing={ + connectable ? ( + + ) : undefined + } + /> + ) + })} + {error &&

{error}

} -
+ ) } diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx index ad77ad25833..73a8e2570e7 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx @@ -107,6 +107,19 @@ vi.mock('@/hooks/queries/credentials', () => ({ }), })) +vi.mock('@/hooks/queries/kb/connectors', () => ({ + memberConnectorKeys: { list: (workspaceId?: string) => ['member-connectors', workspaceId] }, + useWorkspaceMemberConnectors: () => ({ data: [] }), +})) +vi.mock('@/hooks/use-member-enrollment', () => ({ + useMemberEnrollment: () => ({ + connect: vi.fn(), + isAwaiting: () => false, + isPending: false, + error: null, + }), +})) + import { Search } from '@/app/workspace/[workspaceId]/search/search' let root: Root | null = null diff --git a/apps/sim/connectors/box/box.ts b/apps/sim/connectors/box/box.ts index aa5bcae9a82..e21b1e5224f 100644 --- a/apps/sim/connectors/box/box.ts +++ b/apps/sim/connectors/box/box.ts @@ -7,7 +7,9 @@ import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/ import { CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, htmlToPlainText, + isListingScopeUnavailableError, isSkippedDocument, markSkipped, parseTagDate, @@ -426,6 +428,8 @@ async function listFolderPage( export const boxConnector: ConnectorConfig = { ...boxConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -453,8 +457,9 @@ export const boxConnector: ConnectorConfig = { * reporting a successful sync that indexed nothing. */ if (!page && position.folderId === rootFolderId) { - throw new Error( - `Box denied access to folder ${rootFolderId}. Reconnect the Box account or choose another folder.` + throw new ConnectorListingScopeUnavailableError( + `Box denied access to folder ${rootFolderId}. Reconnect the Box account or choose another folder.`, + 403 ) } diff --git a/apps/sim/connectors/google-docs/google-docs.ts b/apps/sim/connectors/google-docs/google-docs.ts index 150f807074e..f5a0d2faac7 100644 --- a/apps/sim/connectors/google-docs/google-docs.ts +++ b/apps/sim/connectors/google-docs/google-docs.ts @@ -11,7 +11,9 @@ import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/ import { buildDriveParentsClause, ConnectorFileTooLargeError, + isListingScopeUnavailableError, joinTagArray, + listingRequestError, markSkipped, parseMultiValue, parseOptionalUnlimitedSafeInteger, @@ -493,6 +495,8 @@ function buildQuery(sourceConfig: Record): string { export const googleDocsConnector: ConnectorConfig = { ...googleDocsConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -553,7 +557,7 @@ export const googleDocsConnector: ConnectorConfig = { status: response.status, error: failure, }) - throw new Error(`Failed to list Google Docs: ${failure}`) + throw listingRequestError(`Failed to list Google Docs: ${failure}`, response.status) } const data = parseDriveFileListResponse(await response.json()) diff --git a/apps/sim/connectors/notion/meta.ts b/apps/sim/connectors/notion/meta.ts index 2395cab412b..ad4a8b13cab 100644 --- a/apps/sim/connectors/notion/meta.ts +++ b/apps/sim/connectors/notion/meta.ts @@ -10,7 +10,6 @@ export const notionConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'notion', requiredScopes: [] }, - permissionScopedListing: { capFieldIds: ['maxPages'] }, configFields: [ { id: 'scope', diff --git a/apps/sim/connectors/notion/notion.ts b/apps/sim/connectors/notion/notion.ts index 501331c425b..fea1c0f0e69 100644 --- a/apps/sim/connectors/notion/notion.ts +++ b/apps/sim/connectors/notion/notion.ts @@ -515,9 +515,6 @@ function pageToStub(page: Record): ExternalDocument { export const notionConnector: ConnectorConfig = { ...notionConnectorMeta, - isListingScopeUnavailableError: (error) => - error instanceof NotionApiError && error.status === 404, - listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/permission-scoped-listing.test.ts b/apps/sim/connectors/permission-scoped-listing.test.ts index d7798f500fd..57546cfe067 100644 --- a/apps/sim/connectors/permission-scoped-listing.test.ts +++ b/apps/sim/connectors/permission-scoped-listing.test.ts @@ -39,7 +39,6 @@ describe('permission-scoped connector listings', () => { 'jsm', 'linear', 'monday', - 'notion', 'salesforce', 'zoom', ]) diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 1b395defd10..be9411fee50 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -176,9 +176,9 @@ function setCachedConnectorStatus( * which is all `onError` needs to undo it — the mutation variables already * carry the ids. * - * Both status-changing mutations resolve into the same list, so they share this - * write instead of each keeping a local `Set` of in-flight ids alongside it — - * that duplicated the server's own state and could not survive a remount. + * The pause and resume mutations share this write instead of each keeping a + * local `Set` of in-flight ids alongside it — that duplicated the server's own + * state and could not survive a remount. * * Deliberately not a snapshot of the whole array: two connectors can be in * flight at once, and restoring a whole-list snapshot would roll the other @@ -335,12 +335,6 @@ async function updateConnectorAccess({ return result.data } -/** - * Moves a connector between workspace and members mode. The switch rewrites - * document access, so everything under the base is refetched: the connector - * list and detail for the new mode and member state, and the document lists - * whose rows may have become hidden or visible. - */ interface StartConnectorMemberEnrollmentParams { knowledgeBaseId: string connectorId: string @@ -391,6 +385,12 @@ export function useStartConnectorMemberEnrollment() { return useMutation({ mutationFn: startConnectorMemberEnrollment }) } +/** + * Moves a connector between workspace and members mode. The switch rewrites + * document access, so everything under the base is refetched: the connector + * list and detail for the new mode and member state, and the document lists + * whose rows may have become hidden or visible. + */ export function useUpdateConnectorAccess() { const queryClient = useQueryClient() diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index 816352b29ba..b9a0179a9d2 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -9,6 +9,8 @@ const logger = createLogger('MemberEnrollment') /** How often the given queries are refreshed while a member connects in another tab. */ const AWAITING_CONNECTION_POLL_MS = 4_000 +/** How long a connection is awaited before the surface stops refreshing on its own. */ +const AWAITING_CONNECTION_TIMEOUT_MS = 10 * 60_000 interface UseMemberEnrollmentProps { /** Queries carrying the viewer's membership, refreshed while a connection is awaited. */ @@ -22,6 +24,10 @@ interface UseMemberEnrollmentProps { * Enrollment opens in a new tab — the enrollment page ends by telling the * person to close it — and the membership queries are polled meanwhile so * the surface that started it updates on its own once they are connected. + * + * The tab is opened in the click itself, before the enrollment link is + * minted, because a tab opened after a network round trip is outside the + * click's activation window and popup blockers swallow it. */ export function useMemberEnrollment({ membershipQueryKeys, @@ -29,33 +35,49 @@ export function useMemberEnrollment({ }: UseMemberEnrollmentProps) { const queryClient = useQueryClient() const { mutate: startEnrollment, isPending } = useStartConnectorMemberEnrollment() - const [awaitingConnectorIds, setAwaitingConnectorIds] = useState>( - () => new Set() - ) + const [awaitingSince, setAwaitingSince] = useState>(() => new Map()) const [error, setError] = useState(null) - const awaiting = [...awaitingConnectorIds].some((id) => !connectedConnectorIds.has(id)) + const awaiting = [...awaitingSince.keys()].some((id) => !connectedConnectorIds.has(id)) useEffect(() => { if (!awaiting) return const timer = setInterval(() => { + const now = Date.now() + setAwaitingSince((current) => { + const next = new Map( + [...current].filter( + ([id, since]) => + !connectedConnectorIds.has(id) && now - since < AWAITING_CONNECTION_TIMEOUT_MS + ) + ) + return next.size === current.size ? current : next + }) for (const queryKey of membershipQueryKeys) { void queryClient.invalidateQueries({ queryKey }) } }, AWAITING_CONNECTION_POLL_MS) return () => clearInterval(timer) - }, [awaiting, membershipQueryKeys, queryClient]) + }, [awaiting, connectedConnectorIds, membershipQueryKeys, queryClient]) const connect = useCallback( (knowledgeBaseId: string, connectorId: string) => { setError(null) + const tab = window.open('about:blank', '_blank') + if (tab) tab.opener = null startEnrollment( { knowledgeBaseId, connectorId }, { onSuccess: ({ url }) => { - window.open(url, '_blank', 'noopener') - setAwaitingConnectorIds((current) => new Set([...current, connectorId])) + if (tab && !tab.closed) { + tab.location.href = url + } else { + window.location.assign(url) + return + } + setAwaitingSince((current) => new Map(current).set(connectorId, Date.now())) }, onError: (err) => { + tab?.close() logger.error('Failed to start member enrollment', { error: err.message }) setError(err.message) }, @@ -67,8 +89,8 @@ export function useMemberEnrollment({ const isAwaiting = useCallback( (connectorId: string) => - awaitingConnectorIds.has(connectorId) && !connectedConnectorIds.has(connectorId), - [awaitingConnectorIds, connectedConnectorIds] + awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId), + [awaitingSince, connectedConnectorIds] ) return { connect, isAwaiting, isPending, error } diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index fcf307c731e..8b2f28aba33 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -68,9 +68,6 @@ function canonicalGoogleScope(scope: string): string { return scope } -const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive' -const DRIVE_READONLY_SCOPE = 'https://www.googleapis.com/auth/drive.readonly' - function hasRequiredGoogleScopes( providerId: string, grantedScopes: string[], @@ -80,18 +77,12 @@ function hasRequiredGoogleScopes( return requiredScopes.every((requestedScope) => { const required = canonicalGoogleScope(requestedScope) if (granted.has(required)) return true - if ( + return ( providerId === 'google-email' && granted.has(GMAIL_MODIFY_SCOPE) && (required === GMAIL_READONLY_SCOPE || required === GMAIL_SEND_SCOPE || required === GMAIL_LABELS_SCOPE) - ) { - return true - } - /** Full Drive access implies read-only access, which is all a crawler asks for. */ - return ( - providerId === 'google-drive' && granted.has(DRIVE_SCOPE) && required === DRIVE_READONLY_SCOPE ) }) } diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index e89751ab4b1..bf6de543ba9 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -28,16 +28,6 @@ export type FeatureFlagsConfig = Record */ export type FeatureFlagContext = AppConfigGateContext -/** - * Registry of known feature flags. Each maps to the secret consulted ONLY when - * AppConfig is not the source of truth (self-hosted/OSS, local dev, or hosted - * without APPCONFIG_*). A truthy secret turns the flag on globally. - * - * Gating by workspace/org/user/admin is available ONLY through the hosted AppConfig document - * — it deliberately cannot be expressed here, so no environment can grant (e.g.) - * admin access from a code literal. To add a flag, register its name and the secret - * to fall back on. - */ /** * The single definition of a feature flag. Everything about a flag lives in one * place: its name (the registry key), a human-readable `description`, and the diff --git a/apps/sim/lib/credential-groups/credentials.ts b/apps/sim/lib/credential-groups/credentials.ts index 735959a3607..f5d84e23c75 100644 --- a/apps/sim/lib/credential-groups/credentials.ts +++ b/apps/sim/lib/credential-groups/credentials.ts @@ -68,7 +68,10 @@ export interface ManagedCredentialGroupBinding { * credential itself. */ export function isManagedCredentialGroupBindingLive( - binding: ManagedCredentialGroupBinding + binding: Pick< + ManagedCredentialGroupBinding, + 'managedOauthStatus' | 'enrollmentStatus' | 'groupStatus' | 'optionStatus' + > ): boolean { return ( binding.managedOauthStatus === 'active' && diff --git a/apps/sim/lib/credential-groups/oauth.ts b/apps/sim/lib/credential-groups/oauth.ts index 569a4626d81..489120c2d40 100644 --- a/apps/sim/lib/credential-groups/oauth.ts +++ b/apps/sim/lib/credential-groups/oauth.ts @@ -304,7 +304,7 @@ async function persistGrant( * Loaded lazily: credential groups do not otherwise depend on knowledge. */ const { dispatchMemberSyncsForCredentialOption } = await import( - '@/lib/knowledge/connectors/member-provisioning' + '@/lib/knowledge/connectors/member-queue' ) await dispatchMemberSyncsForCredentialOption({ workspaceId: context.workspaceId, diff --git a/apps/sim/lib/knowledge/access/availability.ts b/apps/sim/lib/knowledge/access/availability.ts index f3d3f228823..b46d1b0a25c 100644 --- a/apps/sim/lib/knowledge/access/availability.ts +++ b/apps/sim/lib/knowledge/access/availability.ts @@ -1,4 +1,7 @@ -import { getWorkspaceOwnerSubscriptionAccess } from '@/lib/billing/core/workspace-access' +import { + getWorkspaceOwnerSubscriptionAccess, + type WorkspaceOwnerSubscriptionAccess, +} from '@/lib/billing/core/workspace-access' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' @@ -13,6 +16,8 @@ import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availabilit export interface KnowledgeMemberAccessContext { workspaceId: string userId?: string + /** The workspace owner's plan, when the caller already holds it. */ + ownerBilling?: WorkspaceOwnerSubscriptionAccess } /** @@ -30,6 +35,7 @@ export async function isKnowledgeMemberAccessAvailable( context: KnowledgeMemberAccessContext ): Promise { if (!(await isFeatureEnabled('knowledge-member-access', context))) return false - const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(context.workspaceId) + const ownerBilling = + context.ownerBilling ?? (await getWorkspaceOwnerSubscriptionAccess(context.workspaceId)) return isCredentialGroupsAvailable({ workspaceId: context.workspaceId, ownerBilling }) } diff --git a/apps/sim/lib/knowledge/access/index.ts b/apps/sim/lib/knowledge/access/index.ts deleted file mode 100644 index df2d0aec960..00000000000 --- a/apps/sim/lib/knowledge/access/index.ts +++ /dev/null @@ -1,30 +0,0 @@ -export { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -export { - createKnowledgeAccessProvider, - type KnowledgeAccessScopeContext, - resolveKnowledgeAccessScope, - resolveUserKnowledgeAccessScope, - WORKSPACE_ACCESS_SCOPE, -} from '@/lib/knowledge/access/scope' -export { SYSTEM_ACCESS_SCOPE, type SystemAccessScope } from '@/lib/knowledge/access/system' -export { - ACCESS_TOKEN_PATTERN, - buildAclFromObservers, - EMPTY_ACL, - isAccessToken, - isWorkspaceOnlyTokenSet, - NO_TENANT_SEGMENT, - type SubjectCredential, - sortAccessTokens, - subjectToken, - WORKSPACE_ACL, -} from '@/lib/knowledge/access/tokens' -export { - type KnowledgeAccessProvider, - type KnowledgeAccessScope, - PUBLIC_ACCESS_TOKEN, - type UserAccessScope, - WORKSPACE_ACCESS_TOKEN, - WORKSPACE_ACCESS_TOKENS, - type WorkspaceAccessScope, -} from '@/lib/knowledge/access/types' diff --git a/apps/sim/lib/knowledge/access/predicate.test.ts b/apps/sim/lib/knowledge/access/predicate.test.ts index 8dd48a9bf0d..59f4bc7e8b8 100644 --- a/apps/sim/lib/knowledge/access/predicate.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.test.ts @@ -16,7 +16,7 @@ process.env.DATABASE_URL ??= 'postgresql://user:pass@localhost:5432/test' const { PgDialect } = await import('drizzle-orm/pg-core') const { knowledgeAccessCondition } = await import('@/lib/knowledge/access/predicate') -const { SYSTEM_ACCESS_SCOPE } = await import('@/lib/knowledge/access/system') +const { SYSTEM_ACCESS_SCOPE } = await import('@/lib/knowledge/access/types') function render(condition: ReturnType) { return new PgDialect().sqlToQuery(condition) diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index 9524a6ac7c4..eb9326f5bea 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -1,7 +1,6 @@ import { document } from '@sim/db/schema' import { type SQL, sql } from 'drizzle-orm' -import type { SystemAccessScope } from '@/lib/knowledge/access/system' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' /** * The single read-side access predicate: the document's ACL overlaps the diff --git a/apps/sim/lib/knowledge/access/system.ts b/apps/sim/lib/knowledge/access/system.ts deleted file mode 100644 index 566162ca591..00000000000 --- a/apps/sim/lib/knowledge/access/system.ts +++ /dev/null @@ -1,16 +0,0 @@ -declare const systemAccessScopeBrand: unique symbol - -/** - * The one exemption from access filtering: a background job acting on rows it - * owns (document processing, connector sync). It is a branded type so it cannot - * be assembled from a literal, and this module is its only source, so every - * caller is one grep away. Never construct it on a request path. - */ -export interface SystemAccessScope { - readonly kind: 'system' - readonly [systemAccessScopeBrand]: true -} - -export const SYSTEM_ACCESS_SCOPE: SystemAccessScope = Object.freeze({ - kind: 'system', -}) as SystemAccessScope diff --git a/apps/sim/lib/knowledge/access/tokens.test.ts b/apps/sim/lib/knowledge/access/tokens.test.ts index 282710b1a69..de56395ad7d 100644 --- a/apps/sim/lib/knowledge/access/tokens.test.ts +++ b/apps/sim/lib/knowledge/access/tokens.test.ts @@ -4,9 +4,7 @@ import { describe, expect, it } from 'vitest' import { ACCESS_TOKEN_PATTERN, - buildAclFromObservers, isAccessToken, - isWorkspaceOnlyTokenSet, sortAccessTokens, subjectToken, } from '@/lib/knowledge/access/tokens' @@ -81,7 +79,7 @@ describe('subjectToken', () => { }) }) -describe('sortAccessTokens / buildAclFromObservers', () => { +describe('sortAccessTokens', () => { it('sorts by code unit and dedupes', () => { expect(sortAccessTokens(['ws', 'pub', 's:b:-:1', 'pub', 's:B:-:1'])).toEqual([ 'pub', @@ -94,20 +92,4 @@ describe('sortAccessTokens / buildAclFromObservers', () => { it('never uses locale ordering', () => { expect(sortAccessTokens(['s:x:-:b', 's:x:-:B'])).toEqual(['s:x:-:B', 's:x:-:b']) }) - - it('builds an empty ACL from no observers and rejects malformed tokens', () => { - expect(buildAclFromObservers([])).toEqual([]) - expect(buildAclFromObservers(['s:confluence:-:2', 's:confluence:-:1'])).toEqual([ - 's:confluence:-:1', - 's:confluence:-:2', - ]) - expect(() => buildAclFromObservers(['s:confluence:-:1', 'bogus'])).toThrow('malformed') - }) -}) - -describe('isWorkspaceOnlyTokenSet', () => { - it('distinguishes the workspace pair from a personal set', () => { - expect(isWorkspaceOnlyTokenSet(['pub', 'ws'])).toBe(true) - expect(isWorkspaceOnlyTokenSet(['pub', 's:confluence:-:1', 'ws'])).toBe(false) - }) }) diff --git a/apps/sim/lib/knowledge/access/tokens.ts b/apps/sim/lib/knowledge/access/tokens.ts index bb6b5d0c9ab..dc2841edc58 100644 --- a/apps/sim/lib/knowledge/access/tokens.ts +++ b/apps/sim/lib/knowledge/access/tokens.ts @@ -66,15 +66,3 @@ export function sortAccessTokens(tokens: Iterable): string[] { * observers. Rejects anything that is not a well-formed token so a malformed * value fails here rather than denying access silently. */ -export function buildAclFromObservers(tokens: Iterable): string[] { - const sorted = sortAccessTokens(tokens) - for (const token of sorted) { - if (!isAccessToken(token)) throw new Error(`Access token is malformed: ${token}`) - } - return sorted -} - -/** Whether a scope's token set grants nothing beyond what every workspace member holds. */ -export function isWorkspaceOnlyTokenSet(tokens: readonly string[]): boolean { - return tokens.every((token) => token === WORKSPACE_ACCESS_TOKEN || token === PUBLIC_ACCESS_TOKEN) -} diff --git a/apps/sim/lib/knowledge/access/types.ts b/apps/sim/lib/knowledge/access/types.ts index e44b8bfdcbb..2d3346d6b91 100644 --- a/apps/sim/lib/knowledge/access/types.ts +++ b/apps/sim/lib/knowledge/access/types.ts @@ -38,3 +38,20 @@ export type KnowledgeAccessScope = WorkspaceAccessScope | UserAccessScope export interface KnowledgeAccessProvider { get(): Promise } + +declare const systemAccessScopeBrand: unique symbol + +/** + * The one exemption from access filtering: a background job acting on rows it + * owns (document processing, connector sync). It is a branded type so it cannot + * be assembled from a literal, and this module is its only source, so every + * caller is one grep away. Never construct it on a request path. + */ +export interface SystemAccessScope { + readonly kind: 'system' + readonly [systemAccessScopeBrand]: true +} + +export const SYSTEM_ACCESS_SCOPE: SystemAccessScope = Object.freeze({ + kind: 'system', +}) as SystemAccessScope diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index 10ee7defc95..5106eb13f14 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -16,6 +16,7 @@ import { } from '@/lib/knowledge/application/connectors' import { resolveActiveKnowledgeConnectorContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { createViewerConnectorEnrollmentLink } from '@/lib/knowledge/connectors/member-provisioning' import { performUpdateKnowledgeConnectorAccess, resolveKnowledgeConnectorMembersBinding, @@ -24,11 +25,6 @@ import { getKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' import type { KnowledgeOperationSource } from '@/lib/knowledge/orchestration/shared' import { getConnectorMeta } from '@/connectors/registry' -/** The enrollment link needs the credential-group services; loaded only when a member connects. */ -async function loadMemberProvisioning() { - return import('@/lib/knowledge/connectors/member-provisioning') -} - export interface StartKnowledgeConnectorMemberEnrollmentInput { knowledgeBaseId: string connectorId: string @@ -64,7 +60,7 @@ export const startKnowledgeConnectorMemberEnrollment = defineAuthorizedKnowledge 'Per-member access is not available for this workspace' ) } - const url = await (await loadMemberProvisioning()).createViewerConnectorEnrollmentLink({ + const url = await createViewerConnectorEnrollmentLink({ userId, workspaceId, credentialGroupId: connector.credentialGroupId, @@ -183,8 +179,8 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ /** * Workspace mode needs a credential the caller may use, and one that yields a - * token, since the connector syncs as it from then on. An API-key connector - * has no credential to name and keeps its stored key. + * token, since the connector syncs as it from then on. Only an OAuth connector + * can change modes: an API-key connector has no account to sync per member. */ async function requireUsableCredential(input: { credentialId: string | undefined diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 6cda94ad6a5..f6c5a4287ca 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -34,13 +34,17 @@ import { resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { resolveViewerConnectorMemberships } from '@/lib/knowledge/connectors/member-provisioning' import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' import { DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, } from '@/lib/knowledge/constants' -import { resolveKnowledgeConnectorMembersBinding } from '@/lib/knowledge/orchestration/connector-access' +import { + type ResolvedMembersBinding, + resolveKnowledgeConnectorMembersBinding, +} from '@/lib/knowledge/orchestration/connector-access' import { getKnowledgeConnector, type KnowledgeConnectorRow, @@ -65,11 +69,6 @@ interface KnowledgeConnectorApplicationInput { source?: KnowledgeOperationSource } -/** The viewer's standing with per-member connectors; loaded only when the list holds one. */ -async function loadMemberProvisioning() { - return import('@/lib/knowledge/connectors/member-provisioning') -} - export interface ListKnowledgeConnectorsInput extends KnowledgeConnectorApplicationInput { knowledgeBaseId: string sortBy?: 'connectorType' | 'createdAt' | 'updatedAt' @@ -330,7 +329,7 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ const viewerUserId = resolvePrincipalSubjectUserId(principal) const memberships = viewerUserId && context.workspaceId - ? await (await loadMemberProvisioning()).resolveViewerConnectorMemberships({ + ? await resolveViewerConnectorMemberships({ userId: viewerUserId, workspaceId: context.workspaceId, connectors: page, @@ -388,7 +387,7 @@ export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ ) ) .orderBy(asc(knowledgeBase.name), asc(knowledgeConnector.createdAt)) - const memberships = await (await loadMemberProvisioning()).resolveViewerConnectorMemberships({ + const memberships = await resolveViewerConnectorMemberships({ userId: viewerUserId, workspaceId: context.workspaceId, connectors: rows, @@ -447,7 +446,7 @@ export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ const viewerUserId = resolvePrincipalSubjectUserId(principal) const memberships = viewerUserId && context.workspaceId - ? await (await loadMemberProvisioning()).resolveViewerConnectorMemberships({ + ? await resolveViewerConnectorMemberships({ userId: viewerUserId, workspaceId: context.workspaceId, connectors: [connector], @@ -512,7 +511,7 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ workspaceId, input.connectorType ) - let membersBinding: { credentialGroupId: string; credentialGroupOptionId: string } | undefined + let membersBinding: ResolvedMembersBinding | undefined if (input.accessMode === 'members') { /** * Members mode grants the connector every enrolled member's credential, @@ -555,7 +554,8 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ connectorType: input.connectorType, credentialId: input.credentialId, apiKey: input.apiKey, - sourceConfig: input.sourceConfig, + /** Members mode stores the config with its listing caps cleared. */ + sourceConfig: membersBinding?.sourceConfig ?? input.sourceConfig, syncIntervalMinutes: input.syncIntervalMinutes, membersBinding, resolveBillingAttribution: () => diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index 88bdb8a0e33..eccb44423e4 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -398,10 +398,6 @@ export const knowledgeOperations = { capability: 'knowledge.use', principalKinds: ['session'], }), - /** - * A workspace member joining a per-member connector: any reader may connect - * their own account, which only ever widens what they themselves see. - */ /** Every per-member connector in the workspace, with where the viewer stands on each. */ listWorkspaceMemberConnectors: defineWorkspaceOperation({ id: 'knowledge.connectors.members.list', @@ -410,6 +406,10 @@ export const knowledgeOperations = { capability: 'knowledge.use', principalKinds: ['session'], }), + /** + * A workspace member joining a per-member connector: any reader may connect + * their own account, which only ever widens what they themselves see. + */ enrollConnectorMember: defineWorkspaceOperation({ id: 'knowledge.connectors.members.enroll', minimumRole: 'read', diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index cc953e3d9f1..2b39c26da7f 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -35,13 +35,13 @@ import { import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models' +import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' import { rerank } from '@/lib/knowledge/reranker' import type { RerankerStatus } from '@/lib/knowledge/reranker-models' import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' import { executeKnowledgeSearch, - generateSearchEmbedding, getDocumentMetadataByIds, type SearchResult, } from '@/lib/knowledge/search/queries' diff --git a/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts b/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts index 36975a6b6c9..2dfacc51ff2 100644 --- a/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts +++ b/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts @@ -10,7 +10,7 @@ vi.unmock('@sim/db/schema') vi.unmock('drizzle-orm') import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' -import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/system' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { queryChunks } from '@/lib/knowledge/chunks/service' import type { ChunkSortBy } from '@/lib/knowledge/chunks/types' diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index d70b8a01dae..8684c847259 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -192,22 +192,6 @@ export async function materializeDocumentAcls( return updated } -/** Rewrites every drifted document ACL of the connector; used when membership changes wholesale. */ -export async function materializeAllDocumentAcls(connectorId: string): Promise { - const rows = await db - .update(document) - .set({ acl: observedAcl() }) - .where( - and( - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - sql`${document.acl} IS DISTINCT FROM ${observedAcl()}` - ) - ) - .returning({ id: document.id }) - return rows.length -} - export interface MemberDocumentLifecycleResult { tombstoned: number resurrected: number diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts index e013e87b6e4..a7aa1891de9 100644 --- a/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts @@ -21,6 +21,7 @@ vi.mock('@/lib/credential-groups/service', () => ({ vi.mock('@/lib/workspaces/permissions/utils', () => ({ getUsersWithPermissions: vi.fn() })) import { + chooseSharedMembersBinding, deriveViewerConnectorMembership, pickProvisionedGroupName, } from '@/lib/knowledge/connectors/member-provisioning' @@ -46,6 +47,24 @@ describe('pickProvisionedGroupName', () => { }) }) +describe('chooseSharedMembersBinding', () => { + const a = { credentialGroupId: 'g1', credentialGroupOptionId: 'o1' } + const b = { credentialGroupId: 'g2', credentialGroupOptionId: 'o2' } + + it('reuses the option other members-mode connectors sync through', () => { + expect(chooseSharedMembersBinding([a, b], new Set(['o2']))).toBe(b) + }) + + it('creates a new group rather than repurpose one nobody syncs through', () => { + expect(chooseSharedMembersBinding([a, b], new Set())).toBeUndefined() + expect(chooseSharedMembersBinding([], new Set())).toBeUndefined() + }) + + it('leaves two shared options for the caller to choose between', () => { + expect(chooseSharedMembersBinding([a, b], new Set(['o1', 'o2']))).toBeNull() + }) +}) + describe('deriveViewerConnectorMembership', () => { it.each([ [true, 'active', 'completed', 'connected'], diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.ts index dfcdce546d5..2ce0d531d79 100644 --- a/apps/sim/lib/knowledge/connectors/member-provisioning.ts +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.ts @@ -8,8 +8,8 @@ import { } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { normalizeEmail } from '@sim/utils/string' import { and, eq, inArray, isNull } from 'drizzle-orm' -import { resolveSystemBillingAttribution } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { createCredentialGroupInvitationLink, @@ -22,7 +22,6 @@ import { } from '@/lib/credential-groups/providers' import { createCredentialGroup, listCredentialGroups } from '@/lib/credential-groups/service' import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' -import { dispatchMemberSync } from '@/lib/knowledge/connectors/member-queue' import { getUsersWithPermissions } from '@/lib/workspaces/permissions/utils' import type { ConnectorMeta } from '@/connectors/types' @@ -58,11 +57,50 @@ export function pickProvisionedGroupName( ) } +/** + * Among the workspace's active options collecting the connector's accounts, + * the one other members-mode connectors already sync through, so one + * connection serves every connector of a provider. A group nobody syncs + * through is never reused: it was curated for something else, and joining + * it would invite the whole workspace to it. Returns `undefined` when a new + * group is needed and `null` when two shared options make the choice + * ambiguous. + */ +export function chooseSharedMembersBinding( + candidates: readonly ProvisionedMembersBinding[], + optionIdsServingMemberConnectors: ReadonlySet +): ProvisionedMembersBinding | null | undefined { + const shared = candidates.filter((candidate) => + optionIdsServingMemberConnectors.has(candidate.credentialGroupOptionId) + ) + if (shared.length === 1) return shared[0] + return shared.length > 1 ? null : undefined +} + +async function listOptionIdsServingMemberConnectors( + workspaceId: string, + optionIds: readonly string[] +): Promise> { + if (optionIds.length === 0) return new Set() + const rows = await db + .select({ optionId: knowledgeConnector.credentialGroupOptionId }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.credentialGroupOptionId, [...optionIds]), + isNull(knowledgeConnector.deletedAt) + ) + ) + return new Set(rows.flatMap((row) => (row.optionId ? [row.optionId] : []))) +} + /** * The Credential Group option a members-mode connector crawls through when - * the caller named none: the workspace's one active option collecting the - * connector's accounts, or a group created for the purpose. Two or more - * candidate options is an ambiguity the caller has to resolve by naming one. + * the caller named none: the option this provider's other members-mode + * connectors share, or a group created for the purpose. */ export async function provisionKnowledgeConnectorMembersBinding(input: { workspaceId: string @@ -95,11 +133,18 @@ export async function provisionKnowledgeConnectorMembersBinding(input: { candidates.push({ credentialGroupId: group.id, credentialGroupOptionId: option.id }) } } - if (candidates.length === 1) return candidates[0] - if (candidates.length > 1) { + const shared = chooseSharedMembersBinding( + candidates, + await listOptionIdsServingMemberConnectors( + input.workspaceId, + candidates.map((candidate) => candidate.credentialGroupOptionId) + ) + ) + if (shared) return shared + if (shared === null) { throw new OrchestrationError( 'validation', - `Several Credential Groups collect ${connectorMeta.name} accounts; choose which one this connector syncs through` + `Several Credential Groups collect ${connectorMeta.name} accounts for other connectors; choose which one this connector syncs through` ) } @@ -145,10 +190,10 @@ export async function inviteWorkspaceMembersToCredentialGroup(input: { .from(credentialGroupEnrollment) .where(eq(credentialGroupEnrollment.credentialGroupId, input.credentialGroupId)), ]) - const enrolledEmails = new Set(enrolled.map((row) => row.email.trim().toLocaleLowerCase())) - const pending = [ - ...new Set(members.map((member) => member.email.trim().toLocaleLowerCase())), - ].filter((email) => email && !enrolledEmails.has(email)) + const enrolledEmails = new Set(enrolled.map((row) => normalizeEmail(row.email))) + const pending = [...new Set(members.map((member) => normalizeEmail(member.email)))].filter( + (email) => email && !enrolledEmails.has(email) + ) const result: InviteWorkspaceMembersResult = { invited: 0, failed: 0 } for (let offset = 0; offset < pending.length; offset += INVITATION_BATCH_SIZE) { @@ -232,7 +277,7 @@ export async function resolveViewerConnectorMemberships(input: { .where(eq(user.id, input.userId)) .limit(1) if (!viewer) return result - const email = viewer.email.trim().toLocaleLowerCase() + const email = normalizeEmail(viewer.email) const groupIds = [...new Set(memberConnectors.map((connector) => connector.credentialGroupId!))] const rows = await db .select({ @@ -300,7 +345,7 @@ export async function createViewerConnectorEnrollmentLink(input: { 'Verify your email address before connecting an account' ) } - const email = viewer.email.trim().toLocaleLowerCase() + const email = normalizeEmail(viewer.email) const [enrollment] = await db .select({ status: credentialGroupEnrollment.status }) .from(credentialGroupEnrollment) @@ -325,45 +370,3 @@ export async function createViewerConnectorEnrollmentLink(input: { ) return invitationLink } - -/** - * Queues a member run for every connector that crawls through the option a - * member just connected, so their documents arrive within minutes rather - * than at the next scheduled run. Best effort: a refused dispatch is logged - * and the schedule catches up. - */ -export async function dispatchMemberSyncsForCredentialOption(input: { - workspaceId: string - credentialGroupOptionId: string - requestId?: string -}): Promise { - const connectors = await db - .select({ id: knowledgeConnector.id }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where( - and( - eq(knowledgeBase.workspaceId, input.workspaceId), - isNull(knowledgeBase.deletedAt), - eq(knowledgeConnector.accessMode, 'members'), - eq(knowledgeConnector.credentialGroupOptionId, input.credentialGroupOptionId), - eq(knowledgeConnector.status, 'active'), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - if (connectors.length === 0) return - const billingAttribution = await resolveSystemBillingAttribution(input.workspaceId) - for (const connector of connectors) { - const dispatch = await dispatchMemberSync(connector.id, { - billingAttribution, - requestId: input.requestId, - }) - if (!dispatch.queued) { - logger.info('Member sync after a member connected was not queued', { - connectorId: connector.id, - reason: dispatch.reason, - }) - } - } -} diff --git a/apps/sim/lib/knowledge/connectors/member-queue.ts b/apps/sim/lib/knowledge/connectors/member-queue.ts index 972ea2fa158..4159ad498b3 100644 --- a/apps/sim/lib/knowledge/connectors/member-queue.ts +++ b/apps/sim/lib/knowledge/connectors/member-queue.ts @@ -9,6 +9,7 @@ import { and, eq, inArray, isNull } from 'drizzle-orm' import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, + resolveSystemBillingAttribution, } from '@/lib/billing/core/billing-attribution' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' @@ -318,3 +319,41 @@ export async function dispatchMemberSync( }) return { queued: true } } + +/** + * Queues a member run for every connector that crawls through the option a + * member just connected, so their documents arrive within minutes rather + * than at the next scheduled run. Best effort: a refused dispatch is logged + * and the schedule catches up. + */ +export async function dispatchMemberSyncsForCredentialOption(input: { + workspaceId: string + credentialGroupOptionId: string +}): Promise { + const connectors = await db + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, input.workspaceId), + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.accessMode, 'members'), + eq(knowledgeConnector.credentialGroupOptionId, input.credentialGroupOptionId), + inArray(knowledgeConnector.status, ['active', 'error']), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + if (connectors.length === 0) return + const billingAttribution = await resolveSystemBillingAttribution(input.workspaceId) + for (const connector of connectors) { + const dispatch = await dispatchMemberSync(connector.id, { billingAttribution }) + if (!dispatch.queued) { + logger.info('Member sync after a member connected was not queued', { + connectorId: connector.id, + reason: dispatch.reason, + }) + } + } +} diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 28281ed2c27..906ef87fc87 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -17,6 +17,7 @@ import { } from '@/lib/billing/core/billing-attribution' import { type CredentialGroupOptionCredentialReference, + isManagedCredentialGroupBindingLive, loadCredentialGroupCredentialListContext, } from '@/lib/credential-groups/credentials' import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' @@ -204,12 +205,12 @@ export function deriveMemberActive( >, option: { groupActive: boolean; optionActive: boolean } ): boolean { - return ( - option.groupActive && - option.optionActive && - credential.managedOauthStatus === 'active' && - (credential.enrollmentStatus === 'in_progress' || credential.enrollmentStatus === 'completed') - ) + return isManagedCredentialGroupBindingLive({ + managedOauthStatus: credential.managedOauthStatus, + enrollmentStatus: credential.enrollmentStatus, + groupStatus: option.groupActive ? 'active' : 'disabled', + optionStatus: option.optionActive ? 'active' : 'disabled', + }) } /** @@ -909,12 +910,7 @@ async function applyMemberListing( ...(outcome.mode === 'changes' && outcome.complete ? { memberSyncedThrough: outcome.listingStartedAt } : {}), - ...(outcome.changeCursor !== undefined - ? { - changeCursor: outcome.changeCursor, - changeCursorAt: outcome.changeCursor === null ? null : now, - } - : {}), + ...(outcome.changeCursor !== undefined ? { changeCursor: outcome.changeCursor } : {}), updatedAt: now, }) .where(eq(knowledgeConnectorMember.id, outcome.member.id)) @@ -1068,8 +1064,10 @@ async function failMemberSyncLog(runId: string, result: MemberSyncResult, errorM /** * Ends a run without doing anything because the feature is not available to * the workspace right now. The connector keeps its members and their - * observations, records nothing as a failure, and is looked at again on its - * next schedule; a manual-only connector waits for the next manual sync. + * observations, and its failure ladder does not advance; the reason is left + * on the connector and the run's log so an admin can see why nothing syncs. + * It is looked at again on its next schedule; a manual-only connector waits + * for the next manual sync. */ async function deferMemberSync(run: MemberSyncRun, syncIntervalMinutes: number): Promise { const now = new Date() diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts index 1520b443b76..9df3472ac6d 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts @@ -21,7 +21,7 @@ vi.mock('@/lib/knowledge/documents/processing-claim', () => ({ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { OutboxEventContext } from '@/lib/core/outbox/service' -import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/system' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index 5d58347fe7a..220f932f7db 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -1,6 +1,6 @@ import { assertBillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { OutboxHandler, OutboxHandlerRegistry } from '@/lib/core/outbox/service' -import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/system' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { reclaimStaleDocumentProcessingClaim } from '@/lib/knowledge/documents/processing-claim' import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index d6e9140c18b..4995f4bb382 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -77,8 +77,7 @@ import { mergeDurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import type { SystemAccessScope } from '@/lib/knowledge/access/system' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' import { assertDocumentChunkCountWithinLimit, isPermanentDocumentProcessingError, diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts index 4a4f7ba310b..1f0052fa6f5 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts @@ -148,7 +148,6 @@ describe('resolveKnowledgeConnectorMembersBinding', () => { ).resolves.toEqual({ credentialGroupId: 'group-9', credentialGroupOptionId: 'option-9', - workspaceId: 'ws-1', sourceConfig: {}, }) expect(mocks.provision).toHaveBeenCalledWith({ diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index 21e658698ce..2be3ade31fe 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -16,6 +16,7 @@ import { stripListingCapFields, validateKnowledgeConnectorMembersBinding, } from '@/lib/knowledge/connectors/member-access' +import { provisionKnowledgeConnectorMembersBinding } from '@/lib/knowledge/connectors/member-provisioning' import { type ConnectorWithoutSecret, getKnowledgeConnector, @@ -61,7 +62,6 @@ export interface KnowledgeConnectorMembersBinding { } export interface ResolvedMembersBinding extends KnowledgeConnectorMembersBinding { - workspaceId: string /** The connector's source config with the listing caps cleared, which members mode stores. */ sourceConfig: Record } @@ -99,9 +99,7 @@ export async function resolveKnowledgeConnectorMembersBinding(input: { const sourceConfig = stripListingCapFields(input.connectorMeta, input.sourceConfig) const binding = input.binding ?? - (await ( - await import('@/lib/knowledge/connectors/member-provisioning') - ).provisionKnowledgeConnectorMembersBinding({ + (await provisionKnowledgeConnectorMembersBinding({ workspaceId: input.workspaceId, connectorMeta: input.connectorMeta, userId: input.actingUserId, @@ -117,7 +115,7 @@ export async function resolveKnowledgeConnectorMembersBinding(input: { sourceConfig, }) if (!validation.ok) throw new OrchestrationError('validation', validation.message) - return { ...binding, workspaceId: input.workspaceId, sourceConfig } + return { ...binding, sourceConfig } } /** diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index c5b4fb894de..e10fd8cd837 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -304,7 +304,12 @@ export async function performCreateKnowledgeConnector( if (membersBinding) { try { await grantKnowledgeConnectorCredentialAccess( - { workspaceId, ...membersBinding, connectorId }, + { + workspaceId, + credentialGroupId: membersBinding.credentialGroupId, + credentialGroupOptionId: membersBinding.credentialGroupOptionId, + connectorId, + }, params.userId ) } catch (error) { diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index e473e4b6d84..60fbfd8700c 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -5,8 +5,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { and, eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' -import { RRF_K } from '@/lib/knowledge/search/rank' -import { applyRecencyBoost } from '@/lib/knowledge/search/recency' +import { applyRecencyBoost, RRF_K } from '@/lib/knowledge/search/recency' import { coerceTagFilterValue, escapeLikePattern, @@ -145,8 +144,6 @@ export interface SearchParams { distanceThreshold?: number } -export { generateSearchEmbedding } from '@/lib/knowledge/embeddings' - /** All valid tag slot keys */ const TAG_SLOT_KEYS = [ // Text tags (7 slots) diff --git a/apps/sim/lib/knowledge/search/rank.ts b/apps/sim/lib/knowledge/search/rank.ts deleted file mode 100644 index 500316b6992..00000000000 --- a/apps/sim/lib/knowledge/search/rank.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Reciprocal-rank-fusion damping constant, shared by fusion and the recency - * boost so a rank means the same to both: `score = 1 / (RRF_K + rank)`. 60 is - * the value from the original RRF paper and matches the docs search retriever - * (`apps/docs/app/api/search/route.ts`). - */ -export const RRF_K = 60 diff --git a/apps/sim/lib/knowledge/search/recency.ts b/apps/sim/lib/knowledge/search/recency.ts index f3d53c13dab..0fa7d515cc1 100644 --- a/apps/sim/lib/knowledge/search/recency.ts +++ b/apps/sim/lib/knowledge/search/recency.ts @@ -1,4 +1,10 @@ -import { RRF_K } from '@/lib/knowledge/search/rank' +/** + * Reciprocal-rank-fusion damping constant, shared by fusion and the recency + * boost so a rank means the same to both: `score = 1 / (RRF_K + rank)`. 60 is + * the value from the original RRF paper and matches the docs search retriever + * (`apps/docs/app/api/search/route.ts`). + */ +export const RRF_K = 60 /** Age at which a document's recency boost has decayed to half. */ export const RECENCY_HALF_LIFE_DAYS = 90 diff --git a/apps/sim/lib/workspaces/host-context.ts b/apps/sim/lib/workspaces/host-context.ts index 5798ef79432..78cd140507f 100644 --- a/apps/sim/lib/workspaces/host-context.ts +++ b/apps/sim/lib/workspaces/host-context.ts @@ -31,7 +31,7 @@ async function resolveWorkspaceHostContextForViewer( ]) const [credentialGroupsAvailable, knowledgeMemberAccessAvailable] = await Promise.all([ isCredentialGroupsAvailable({ workspaceId, ownerBilling }), - isKnowledgeMemberAccessAvailable({ workspaceId }), + isKnowledgeMemberAccessAvailable({ workspaceId, ownerBilling }), ]) return { diff --git a/packages/db/credential-group-resource-policies.ts b/packages/db/credential-group-resource-policies.ts index a3c9a67d0b7..4b814b62929 100644 --- a/packages/db/credential-group-resource-policies.ts +++ b/packages/db/credential-group-resource-policies.ts @@ -6,6 +6,8 @@ export const CREDENTIAL_GROUP_POLICY_DOCUMENT_MAX_BYTES = 32 * 1024 const ACTOR_ACCESS_SID = 'CredentialGroupActorCredentialAccess' const WORKFLOW_ACCESS_SID = 'WorkflowCredentialAccess' +/** Statements the knowledge module writes for connectors crawling through an option. */ +const KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX = 'KnowledgeConnectorCredentialAccess:' const CREDENTIAL_USE_ACTION = 'credential_groups.credentials.use' const ACTOR_OWNS_CREDENTIAL_CONDITION_KEY = 'credential_group:ActorOwnsCredential' const DEPLOYMENT_MODE_CONDITION_KEY = 'execution:WorkflowMode' @@ -34,6 +36,16 @@ interface CredentialGroupWorkflowAccessStatement { } } +/** + * A statement the knowledge module writes so one of its connectors can crawl + * through a group option. The knowledge module owns and validates its shape; + * this package only carries it through unchanged. + */ +interface CredentialGroupKnowledgeConnectorAccessStatement { + sid: `${typeof KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX}${string}` + [key: string]: unknown +} + export interface CredentialGroupWorkflowAccessPolicyDocument { version: 1 resource: { @@ -41,8 +53,12 @@ export interface CredentialGroupWorkflowAccessPolicyDocument { id: string } statements: - | [CredentialGroupActorAccessStatement] - | [CredentialGroupActorAccessStatement, CredentialGroupWorkflowAccessStatement] + | [CredentialGroupActorAccessStatement, ...CredentialGroupKnowledgeConnectorAccessStatement[]] + | [ + CredentialGroupActorAccessStatement, + CredentialGroupWorkflowAccessStatement, + ...CredentialGroupKnowledgeConnectorAccessStatement[], + ] } export interface MissingCredentialGroupPolicyRow { @@ -144,6 +160,15 @@ export function createDefaultCredentialGroupPolicyDocument( } } +function isKnowledgeConnectorAccessStatement( + statement: Record +): statement is CredentialGroupKnowledgeConnectorAccessStatement { + return ( + typeof statement.sid === 'string' && + statement.sid.startsWith(KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX) + ) +} + export function parseCredentialGroupPolicyDocument( value: unknown, expectedResourceId: string @@ -166,17 +191,29 @@ export function parseCredentialGroupPolicyDocument( throw new Error('Credential Group policy resource does not match its canonical resource') } - if ( - !Array.isArray(document.statements) || - document.statements.length < 1 || - document.statements.length > 2 - ) { + if (!Array.isArray(document.statements) || document.statements.length < 1) { + throw new Error('Credential Group policy must contain its actor statement') + } + /** + * Knowledge connectors that crawl through this group's options carry their + * own statements after the actor and workflow ones. They are owned by the + * knowledge module, so this canonicaliser passes them through untouched + * rather than rewriting or dropping them. + */ + const knowledgeStatements: CredentialGroupKnowledgeConnectorAccessStatement[] = [] + const ownStatements: unknown[] = [] + for (const statement of document.statements) { + const record = requireRecord(statement, 'Credential Group statement') + if (isKnowledgeConnectorAccessStatement(record)) knowledgeStatements.push(record) + else ownStatements.push(statement) + } + if (ownStatements.length > 2) { throw new Error( 'Credential Group policy must contain its actor statement and optional workflow statement' ) } - const actorStatement = requireRecord(document.statements[0], 'Credential Group actor statement') + const actorStatement = requireRecord(ownStatements[0], 'Credential Group actor statement') requireExactKeys( actorStatement, ['sid', 'effect', 'actions', 'principals', 'condition'], @@ -219,15 +256,15 @@ export function parseCredentialGroupPolicyDocument( } const canonicalActorStatement = createCredentialGroupActorAccessStatement() - if (document.statements.length === 1) { + if (ownStatements.length === 1) { return { version: 1, resource: { type: 'credential_group', id: canonicalResourceId }, - statements: [canonicalActorStatement], + statements: [canonicalActorStatement, ...knowledgeStatements], } } - const statement = requireRecord(document.statements[1], 'Credential Group workflow statement') + const statement = requireRecord(ownStatements[1], 'Credential Group workflow statement') requireExactKeys( statement, ['sid', 'effect', 'actions', 'principals', 'condition'], @@ -311,6 +348,7 @@ export function parseCredentialGroupPolicyDocument( }, }, }, + ...knowledgeStatements, ], } } diff --git a/packages/db/migrations/0318_permission_aware_knowledge.sql b/packages/db/migrations/0318_permission_aware_knowledge.sql index 3fafe80e158..64c258333fc 100644 --- a/packages/db/migrations/0318_permission_aware_knowledge.sql +++ b/packages/db/migrations/0318_permission_aware_knowledge.sql @@ -42,7 +42,6 @@ CREATE TABLE IF NOT EXISTS "knowledge_connector_member" ( "last_error" text, "member_synced_through" timestamp, "change_cursor" text, - "change_cursor_at" timestamp, "suspended_at" timestamp, "created_at" timestamp DEFAULT now() NOT NULL, "updated_at" timestamp DEFAULT now() NOT NULL, diff --git a/packages/db/migrations/meta/0318_snapshot.json b/packages/db/migrations/meta/0318_snapshot.json index 65908787b04..e3ec6a6cd7b 100644 --- a/packages/db/migrations/meta/0318_snapshot.json +++ b/packages/db/migrations/meta/0318_snapshot.json @@ -8257,12 +8257,6 @@ "primaryKey": false, "notNull": false }, - "change_cursor_at": { - "name": "change_cursor_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false - }, "suspended_at": { "name": "suspended_at", "type": "timestamp", diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 40876ca6ce4..4ab1f0d953d 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -4811,7 +4811,6 @@ export const knowledgeConnectorMember = pgTable( * has to be reopened. */ changeCursor: text('change_cursor'), - changeCursorAt: timestamp('change_cursor_at'), suspendedAt: timestamp('suspended_at'), createdAt: timestamp('created_at').notNull().defaultNow(), updatedAt: timestamp('updated_at').notNull().defaultNow(), diff --git a/packages/testing/src/mocks/schema.mock.ts b/packages/testing/src/mocks/schema.mock.ts index 6f2557b5362..d757b43cb4a 100644 --- a/packages/testing/src/mocks/schema.mock.ts +++ b/packages/testing/src/mocks/schema.mock.ts @@ -1426,7 +1426,6 @@ export const schemaMock = { lastError: 'knowledgeConnectorMember.lastError', memberSyncedThrough: 'knowledgeConnectorMember.memberSyncedThrough', changeCursor: 'knowledgeConnectorMember.changeCursor', - changeCursorAt: 'knowledgeConnectorMember.changeCursorAt', suspendedAt: 'knowledgeConnectorMember.suspendedAt', createdAt: 'knowledgeConnectorMember.createdAt', updatedAt: 'knowledgeConnectorMember.updatedAt', From 1610baf396bb93c767756c504662c9171ea96925 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 22:14:06 -0700 Subject: [PATCH 23/76] fix(knowledge): repair CI and apply cleanup passes - Bump the chart version for the member-sync cron and secret - Pass an access scope to getDocuments in the list-convention test and resolve a knowledge scope only for knowledge-base file reads - Drop memoisation nothing observes, read the enrollment error from the mutation, hoist the empty connector list, use the default Cancel variant, let Badge own its gap, and keep the sidebar lit on the Search tab --- .../connectors-section/connectors-section.tsx | 2 +- .../edit-connector-modal.tsx | 17 ++--- .../member-connectors-section.tsx | 3 +- .../w/components/sidebar/sidebar.tsx | 9 ++- apps/sim/hooks/use-member-enrollment.ts | 63 ++++++++----------- apps/sim/lib/api/list-convention.test.ts | 4 +- .../payloads/materialization.server.ts | 7 ++- helm/sim/Chart.yaml | 2 +- 8 files changed, 50 insertions(+), 57 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index 5bb7b73f09b..8e1627c0fe2 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -444,7 +444,7 @@ function ConnectorCard({ {syncsPerMember && ( - + Per member diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index 2a8355abed0..57d5d02db74 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -242,8 +242,7 @@ export function EditConnectorModal({ const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling) - const persistedAccess = useMemo(() => currentAccess(connector), [connector]) - const accessDirty = accessChanged(persistedAccess, access) + const accessDirty = accessChanged(currentAccess(connector), access) const groupOptions = useConnectorMemberGroupOptions({ workspaceId, connectorConfig, @@ -260,14 +259,10 @@ export function EditConnectorModal({ /** A disabled member sync is re-enabled by applying the current binding again. */ const canReenableMemberSync = !accessDirty && connector.accessMode === 'members' && connector.memberSyncStatus === 'disabled' - const memberCapFieldIds = useMemo( - () => - new Set( - access.accessMode === 'members' - ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) - : [] - ), - [access.accessMode, connectorConfig] + const memberCapFieldIds = new Set( + access.accessMode === 'members' + ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) + : [] ) const persistedCanonicalModes = useMemo( @@ -590,7 +585,7 @@ function SettingsTab({ ? 'Switch to per-member access' : 'Switch to workspace access'} -
diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx index 3e405779ecb..7dcdca4a9be 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx @@ -15,6 +15,7 @@ import { import { useMemberEnrollment } from '@/hooks/use-member-enrollment' const SHARED_WITH_YOU_LABEL = 'Shared with you' +const EMPTY_CONNECTORS: WorkspaceMemberConnector[] = [] /** Memberships the viewer can act on themselves. */ const CONNECTABLE: ReadonlySet = new Set([ @@ -54,7 +55,7 @@ interface MemberConnectorsSectionProps { * page offers, so a person can do it from whichever surface they are on. */ export function MemberConnectorsSection({ workspaceId, search }: MemberConnectorsSectionProps) { - const { data: connectors = [] } = useWorkspaceMemberConnectors(workspaceId) + const { data: connectors = EMPTY_CONNECTORS } = useWorkspaceMemberConnectors(workspaceId) const connectedConnectorIds = useMemo( () => new Set( diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx index da5bdaeed8e..da022ddb001 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx @@ -793,9 +793,12 @@ export const Sidebar = memo(function Sidebar({ label: 'Integrations', icon: Integration, href: `/workspace/${workspaceId}/integrations`, - /* Skills is a tab of this surface, not its own nav item — keep the entry - lit while the user is on it. */ - additionalActivePaths: [`/workspace/${workspaceId}/skills`], + /* Skills and Search are tabs of this surface, not their own nav items — + keep the entry lit while the user is on either. */ + additionalActivePaths: [ + `/workspace/${workspaceId}/skills`, + `/workspace/${workspaceId}/search`, + ], hidden: permissionConfig.hideIntegrationsTab, }, ].filter((item) => !item.hidden), diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index b9a0179a9d2..fe879a6cd59 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -1,6 +1,6 @@ 'use client' -import { useCallback, useEffect, useState } from 'react' +import { useEffect, useState } from 'react' import { createLogger } from '@sim/logger' import { type QueryKey, useQueryClient } from '@tanstack/react-query' import { useStartConnectorMemberEnrollment } from '@/hooks/queries/kb/connectors' @@ -34,9 +34,8 @@ export function useMemberEnrollment({ connectedConnectorIds, }: UseMemberEnrollmentProps) { const queryClient = useQueryClient() - const { mutate: startEnrollment, isPending } = useStartConnectorMemberEnrollment() + const { mutate: startEnrollment, isPending, error } = useStartConnectorMemberEnrollment() const [awaitingSince, setAwaitingSince] = useState>(() => new Map()) - const [error, setError] = useState(null) const awaiting = [...awaitingSince.keys()].some((id) => !connectedConnectorIds.has(id)) useEffect(() => { @@ -59,39 +58,31 @@ export function useMemberEnrollment({ return () => clearInterval(timer) }, [awaiting, connectedConnectorIds, membershipQueryKeys, queryClient]) - const connect = useCallback( - (knowledgeBaseId: string, connectorId: string) => { - setError(null) - const tab = window.open('about:blank', '_blank') - if (tab) tab.opener = null - startEnrollment( - { knowledgeBaseId, connectorId }, - { - onSuccess: ({ url }) => { - if (tab && !tab.closed) { - tab.location.href = url - } else { - window.location.assign(url) - return - } - setAwaitingSince((current) => new Map(current).set(connectorId, Date.now())) - }, - onError: (err) => { - tab?.close() - logger.error('Failed to start member enrollment', { error: err.message }) - setError(err.message) - }, - } - ) - }, - [startEnrollment] - ) + const connect = (knowledgeBaseId: string, connectorId: string) => { + const tab = window.open('about:blank', '_blank') + if (tab) tab.opener = null + startEnrollment( + { knowledgeBaseId, connectorId }, + { + onSuccess: ({ url }) => { + if (tab && !tab.closed) { + tab.location.href = url + } else { + window.location.assign(url) + return + } + setAwaitingSince((current) => new Map(current).set(connectorId, Date.now())) + }, + onError: (err) => { + tab?.close() + logger.error('Failed to start member enrollment', { error: err.message }) + }, + } + ) + } - const isAwaiting = useCallback( - (connectorId: string) => - awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId), - [awaitingSince, connectedConnectorIds] - ) + const isAwaiting = (connectorId: string) => + awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId) - return { connect, isAwaiting, isPending, error } + return { connect, isAwaiting, isPending, error: error?.message ?? null } } diff --git a/apps/sim/lib/api/list-convention.test.ts b/apps/sim/lib/api/list-convention.test.ts index 255f6edd28b..ac7fa19028d 100644 --- a/apps/sim/lib/api/list-convention.test.ts +++ b/apps/sim/lib/api/list-convention.test.ts @@ -56,6 +56,7 @@ vi.mock('@/lib/workflows/skills/builtin-skills', () => ({ import { listVisibleWorkspaceCredentials } from '@/lib/credentials/queries' import { listFoldersForWorkspace } from '@/lib/folders/queries' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { getDocuments } from '@/lib/knowledge/documents/service' import { getWorkspaceKnowledgeBases } from '@/lib/knowledge/service' import { listWorkspaceMcpServers } from '@/lib/mcp/queries' @@ -182,7 +183,8 @@ const CASES: ListCase[] = [ getDocuments( 'knowledge-1', { search, sortBy: sortBy as never, sortOrder: sortOrder as never }, - 'request-1' + 'request-1', + WORKSPACE_ACCESS_SCOPE ), sort: { sortBy: 'fileSize', diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts index e05e5766938..64ade013d81 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -319,9 +319,10 @@ export async function assertUserFileContentAccess( * is one; `options.userId` alone may be the workflow owner standing in for an * actorless run and must not widen what the run can read. */ - const knowledgeAccess = options.principal - ? await resolveKnowledgeAccessScope(options.principal, { workspaceId: options.workspaceId }) - : undefined + const knowledgeAccess = + context === 'knowledge-base' && options.principal + ? await resolveKnowledgeAccessScope(options.principal, { workspaceId: options.workspaceId }) + : undefined const hasAccess = await verifyFileAccess(file.key, options.userId, undefined, context, false, { knowledgeAccess, }) diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index 9be1d25afa3..690a816b28b 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.7.0 +version: 1.8.0 appVersion: "v0.8.18" kubeVersion: ">=1.25.0-0" home: https://sim.ai From b930eaded4c71b37f87ce7f5f6b9e1b215b3510e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 22:46:15 -0700 Subject: [PATCH 24/76] fix(knowledge): apply the audit swarm's findings - Mutation responses failed contract validation because a viewer's membership was required; a mutation now answers with null - Bulk enable/disable wrote to documents outside the caller's scope - The staleness sweep tombstoned deferred and backing-off connectors; it now only touches connectors that completed a run in the window - Members pace on the plain interval so the connector's jittered run finds them due; a member who alone exhausts the run budget backs off; an account that connects mid-run keeps the connector re-dispatching - Gmail and Google Calendar kept a 500-item cap once the cap field was cleared; caps are now written as 0, which every connector reads as unlimited, on the update path as well - Jira, ClickUp, Asana, Linear, and Dropbox classify an unreachable scope on the source's own error rather than a bare 404; an unmatched Atlassian domain is a configuration error, not this person's access - A Credential Group or option serving a members-mode connector cannot be removed under it; the OAuth completion dispatch cannot fail the callback; the content queue refuses a connector that syncs per member - The iterative vector scan runs only for a personal token set; the ACL check validates each element; the flag is read before the enrollment join; connector info is loaded only where it is shown - Search tab: the empty state counts member rows, rows stack, one membership vocabulary shared with the knowledge base banner; an admin can rebind a per-member connector to any matching group --- .../app/api/knowledge/search/utils.test.ts | 12 +- .../add-connector-modal.tsx | 41 +++--- .../connector-access-field.tsx | 75 +++------- .../connectors-section/connectors-section.tsx | 32 ++-- .../knowledge/[id]/components/consts.ts | 4 + .../edit-connector-modal.tsx | 48 +++--- .../member-connect-banner.tsx | 99 +++++-------- .../use-connector-member-group-options.ts | 12 +- .../member-connectors-section.tsx | 138 ++++++++---------- .../workspace/[workspaceId]/search/search.tsx | 30 +++- apps/sim/connectors/asana/asana.ts | 3 +- apps/sim/connectors/clickup/clickup.ts | 6 +- apps/sim/connectors/dropbox/dropbox.ts | 18 ++- apps/sim/connectors/gmail/gmail.ts | 25 +++- .../google-calendar/google-calendar.ts | 6 +- apps/sim/connectors/jira/jira.ts | 7 +- apps/sim/connectors/linear/linear.ts | 25 +++- apps/sim/connectors/utils.ts | 13 +- apps/sim/hooks/queries/kb/connectors.ts | 5 +- apps/sim/hooks/use-member-enrollment.ts | 109 +++++++++++--- .../contracts/knowledge/connectors.test.ts | 7 + .../lib/api/contracts/knowledge/connectors.ts | 6 +- apps/sim/lib/atlassian/discovery.ts | 16 +- apps/sim/lib/core/application/index.ts | 2 +- .../application/workspace-authorization.ts | 10 +- apps/sim/lib/credential-groups/credentials.ts | 11 +- apps/sim/lib/credential-groups/oauth.ts | 20 +-- apps/sim/lib/credential-groups/service.ts | 59 +++++++- .../lib/knowledge/access/predicate.test.ts | 10 +- apps/sim/lib/knowledge/access/predicate.ts | 13 +- apps/sim/lib/knowledge/access/scope.ts | 21 +-- apps/sim/lib/knowledge/access/tokens.ts | 8 +- .../lib/knowledge/api/internal-route.test.ts | 40 +++++ apps/sim/lib/knowledge/api/internal-route.ts | 3 + .../lib/knowledge/application/connectors.ts | 26 ++-- .../knowledge/application/knowledge-bases.ts | 30 +--- .../lib/knowledge/connectors/member-access.ts | 7 +- .../connectors/member-observations.ts | 88 ++++++++--- .../connectors/member-sync-engine.test.ts | 13 ++ .../connectors/member-sync-engine.ts | 123 ++++++++++------ .../lib/knowledge/connectors/queue.test.ts | 3 + apps/sim/lib/knowledge/connectors/queue.ts | 3 +- .../sim/lib/knowledge/connectors/sync-lock.ts | 2 +- .../knowledge/connectors/sync-persistence.ts | 26 ++-- .../knowledge/connectors/sync-primitives.ts | 19 ++- apps/sim/lib/knowledge/documents/service.ts | 5 +- .../orchestration/connector-access.ts | 52 ++----- .../lib/knowledge/orchestration/connectors.ts | 15 +- apps/sim/lib/knowledge/search/queries.ts | 29 +++- apps/sim/lib/knowledge/service.ts | 41 +++++- .../0318_permission_aware_knowledge.sql | 2 +- .../db/migrations/meta/0318_snapshot.json | 2 +- packages/db/schema.ts | 7 +- 53 files changed, 881 insertions(+), 546 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index f6c3c0f603c..7d97da6c79f 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -510,12 +510,12 @@ describe('Knowledge Search Utils', () => { it('runs both legs and fuses them in hybrid mode', async () => { /** - * Chains dequeue in creation order. The vector leg opens its transaction - * and applies the scan settings before selecting, so the keyword leg's - * ranking pass is built first, then the vector select, then hydration. + * Chains dequeue in creation order. A workspace-scoped vector leg selects + * directly (the iterative scan is reserved for a personal token set), so + * its select is built first, then the keyword ranking pass, then hydration. */ - queueTableRows(schemaMock.embedding, [{ id: 'keyword-hit', keywordRank: 0.9 }]) queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) + queueTableRows(schemaMock.embedding, [{ id: 'keyword-hit', keywordRank: 0.9 }]) queueTableRows(schemaMock.embedding, [makeResult('keyword-hit')]) const results = await executeKnowledgeSearch({ @@ -532,9 +532,9 @@ describe('Knowledge Search Utils', () => { }) it('falls back to vector results when the keyword leg fails', async () => { - /** The failing ranking chain is still built first and takes the first queued set. */ - queueTableRows(schemaMock.embedding, [{ id: 'never-ranked', keywordRank: 0 }]) queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) + /** The failing ranking chain is still built and takes the second queued set. */ + queueTableRows(schemaMock.embedding, [{ id: 'never-ranked', keywordRank: 0 }]) /** * Both legs share one `orderBy` spy, so target the keyword leg by its diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index 9a28f548567..7e0fdeeb337 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -36,10 +36,16 @@ import { } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field' import { ConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields' import { hasWorkspaceMaxConnectorAccess } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements' -import { SYNC_INTERVALS } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' +import { + BROWSE_WITH_HINT, + SYNC_INTERVALS, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' -import { useConnectorMemberGroupOptions } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' +import { + memberCapFieldIds, + useConnectorMemberGroupOptions, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { getBlock } from '@/blocks' @@ -104,11 +110,9 @@ export function AddConnectorModal({ /** Several groups collect this provider's accounts: the admin has to say which. */ const membersChoiceOpen = isMembersMode && groupOptions.needsChoice && !access.credentialGroupOptionId - /** Fields a per-member crawl refuses: a cap would hide part of a member's corpus. */ - const memberCapFieldIds = useMemo( - () => - new Set(isMembersMode ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) : []), - [isMembersMode, connectorConfig] + const hiddenCapFieldIds = useMemo( + () => memberCapFieldIds(connectorConfig, access.accessMode), + [connectorConfig, access.accessMode] ) /** True when the connector declares its key optional (public sources need none). */ const isApiKeyOptional = @@ -202,7 +206,7 @@ export function AddConnectorModal({ for (const field of connectorConfig.configFields) { if (!field.required) continue if (!isFieldVisible(field)) continue - if (memberCapFieldIds.has(field.id)) continue + if (hiddenCapFieldIds.has(field.id)) continue if (!isFieldPopulated(field)) return false } return true @@ -211,7 +215,7 @@ export function AddConnectorModal({ isApiKeyMode, isMembersMode, membersChoiceOpen, - memberCapFieldIds, + hiddenCapFieldIds, isApiKeyOptional, apiKeyValue, effectiveCredentialId, @@ -226,7 +230,7 @@ export function AddConnectorModal({ const resolvedConfig: Record = {} for (const [key, value] of Object.entries(resolveSourceConfig())) { - if (memberCapFieldIds.has(key)) continue + if (hiddenCapFieldIds.has(key)) continue if (Array.isArray(value)) { if (value.length > 0) resolvedConfig[key] = value } else if (typeof value === 'string') { @@ -345,7 +349,6 @@ export function AddConnectorModal({ <> {!isApiKeyMode && memberAccessAvailable && ( - isFieldVisible(field) && !memberCapFieldIds.has(field.id) + isFieldVisible(field) && !hiddenCapFieldIds.has(field.id) } onFieldChange={handleFieldChange} onToggleCanonicalMode={toggleCanonicalMode} @@ -502,7 +501,13 @@ export function AddConnectorModal({ onOpenChange(false)} primaryAction={{ - label: isCreating ? 'Connecting…' : 'Connect & Sync', + label: isCreating + ? isMembersMode + ? 'Creating…' + : 'Connecting…' + : isMembersMode + ? 'Create & Invite' + : 'Connect & Sync', onClick: handleSubmit, disabled: !canSubmit || isCreating, }} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx index 081584039c5..e8a2fa4cdef 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx @@ -2,13 +2,11 @@ import type { ReactNode } from 'react' import { ButtonGroup, ButtonGroupItem, ChipCombobox, ChipModalField } from '@sim/emcn' -import Link from 'next/link' import { type ConnectorMemberGroupOptions, decodeConnectorMemberGroupOption, encodeConnectorMemberGroupOption, } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' -import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import type { ConnectorMeta } from '@/connectors/types' /** What the caller chose; `members` may name the option the connector crawls with. */ @@ -19,7 +17,6 @@ export interface ConnectorAccessSelection { } interface ConnectorAccessFieldProps { - workspaceId: string connectorConfig: ConnectorMeta value: ConnectorAccessSelection onChange: (value: ConnectorAccessSelection) => void @@ -30,6 +27,11 @@ interface ConnectorAccessFieldProps { disabled?: boolean /** Whether per-member access may be chosen; false leaves only the way back to workspace access. */ allowMembers?: boolean + /** + * Whether the connector already syncs per member, so any matching group may + * be chosen, not only when several make the choice necessary. + */ + canRebind?: boolean /** Rendered under the selection, for a caller that applies the change with its own control. */ footer?: ReactNode } @@ -43,7 +45,6 @@ interface ConnectorAccessFieldProps { * several matching groups is asked which one to use. */ export function ConnectorAccessField({ - workspaceId, connectorConfig, value, onChange, @@ -51,11 +52,9 @@ export function ConnectorAccessField({ canAdmin, disabled = false, allowMembers = true, + canRebind = false, footer, }: ConnectorAccessFieldProps) { - const { features } = useWorkspaceHostContext() - const credentialGroupsAvailable = features?.credentialGroups === true - if (!groupOptions.supported) return null if (!canAdmin) { @@ -79,11 +78,7 @@ export function ConnectorAccessField({ ? encodeConnectorMemberGroupOption(value.credentialGroupId, value.credentialGroupOptionId) : undefined const { options, needsChoice, isLoading, error } = groupOptions - const membersHint = !credentialGroupsAvailable - ? 'Per-member access needs Credential Groups, which are not available on this plan.' - : !allowMembers - ? 'Per-member access is turned off for this workspace.' - : undefined + const showPicker = needsChoice || (canRebind && options.length > 0) return (
@@ -106,47 +103,23 @@ export function ConnectorAccessField({ Workspace - + Per member - {value.accessMode === 'members' && ( - <> - {needsChoice && ( - { - const decoded = decodeConnectorMemberGroupOption(next) - if (decoded) onChange({ accessMode: 'members', ...decoded }) - }} - placeholder='Choose which credential group members connect through' - isLoading={isLoading} - disabled={disabled || Boolean(error)} - /> - )} -

- {options.length === 1 - ? `Members connect through ${options[0].label}. ` - : options.length === 0 - ? `A credential group named ${connectorConfig.name} is created. ` - : ''} - Everyone in the workspace is invited by email to connect their own{' '} - {connectorConfig.name} account as the first sync starts, and people who join later are - invited automatically. Manage members in{' '} - - Settings - - . -

- + {value.accessMode === 'members' && showPicker && ( + { + const decoded = decodeConnectorMemberGroupOption(next) + if (decoded) onChange({ accessMode: 'members', ...decoded }) + }} + placeholder='Choose which credential group members connect through' + isLoading={isLoading} + disabled={disabled || Boolean(error)} + /> )} {footer} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index 8e1627c0fe2..e51d987953a 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -88,10 +88,10 @@ const SYNC_IN_FLIGHT_TOOLTIP = { } as const /** The member engine's own in-flight states, shown when the connector syncs per member. */ -const MEMBER_SYNC_IN_FLIGHT_TOOLTIP = { +const MEMBER_SYNC_IN_FLIGHT_TOOLTIP: Partial> = { pending: 'Member sync queued', running: 'Syncing members', -} as const +} /** How each member-engine status reads on the card's badge. */ const MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS = { @@ -383,11 +383,7 @@ function ConnectorCard({ syncInFlight || connector.status === 'disabled' || isPaused || memberSyncDisabled const syncTooltip = SYNC_IN_FLIGHT_TOOLTIP[connector.status as keyof typeof SYNC_IN_FLIGHT_TOOLTIP] ?? - (syncsPerMember - ? MEMBER_SYNC_IN_FLIGHT_TOOLTIP[ - connector.memberSyncStatus as keyof typeof MEMBER_SYNC_IN_FLIGHT_TOOLTIP - ] - : undefined) ?? + (syncsPerMember ? MEMBER_SYNC_IN_FLIGHT_TOOLTIP[connector.memberSyncStatus] : undefined) ?? (isPaused ? 'Resume to sync' : memberSyncDisabled @@ -442,18 +438,9 @@ function ConnectorCard({ {statusConfig.label} {syncsPerMember && ( - - - - - Per member - - - - Synced once per enrolled member; each person sees only the documents their own - account can open. - - + + Per member + )}
@@ -946,7 +933,7 @@ interface MemberSyncHistoryProps { * membership stands. A run that ended with members still due re-dispatches * itself, so several short rows in a row are one drain, not a fault. */ -export function MemberSyncHistory({ logs, members, isLoading }: MemberSyncHistoryProps) { +function MemberSyncHistory({ logs, members, isLoading }: MemberSyncHistoryProps) { if (isLoading) { return (
@@ -1002,7 +989,10 @@ export function MemberSyncHistory({ logs, members, isLoading }: MemberSyncHistor ? '' : 's'} {log.membersFailed > 0 && ( - !{log.membersFailed} + + {' '} + · {log.membersFailed} failed + )} {changes > 0 ? ( <> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts index 4ab8aff2a96..6551b3be525 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts @@ -1,3 +1,7 @@ +/** Under the account picker of a per-member connector, whose account only browses. */ +export const BROWSE_WITH_HINT = + 'Only used to pick folders and spaces below. The connector syncs as each member, not as this account.' + export const SYNC_INTERVALS = [ { label: 'Live', value: 5, requiresMax: true }, { label: 'Every hour', value: 60, requiresMax: false }, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index 57d5d02db74..1caa439d744 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -27,14 +27,20 @@ import { } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field' import { ConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields' import { hasWorkspaceMaxConnectorAccess } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements' -import { SYNC_INTERVALS } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' +import { + BROWSE_WITH_HINT, + SYNC_INTERVALS, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge' import type { ConfigFieldMap, ConfigFieldValue, } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' -import { useConnectorMemberGroupOptions } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' +import { + memberCapFieldIds, + useConnectorMemberGroupOptions, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { withBrandIcon } from '@/blocks/brand-icon' @@ -259,11 +265,7 @@ export function EditConnectorModal({ /** A disabled member sync is re-enabled by applying the current binding again. */ const canReenableMemberSync = !accessDirty && connector.accessMode === 'members' && connector.memberSyncStatus === 'disabled' - const memberCapFieldIds = new Set( - access.accessMode === 'members' - ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) - : [] - ) + const hiddenCapFieldIds = memberCapFieldIds(connectorConfig, access.accessMode) const persistedCanonicalModes = useMemo( () => readPersistedCanonicalModes(connector.sourceConfig), @@ -394,13 +396,14 @@ export function EditConnectorModal({ {activeTab === 'settings' ? ( isFieldVisible(field) && !memberCapFieldIds.has(field.id)} + isFieldVisible={(field) => isFieldVisible(field) && !hiddenCapFieldIds.has(field.id)} syncInterval={syncInterval} setSyncInterval={setSyncInterval} hasMaxAccess={hasMaxAccess} @@ -445,6 +448,8 @@ export function EditConnectorModal({ interface SettingsTabProps { connectorConfig: ConnectorMeta | null + /** The mode the connector is saved in, which the draft `access` may differ from. */ + persistedAccessMode: 'workspace' | 'members' sourceConfig: ConfigFieldMap credentialId: string | null canonicalGroups: Map @@ -477,6 +482,7 @@ interface SettingsTabProps { function SettingsTab({ connectorConfig, + persistedAccessMode, sourceConfig, credentialId, canonicalGroups, @@ -511,6 +517,8 @@ function SettingsTab({ ? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider) : null const syncsPerMember = access.accessMode === 'members' + /** Staying per member but through a different group. */ + const isRebind = accessDirty && persistedAccessMode === 'members' && syncsPerMember const { data: rawCredentials = [], isLoading: credentialsLoading } = useOAuthCredentials( providerId ?? undefined, { enabled: (needsWorkspaceCredential || syncsPerMember) && Boolean(providerId), workspaceId } @@ -533,12 +541,12 @@ function SettingsTab({ <> {connectorConfig && connectorConfig.auth.mode === 'oauth' && showAccessField && ( {isSwitchingAccess ? 'Switching…' - : access.accessMode === 'members' - ? 'Switch to per-member access' - : 'Switch to workspace access'} + : isRebind + ? 'Change credential group' + : access.accessMode === 'members' + ? 'Switch to per-member access' + : 'Switch to workspace access'}

- {access.accessMode === 'members' - ? 'Everyone in the workspace is invited to connect their account. Documents stay hidden until members connect and sync; listing caps are cleared.' - : 'Every workspace member can read every synced document once the next sync completes.'} + {isRebind + ? 'Members of the previous group lose access; members of the new group are invited to connect.' + : access.accessMode === 'members' + ? 'Everyone in the workspace is invited to connect their account. Documents stay hidden until members connect and sync; listing caps are cleared.' + : 'Every workspace member can read every synced document once the next sync completes.'}

) : undefined @@ -601,11 +613,7 @@ function SettingsTab({ )} {connectorConfig && syncsPerMember && ( - + = new Set([ - 'needs_reauth', - 'invited', - 'not_enrolled', -]) - /** * What the viewer must do for each per-member connector, if anything, and * what is happening for them once they have connected. */ export function MemberConnectBanner({ knowledgeBaseId, connectors }: MemberConnectBannerProps) { - const rows = connectors.filter( - (connector) => - connector.accessMode === 'members' && - connector.viewerMembership !== null && - (connector.viewerMembership !== 'connected' || - connector.memberSyncStatus === 'pending' || - connector.memberSyncStatus === 'running') - ) const connectedConnectorIds = useMemo( () => new Set( @@ -58,50 +44,45 @@ export function MemberConnectBanner({ knowledgeBaseId, connectors }: MemberConne connectedConnectorIds, }) + const rows = connectors.flatMap((connector) => { + const membership = connector.viewerMembership + if (connector.accessMode !== 'members' || membership === null) return [] + const waiting = isAwaiting(connector.id) + const text = describeMembership({ + membership, + memberSyncStatus: connector.memberSyncStatus, + waiting, + name: connectorName(connector), + }) + return text ? [{ connector, membership, waiting, text }] : [] + }) if (rows.length === 0) return null return (
- {rows.map((connector) => { - const name = connectorName(connector) - const membership = connector.viewerMembership - const waiting = isAwaiting(connector.id) - const text = - membership === 'connected' - ? `Syncing the ${name} documents shared with you. They appear when the sync completes.` - : membership === 'needs_reauth' - ? `Reconnect your ${name} account to keep seeing the documents shared with you.` - : membership === 'unverified_email' - ? `Verify your email address to see the ${name} documents shared with you.` - : membership === 'revoked' - ? `A workspace admin removed your access to ${name} documents.` - : waiting - ? `Finish connecting your ${name} account in the other tab.` - : `Connect your ${name} account to see the documents shared with you.` - return ( -
-

- {(membership === 'connected' || waiting) && ( - - )} - {text} -

- {membership && CONNECTABLE.has(membership) && ( - + {rows.map(({ connector, membership, waiting, text }) => ( +
+

+ {(membership === 'connected' || waiting) && ( + )} -

- ) - })} + {text} +

+ {CONNECTABLE_MEMBERSHIPS.has(membership) && ( + + )} +
+ ))} {error &&

{error}

}
) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts index d25d4387a9c..6806f4c8af4 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts @@ -31,7 +31,7 @@ export function decodeConnectorMemberGroupOption( } /** The credential-group provider that collects accounts for this connector, if any. */ -export function connectorMemberGroupProvider( +function connectorMemberGroupProvider( connectorConfig: ConnectorMeta ): CredentialGroupStandardOAuthProvider | null { if (connectorConfig.auth.mode !== 'oauth' || !connectorConfig.permissionScopedListing) return null @@ -42,6 +42,16 @@ export function connectorMemberGroupProvider( } } +/** The config fields a per-member connector hides: its listing caps, which the server clears. */ +export function memberCapFieldIds( + connectorConfig: ConnectorMeta | null, + accessMode: 'workspace' | 'members' +): ReadonlySet { + return new Set( + accessMode === 'members' ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) : [] + ) +} + interface UseConnectorMemberGroupOptionsInput { workspaceId: string connectorConfig: ConnectorMeta | null diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx index 7dcdca4a9be..520370b7907 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx @@ -2,51 +2,32 @@ import { useMemo } from 'react' import { Button } from '@sim/emcn' -import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' -import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import { + RESOURCE_LIST_STACK, + SettingsResourceRow, +} from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import { memberConnectorKeys, type WorkspaceMemberConnector } from '@/hooks/queries/kb/connectors' import { - memberConnectorKeys, - useWorkspaceMemberConnectors, - type ViewerConnectorMembership, - type WorkspaceMemberConnector, -} from '@/hooks/queries/kb/connectors' -import { useMemberEnrollment } from '@/hooks/use-member-enrollment' + CONNECTABLE_MEMBERSHIPS, + describeMembership, + enrollmentActionLabel, + useMemberEnrollment, +} from '@/hooks/use-member-enrollment' const SHARED_WITH_YOU_LABEL = 'Shared with you' -const EMPTY_CONNECTORS: WorkspaceMemberConnector[] = [] - -/** Memberships the viewer can act on themselves. */ -const CONNECTABLE: ReadonlySet = new Set([ - 'needs_reauth', - 'invited', - 'not_enrolled', -]) -function describe(connector: WorkspaceMemberConnector, waiting: boolean): string { - switch (connector.viewerMembership) { - case 'connected': - return connector.memberSyncStatus === 'pending' || connector.memberSyncStatus === 'running' - ? `${connector.knowledgeBaseName} · syncing the documents shared with you` - : `${connector.knowledgeBaseName} · connected` - case 'needs_reauth': - return `${connector.knowledgeBaseName} · reconnect to keep seeing the documents shared with you` - case 'unverified_email': - return `${connector.knowledgeBaseName} · verify your email address to see the documents shared with you` - case 'revoked': - return `${connector.knowledgeBaseName} · a workspace admin removed your access` - default: - return waiting - ? `${connector.knowledgeBaseName} · finish connecting in the other tab` - : `${connector.knowledgeBaseName} · connect to see the documents shared with you` - } +/** The name a per-member connector shows, from its registry entry. */ +export function memberConnectorName(connector: WorkspaceMemberConnector): string { + return CONNECTOR_META_REGISTRY[connector.connectorType]?.name ?? connector.connectorType } interface MemberConnectorsSectionProps { workspaceId: string - /** Lower-cased search text; rows whose knowledge base or connector name lacks it are hidden. */ - search: string + /** The per-member connectors to show, already narrowed by the page's search. */ + connectors: WorkspaceMemberConnector[] } /** @@ -54,8 +35,7 @@ interface MemberConnectorsSectionProps { * stands with each. Connecting here is the same enrollment the knowledge base * page offers, so a person can do it from whichever surface they are on. */ -export function MemberConnectorsSection({ workspaceId, search }: MemberConnectorsSectionProps) { - const { data: connectors = EMPTY_CONNECTORS } = useWorkspaceMemberConnectors(workspaceId) +export function MemberConnectorsSection({ workspaceId, connectors }: MemberConnectorsSectionProps) { const connectedConnectorIds = useMemo( () => new Set( @@ -71,51 +51,51 @@ export function MemberConnectorsSection({ workspaceId, search }: MemberConnector connectedConnectorIds, }) - const visible = connectors.filter((connector) => { - if (!search) return true - const name = CONNECTOR_META_REGISTRY[connector.connectorType]?.name ?? connector.connectorType - return [name, connector.knowledgeBaseName].some((text) => text.toLowerCase().includes(search)) - }) - if (visible.length === 0) return null + if (connectors.length === 0) return null return ( <> - - {visible.map((connector) => { - const meta = CONNECTOR_META_REGISTRY[connector.connectorType] - const waiting = isAwaiting(connector.connectorId) - const connectable = CONNECTABLE.has(connector.viewerMembership) - return ( - - ) : undefined - } - title={meta?.name ?? connector.connectorType} - description={describe(connector, waiting)} - trailing={ - connectable ? ( - - ) : undefined - } - /> - ) - })} - + +
+ {connectors.map((connector) => { + const meta = CONNECTOR_META_REGISTRY[connector.connectorType] + const name = memberConnectorName(connector) + const waiting = isAwaiting(connector.connectorId) + const state = + describeMembership({ + membership: connector.viewerMembership, + memberSyncStatus: connector.memberSyncStatus, + waiting, + name, + }) ?? 'Connected.' + return ( + + ) : undefined + } + title={name} + description={`${connector.knowledgeBaseName} · ${state}`} + trailing={ + CONNECTABLE_MEMBERSHIPS.has(connector.viewerMembership) ? ( + + ) : undefined + } + /> + ) + })} +
+
{error &&

{error}

} ) diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx index f19aee99b36..172a2c729a1 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -17,7 +17,10 @@ import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/c import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' import { CONNECTED_LABEL } from '@/app/workspace/[workspaceId]/integrations/search-params' -import { MemberConnectorsSection } from '@/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section' +import { + MemberConnectorsSection, + memberConnectorName, +} from '@/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section' import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' import { connectorSearchParam, @@ -26,10 +29,15 @@ import { import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' import type { WorkspaceCredential } from '@/hooks/queries/credentials' +import { + useWorkspaceMemberConnectors, + type WorkspaceMemberConnector, +} from '@/hooks/queries/kb/connectors' import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' import { usePermissionConfig } from '@/hooks/use-permission-config' +const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] const CONNECTORS_LABEL = 'Sim Search Connectors' interface ConnectorItemProps { @@ -131,8 +139,21 @@ export function Search() { ) : SEARCH_CONNECTORS + const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS } = + useWorkspaceMemberConnectors(workspaceId) + const visibleMemberConnectors = normalizedSearch + ? memberConnectors.filter((connector) => + [memberConnectorName(connector), connector.knowledgeBaseName].some((text) => + text.toLowerCase().includes(normalizedSearch) + ) + ) + : memberConnectors + const showNoResults = - Boolean(normalizedSearch) && visibleCredentials.length === 0 && visibleConnectors.length === 0 + Boolean(normalizedSearch) && + visibleCredentials.length === 0 && + visibleConnectors.length === 0 && + visibleMemberConnectors.length === 0 return (
@@ -151,7 +172,10 @@ export function Search() { />
- + {visibleCredentials.length > 0 && ( diff --git a/apps/sim/connectors/asana/asana.ts b/apps/sim/connectors/asana/asana.ts index b507fade076..f6c3758a61a 100644 --- a/apps/sim/connectors/asana/asana.ts +++ b/apps/sim/connectors/asana/asana.ts @@ -314,7 +314,8 @@ async function listWorkspaceProjects( export const asanaConnector: ConnectorConfig = { ...asanaConnectorMeta, - isListingScopeUnavailableError: (error) => error instanceof AsanaApiError && error.status === 404, + isListingScopeUnavailableError: (error) => + error instanceof AsanaApiError && (error.status === 404 || error.status === 403), listDocuments: async ( accessToken: string, diff --git a/apps/sim/connectors/clickup/clickup.ts b/apps/sim/connectors/clickup/clickup.ts index a86aaa27d20..87a9ac3bce6 100644 --- a/apps/sim/connectors/clickup/clickup.ts +++ b/apps/sim/connectors/clickup/clickup.ts @@ -187,7 +187,11 @@ export const clickupConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list ClickUp Docs', { status: response.status, error: errorText }) - throw listingRequestError('Failed to list ClickUp Docs', response.status) + throw listingRequestError( + 'Failed to list ClickUp Docs', + response.status, + response.status === 404 || (response.status === 401 && /OAUTH_02[37]/.test(errorText)) + ) } const data = (await response.json()) as Record diff --git a/apps/sim/connectors/dropbox/dropbox.ts b/apps/sim/connectors/dropbox/dropbox.ts index 60b1cda51ce..fbeb6c5d4a7 100644 --- a/apps/sim/connectors/dropbox/dropbox.ts +++ b/apps/sim/connectors/dropbox/dropbox.ts @@ -6,10 +6,10 @@ import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/ import { CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, - ConnectorListingScopeUnavailableError, htmlToPlainText, isListingScopeUnavailableError, isSkippedDocument, + listingRequestError, markSkipped, parseTagDate, readBodyWithLimit, @@ -231,13 +231,15 @@ export const dropboxConnector: ConnectorConfig = { status: response.status, error: errorText, }) - /** Dropbox reports a path the caller cannot reach as 409 path/not_found. */ - throw response.status === 409 - ? new ConnectorListingScopeUnavailableError( - `Failed to list Dropbox folder: ${response.status}`, - response.status - ) - : new Error(`Failed to list Dropbox folder: ${response.status}`) + /** + * Dropbox answers every endpoint-specific failure with 409; only + * path/not_found means the caller cannot reach the folder. + */ + throw listingRequestError( + 'Failed to list Dropbox folder', + response.status, + response.status === 409 && /path\/not_found/.test(errorText) + ) } data = await response.json() diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index 4e604f1115f..90b4c429a43 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -3,7 +3,13 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + joinTagArray, + parseMultiValue, + parseOptionalUnlimitedSafeInteger, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('GmailConnector') @@ -446,17 +452,22 @@ export const gmailConnector: ConnectorConfig = { labelIndex = resolved } const searchQuery = buildSearchQuery(sourceConfig, labelIndex) - const maxThreads = sourceConfig.maxThreads - ? Number(sourceConfig.maxThreads) - : DEFAULT_MAX_THREADS + /** Absent means the default cap; an explicit 0 (a per-member sync) means unlimited. */ + const maxThreads = + sourceConfig.maxThreads === undefined + ? DEFAULT_MAX_THREADS + : parseOptionalUnlimitedSafeInteger( + sourceConfig.maxThreads, + 'maxThreads must be a non-negative integer' + ) const totalFetched = (syncContext?.totalThreadsFetched as number) ?? 0 - if (totalFetched >= maxThreads) { + if (maxThreads > 0 && totalFetched >= maxThreads) { return { documents: [], hasMore: false } } - const remaining = maxThreads - totalFetched - const pageSize = Math.min(THREADS_PER_PAGE, remaining) + const pageSize = + maxThreads > 0 ? Math.min(THREADS_PER_PAGE, maxThreads - totalFetched) : THREADS_PER_PAGE const queryParams = new URLSearchParams({ maxResults: String(pageSize), diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index 620ee62906a..d79eff2c868 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -348,9 +348,9 @@ export const googleCalendarConnector: ConnectorConfig = { const calendarId = calendarIds[calendarIndex] const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 - const rawMaxEvents = sourceConfig.maxEvents - ? Number(sourceConfig.maxEvents) - : DEFAULT_MAX_EVENTS + /** Absent means the default cap; an explicit 0 (a per-member sync) means unlimited. */ + const rawMaxEvents = + sourceConfig.maxEvents === undefined ? DEFAULT_MAX_EVENTS : Number(sourceConfig.maxEvents) const maxEvents = Number.isFinite(rawMaxEvents) ? rawMaxEvents : 0 const isCapped = maxEvents > 0 /** diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index 9d2d3af3a6f..419986d9be2 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -231,7 +231,12 @@ export const jiraConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw listingRequestError('Failed to search Jira issues', response.status) + throw listingRequestError( + 'Failed to search Jira issues', + response.status, + response.status === 404 || + (response.status === 400 && /does not exist for the field 'project'/i.test(errorText)) + ) } const data = await response.json() diff --git a/apps/sim/connectors/linear/linear.ts b/apps/sim/connectors/linear/linear.ts index 93aab253e64..47536312153 100644 --- a/apps/sim/connectors/linear/linear.ts +++ b/apps/sim/connectors/linear/linear.ts @@ -6,7 +6,13 @@ import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { linearConnectorMeta } from '@/connectors/linear/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, + joinTagArray, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' import { linearAuthorizationHeader } from '@/tools/linear/utils' const logger = createLogger('LinearConnector') @@ -55,6 +61,15 @@ const MAX_RATE_LIMIT_WAIT_MS = 30_000 /** * Detects Linear's `RATELIMITED` extension code anywhere in a GraphQL error array. */ +function isEntityNotFoundError(entry: unknown): boolean { + if (!entry || typeof entry !== 'object') return false + const error = entry as { message?: unknown; extensions?: { code?: unknown } } + return ( + error.extensions?.code === 'ENTITY_NOT_FOUND' || + (typeof error.message === 'string' && /entity not found/i.test(error.message)) + ) +} + function isRateLimitedErrors(errors: unknown[] | undefined): boolean { if (!Array.isArray(errors)) return false return errors.some((entry) => { @@ -129,7 +144,11 @@ async function linearGraphQL( */ if (Array.isArray(json?.errors) && json.errors.length > 0) { logger.error('Linear GraphQL errors', { errors: json.errors }) - throw new Error(`Linear GraphQL error: ${JSON.stringify(json.errors)}`) + const described = `Linear GraphQL error: ${JSON.stringify(json.errors)}` + /** A team or project the caller cannot see is reported as an entity that does not exist. */ + throw json.errors.some(isEntityNotFoundError) + ? new ConnectorListingScopeUnavailableError(described, response.status) + : new Error(described) } if (!json?.data || typeof json.data !== 'object') { @@ -297,6 +316,8 @@ function issueToDocument(issue: Record): ExternalDocument { export const linearConnector: ConnectorConfig = { ...linearConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index b5ed92d2d0c..4eb5b2f5473 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -714,12 +714,17 @@ export class ConnectorListingScopeUnavailableError extends Error { /** * The error a listing request throws for a failed response: scope-unavailable - * when the source says the scope does not exist for this caller (404), a plain - * error for anything else, which the sync engines retry with backoff. + * when the source says the scope does not exist for this caller (404, or + * whatever `scopeUnavailable` recognises in the source's own error body), a + * plain error for anything else, which the sync engines retry with backoff. */ -export function listingRequestError(message: string, status: number): Error { +export function listingRequestError( + message: string, + status: number, + scopeUnavailable: boolean = status === 404 +): Error { const described = `${message}: ${status}` - return status === 404 + return scopeUnavailable ? new ConnectorListingScopeUnavailableError(described, status) : new Error(described) } diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index be9411fee50..2f97da4cc0e 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -261,6 +261,7 @@ export function useCreateConnector() { onSettled: (_data, _error, { knowledgeBaseId }) => { queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) }, }) } @@ -403,8 +404,9 @@ export function useUpdateConnectorAccess() { queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, }) - /** The base list says whether any connector syncs per member. */ + /** The base list says whether any connector syncs per member, and the Search tab lists them. */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) }, }) } @@ -439,6 +441,7 @@ export function useDeleteConnector() { */ onSettled: (_data, _error, { knowledgeBaseId, deleteDocuments }) => { queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentLists(knowledgeBaseId) }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index fe879a6cd59..cb299be8200 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -1,19 +1,79 @@ 'use client' -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { type QueryKey, useQueryClient } from '@tanstack/react-query' -import { useStartConnectorMemberEnrollment } from '@/hooks/queries/kb/connectors' +import type { MemberSyncStatus } from '@/lib/knowledge/types' +import { + memberConnectorKeys, + useStartConnectorMemberEnrollment, + type ViewerConnectorMembership, +} from '@/hooks/queries/kb/connectors' const logger = createLogger('MemberEnrollment') -/** How often the given queries are refreshed while a member connects in another tab. */ +/** How often the membership queries are refreshed while a member connects in another tab. */ const AWAITING_CONNECTION_POLL_MS = 4_000 /** How long a connection is awaited before the surface stops refreshing on its own. */ const AWAITING_CONNECTION_TIMEOUT_MS = 10 * 60_000 +const POPUP_BLOCKED_MESSAGE = 'Allow pop-ups for this site to connect your account.' + +/** Memberships the viewer can act on themselves. */ +export const CONNECTABLE_MEMBERSHIPS: ReadonlySet = new Set([ + 'needs_reauth', + 'invited', + 'not_enrolled', +]) + +/** The label of the one action a connectable membership offers. */ +export function enrollmentActionLabel( + membership: ViewerConnectorMembership, + waiting: boolean +): string { + if (waiting) return 'Open again' + return membership === 'needs_reauth' ? 'Reconnect' : 'Connect' +} + +interface DescribeMembershipInput { + membership: ViewerConnectorMembership + memberSyncStatus: MemberSyncStatus + /** Whether this surface opened an enrollment tab that has not connected yet. */ + waiting: boolean + /** The connector's display name. */ + name: string +} + +/** + * One sentence on where the viewer stands with a per-member connector, shared + * by every surface that shows it so the wording cannot drift between them. + * Null once the viewer is connected and nothing is happening for them. + */ +export function describeMembership({ + membership, + memberSyncStatus, + waiting, + name, +}: DescribeMembershipInput): string | null { + switch (membership) { + case 'connected': + return memberSyncStatus === 'pending' || memberSyncStatus === 'running' + ? `Syncing the ${name} documents shared with you. They appear when the sync completes.` + : null + case 'needs_reauth': + return `Reconnect your ${name} account to keep seeing the documents shared with you.` + case 'unverified_email': + return `Verify your email address to see the ${name} documents shared with you.` + case 'revoked': + return `A workspace admin removed your access to ${name} documents.` + default: + return waiting + ? `Finish connecting your ${name} account in the other tab.` + : `Connect your ${name} account to see the documents shared with you.` + } +} interface UseMemberEnrollmentProps { - /** Queries carrying the viewer's membership, refreshed while a connection is awaited. */ + /** Queries this surface reads memberships from, refreshed while a connection is awaited. */ membershipQueryKeys: readonly QueryKey[] /** Connector ids the viewer is now connected to; awaiting stops for them. */ connectedConnectorIds: ReadonlySet @@ -21,9 +81,10 @@ interface UseMemberEnrollmentProps { /** * Lets the viewer connect their own account to a per-member connector. - * Enrollment opens in a new tab — the enrollment page ends by telling the - * person to close it — and the membership queries are polled meanwhile so - * the surface that started it updates on its own once they are connected. + * Enrollment opens in a new tab, and the membership queries are polled + * meanwhile so the surface that started it updates on its own once the + * account is connected; the workspace-wide membership list is refreshed too, + * so the other surface catches up as well. * * The tab is opened in the click itself, before the enrollment link is * minted, because a tab opened after a network round trip is outside the @@ -33,9 +94,15 @@ export function useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds, }: UseMemberEnrollmentProps) { + const connectedRef = useRef(connectedConnectorIds) const queryClient = useQueryClient() const { mutate: startEnrollment, isPending, error } = useStartConnectorMemberEnrollment() const [awaitingSince, setAwaitingSince] = useState>(() => new Map()) + const [popupBlocked, setPopupBlocked] = useState(false) + + useEffect(() => { + connectedRef.current = connectedConnectorIds + }, [connectedConnectorIds]) const awaiting = [...awaitingSince.keys()].some((id) => !connectedConnectorIds.has(id)) useEffect(() => { @@ -46,7 +113,7 @@ export function useMemberEnrollment({ const next = new Map( [...current].filter( ([id, since]) => - !connectedConnectorIds.has(id) && now - since < AWAITING_CONNECTION_TIMEOUT_MS + !connectedRef.current.has(id) && now - since < AWAITING_CONNECTION_TIMEOUT_MS ) ) return next.size === current.size ? current : next @@ -54,27 +121,28 @@ export function useMemberEnrollment({ for (const queryKey of membershipQueryKeys) { void queryClient.invalidateQueries({ queryKey }) } + void queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) }, AWAITING_CONNECTION_POLL_MS) return () => clearInterval(timer) - }, [awaiting, connectedConnectorIds, membershipQueryKeys, queryClient]) + }, [awaiting, membershipQueryKeys, queryClient]) const connect = (knowledgeBaseId: string, connectorId: string) => { const tab = window.open('about:blank', '_blank') - if (tab) tab.opener = null + if (!tab) { + setPopupBlocked(true) + return + } + tab.opener = null + setPopupBlocked(false) startEnrollment( { knowledgeBaseId, connectorId }, { onSuccess: ({ url }) => { - if (tab && !tab.closed) { - tab.location.href = url - } else { - window.location.assign(url) - return - } + tab.location.href = url setAwaitingSince((current) => new Map(current).set(connectorId, Date.now())) }, onError: (err) => { - tab?.close() + tab.close() logger.error('Failed to start member enrollment', { error: err.message }) }, } @@ -84,5 +152,10 @@ export function useMemberEnrollment({ const isAwaiting = (connectorId: string) => awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId) - return { connect, isAwaiting, isPending, error: error?.message ?? null } + return { + connect, + isAwaiting, + isPending, + error: popupBlocked ? POPUP_BLOCKED_MESSAGE : (error?.message ?? null), + } } diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts index c0d8118f9ec..5eeef18534f 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts @@ -60,6 +60,13 @@ describe('connector access binding contracts', () => { ).toBe(false) }) + it('refuses a mode switch that names no mode', () => { + expect(updateConnectorAccessBodySchema.safeParse({}).success).toBe(false) + expect(updateConnectorAccessBodySchema.safeParse({ credentialId: 'cred-1' }).success).toBe( + false + ) + }) + it('applies the same rules to a mode switch', () => { expect( updateConnectorAccessBodySchema.safeParse({ diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index 86fb791a09a..c5cfacf43bc 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -4,7 +4,7 @@ import { knowledgeConnectorParamsSchema, successResponseSchema, } from '@/lib/api/contracts/knowledge/shared' -import { booleanQueryFlagSchema } from '@/lib/api/contracts/primitives' +import { booleanQueryFlagSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, @@ -88,6 +88,8 @@ export const createConnectorBodySchema = z export const updateConnectorAccessBodySchema = z .object({ ...connectorAccessBindingShape, + /** A switch names the mode it moves to; nothing is implied by omission. */ + accessMode: connectorRequestedAccessModeSchema, credentialId: z.string().min(1).optional(), }) .superRefine((value, ctx) => { @@ -361,7 +363,7 @@ export type WorkspaceMemberConnector = z.output r.url).join(', ')}` ) diff --git a/apps/sim/lib/core/application/index.ts b/apps/sim/lib/core/application/index.ts index e9c81f031e0..75d6ba3ce2d 100644 --- a/apps/sim/lib/core/application/index.ts +++ b/apps/sim/lib/core/application/index.ts @@ -42,8 +42,8 @@ export { PersonalApiKeysDisabledError, PrincipalKindAuthorizationError, requireAllowedWorkspacePrincipal, + requireCurrentHumanRole, requirePersonalApiKeysAllowed, - requireWorkspaceRole, WorkspaceApiKeyAuthorizationError, WorkspaceApiKeyScopeAuthorizationError, } from '@/lib/core/application/workspace-authorization' diff --git a/apps/sim/lib/core/application/workspace-authorization.ts b/apps/sim/lib/core/application/workspace-authorization.ts index cf2a41096dc..2b834162911 100644 --- a/apps/sim/lib/core/application/workspace-authorization.ts +++ b/apps/sim/lib/core/application/workspace-authorization.ts @@ -248,7 +248,7 @@ export async function requirePersonalApiKeysAllowed( * raising a role, rather than chasing an admin about a group setting that is * not why they were refused. */ -async function requireCurrentHumanRole( +export async function requireCurrentHumanRole( userId: string, context: C, required: PermissionType, @@ -271,14 +271,6 @@ async function requireCurrentHumanRole( * enrolled member is an admin decision even though creating a connector is * not — so the operation keeps its role and the variant asserts its own. */ -export async function requireWorkspaceRole( - userId: string, - context: C, - required: PermissionType -): Promise { - await requireCurrentHumanRole(userId, context, required) -} - async function requireCurrentHumanAccess( userId: string, context: C, diff --git a/apps/sim/lib/credential-groups/credentials.ts b/apps/sim/lib/credential-groups/credentials.ts index f5d84e23c75..9ae83769012 100644 --- a/apps/sim/lib/credential-groups/credentials.ts +++ b/apps/sim/lib/credential-groups/credentials.ts @@ -60,6 +60,9 @@ export interface ManagedCredentialGroupBinding { optionStatus: 'active' | 'disabled' | null } +/** Enrollment statuses under which a person's managed credentials count as theirs. */ +export const LIVE_ENROLLMENT_STATUSES = ['in_progress', 'completed'] as const + /** * Whether a managed credential may be used right now: the credential, its * enrollment, its option, and its group are all live. Every consumer that @@ -75,7 +78,7 @@ export function isManagedCredentialGroupBindingLive( ): boolean { return ( binding.managedOauthStatus === 'active' && - (binding.enrollmentStatus === 'in_progress' || binding.enrollmentStatus === 'completed') && + (LIVE_ENROLLMENT_STATUSES as readonly string[]).includes(binding.enrollmentStatus) && binding.groupStatus === 'active' && binding.optionStatus === 'active' ) @@ -128,7 +131,7 @@ export async function loadCredentialGroupEnrollmentAccess( eq(user.id, userId), eq(user.emailVerified, true), eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), - inArray(credentialGroupEnrollment.status, ['in_progress', 'completed']) + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]) ) ) .limit(1) @@ -155,7 +158,7 @@ export async function loadCredentialGroupEnrollmentAccessForSubject( .where( and( eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), - inArray(credentialGroupEnrollment.status, ['in_progress', 'completed']), + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]), eq(credential.type, 'managed_oauth'), eq(credential.managedOauthStatus, 'active'), eq(credential.providerId, providerId), @@ -359,7 +362,7 @@ export async function listCredentialGroupCredentialReferences({ credentialProviderIds?.length ? inArray(credential.providerId, credentialProviderIds) : undefined, - inArray(credentialGroupEnrollment.status, ['in_progress', 'completed']), + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]), ], limit, cursor diff --git a/apps/sim/lib/credential-groups/oauth.ts b/apps/sim/lib/credential-groups/oauth.ts index 489120c2d40..131df11b1c0 100644 --- a/apps/sim/lib/credential-groups/oauth.ts +++ b/apps/sim/lib/credential-groups/oauth.ts @@ -96,9 +96,9 @@ async function assertCurrentPolicy( return policy } -/** Builds a provider authorization URL after persisting a provider-bound one-time attempt. */ const logger = createLogger('CredentialGroupOAuth') +/** Builds a provider authorization URL after persisting a provider-bound one-time attempt. */ export async function startCredentialGroupOAuth( context: CredentialGroupOAuthContext, invitationToken: string @@ -303,18 +303,20 @@ async function persistGrant( * their next run; queue one now so their documents arrive within minutes. * Loaded lazily: credential groups do not otherwise depend on knowledge. */ - const { dispatchMemberSyncsForCredentialOption } = await import( - '@/lib/knowledge/connectors/member-queue' - ) - await dispatchMemberSyncsForCredentialOption({ - workspaceId: context.workspaceId, - credentialGroupOptionId: context.option.id, - }).catch((error) => { + try { + const { dispatchMemberSyncsForCredentialOption } = await import( + '@/lib/knowledge/connectors/member-queue' + ) + await dispatchMemberSyncsForCredentialOption({ + workspaceId: context.workspaceId, + credentialGroupOptionId: context.option.id, + }) + } catch (error) { logger.warn('Failed to queue member syncs after an account connected', { credentialGroupOptionId: context.option.id, error: getErrorMessage(error), }) - }) + } return completion } diff --git a/apps/sim/lib/credential-groups/service.ts b/apps/sim/lib/credential-groups/service.ts index 6bff6ee700d..f972603d77e 100644 --- a/apps/sim/lib/credential-groups/service.ts +++ b/apps/sim/lib/credential-groups/service.ts @@ -4,9 +4,12 @@ import { credential, credentialGroup, credentialGroupEnrollment, + knowledgeBase, + knowledgeConnector, } from '@sim/db/schema' import { generateId } from '@sim/utils/id' -import { and, desc, eq, inArray } from 'drizzle-orm' +import { and, desc, eq, inArray, isNull } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupWorkflowAccessPolicyCodec, requireDefaultCredentialGroupWorkflowAccessPolicy, @@ -210,6 +213,48 @@ export async function createCredentialGroup( }) } +/** + * Refuses to remove a group, or the given options of it, while a knowledge + * connector syncs per member through one of them: the connector would be left + * bound to nothing, and its members' documents dark, without anyone choosing + * that. `optionIds` null means the whole group. + */ +async function refuseIfServingMemberConnectors( + executor: DbOrTx, + workspaceId: string, + groupId: string, + optionIds: readonly string[] | null +): Promise { + if (optionIds !== null && optionIds.length === 0) return + const serving = await executor + .select({ + knowledgeBaseName: knowledgeBase.name, + connectorType: knowledgeConnector.connectorType, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeConnector.accessMode, 'members'), + eq(knowledgeConnector.credentialGroupId, groupId), + optionIds === null + ? undefined + : inArray(knowledgeConnector.credentialGroupOptionId, [...optionIds]), + isNull(knowledgeConnector.deletedAt) + ) + ) + .limit(5) + if (serving.length === 0) return + const names = serving + .map((row) => `the ${row.connectorType} connector in "${row.knowledgeBaseName}"`) + .join(', ') + throw new OrchestrationError( + 'conflict', + `${optionIds === null ? 'This Credential Group' : 'An option being removed'} is what ${names} ${serving.length === 1 ? 'syncs' : 'sync'} per member through. Switch ${serving.length === 1 ? 'that connector' : 'those connectors'} to another group first.` + ) +} + export async function deleteCredentialGroup( workspaceId: string, groupId: string @@ -223,6 +268,7 @@ export async function deleteCredentialGroup( .for('update') if (!existing) return false + await refuseIfServingMemberConnectors(tx, workspaceId, groupId, null) await deleteResourcePolicyForResource( { workspaceId, resourceType: 'credential_group', resourceId: groupId }, tx @@ -250,6 +296,17 @@ export async function updateCredentialGroup( .for('update') if (!existing) return null + if (body.options !== undefined) { + const keptOptionIds = new Set(body.options.map((option) => option.id)) + await refuseIfServingMemberConnectors( + tx, + workspaceId, + groupId, + existing.options + .filter((option) => !keptOptionIds.has(option.id)) + .map((option) => option.id) + ) + } const nextOptions = body.options !== undefined ? await updateOptions(workspaceId, groupId, body.options, existing.options, tx) diff --git a/apps/sim/lib/knowledge/access/predicate.test.ts b/apps/sim/lib/knowledge/access/predicate.test.ts index 59f4bc7e8b8..3b7db9828f3 100644 --- a/apps/sim/lib/knowledge/access/predicate.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.test.ts @@ -2,10 +2,12 @@ * @vitest-environment node */ -// Renders the real predicate against the real drizzle dialect and schema. The -// shared client sets `fetch_types: false` (packages/db/db.ts), under which an -// array bound as one parameter fails at execution with 22P02, so the assertion -// that matters is that every bind is a scalar. +/** + * Renders the real predicate against the real drizzle dialect and schema. The + * shared client sets `fetch_types: false` (packages/db/db.ts), under which an + * array bound as one parameter fails at execution with 22P02, so the assertion + * that matters is that every bind is a scalar. + */ import { describe, expect, it, vi } from 'vitest' vi.unmock('drizzle-orm') diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index eb9326f5bea..2612e75938f 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -15,8 +15,17 @@ import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/ac export function knowledgeAccessCondition(scope: KnowledgeAccessScope | SystemAccessScope): SQL { if (scope.kind === 'system') return sql`true` if (scope.tokens.length === 0) return sql`false` - return sql`${document.acl} && ARRAY[${sql.join( - scope.tokens.map((token) => sql`${token}`), + return sql`${document.acl} && ${textArrayLiteral(scope.tokens)}` +} + +/** + * A `text[]` literal assembled from scalar binds, for comparing against an + * ACL column. Every place that compares ACLs builds its array this way, for + * the `fetch_types: false` reason above. + */ +export function textArrayLiteral(values: readonly string[]): SQL { + return sql`ARRAY[${sql.join( + values.map((value) => sql`${value}`), sql`, ` )}]::text[]` } diff --git a/apps/sim/lib/knowledge/access/scope.ts b/apps/sim/lib/knowledge/access/scope.ts index 7298e676754..3cae9340d40 100644 --- a/apps/sim/lib/knowledge/access/scope.ts +++ b/apps/sim/lib/knowledge/access/scope.ts @@ -5,6 +5,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { and, eq, inArray, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { LIVE_ENROLLMENT_STATUSES } from '@/lib/credential-groups/credentials' import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { sortAccessTokens, subjectToken } from '@/lib/knowledge/access/tokens' import { @@ -23,7 +24,6 @@ export const WORKSPACE_ACCESS_SCOPE: WorkspaceAccessScope = Object.freeze({ }) /** Enrollment states under which a credential-group membership counts as live. */ -const LIVE_ENROLLMENT_STATUSES = ['in_progress', 'completed'] as const export interface KnowledgeAccessScopeContext { /** Undefined only for a legacy personal knowledge base, which cannot own connectors. */ @@ -51,6 +51,16 @@ async function loadUserAccessTokens( */ const workspaceAccess = await checkWorkspaceAccess(workspaceId, userId) if (!workspaceAccess.hasAccess) return [...WORKSPACE_ACCESS_TOKENS] + /** + * A member token only counts where permission-aware knowledge is on, so + * turning the feature off hides every member-scoped document at once — on + * the next read, before any run has suspended anyone — rather than leaving + * enrolled members reading them until a run happens to land. Read first, so + * a workspace without the feature never pays for the enrollment join. + */ + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId }))) { + return [...WORKSPACE_ACCESS_TOKENS] + } const rows = await db .select({ @@ -107,15 +117,6 @@ async function loadUserAccessTokens( }) } } - /** - * A member token only counts where permission-aware knowledge is on, so - * turning the feature off hides every member-scoped document at once — on - * the next read, before any run has suspended anyone — rather than leaving - * enrolled members reading them until a run happens to land. - */ - if (subjectTokens.size > 0 && !(await isKnowledgeMemberAccessAvailable({ workspaceId }))) { - return [...WORKSPACE_ACCESS_TOKENS] - } return sortAccessTokens(new Set([...WORKSPACE_ACCESS_TOKENS, ...subjectTokens])) } diff --git a/apps/sim/lib/knowledge/access/tokens.ts b/apps/sim/lib/knowledge/access/tokens.ts index dc2841edc58..ff4cedfc63b 100644 --- a/apps/sim/lib/knowledge/access/tokens.ts +++ b/apps/sim/lib/knowledge/access/tokens.ts @@ -1,4 +1,4 @@ -import { PUBLIC_ACCESS_TOKEN, WORKSPACE_ACCESS_TOKEN } from '@/lib/knowledge/access/types' +import { WORKSPACE_ACCESS_TOKEN } from '@/lib/knowledge/access/types' /** * Shape of one access token, mirroring `doc_acl_token_shape_check` in the @@ -60,9 +60,3 @@ export function sortAccessTokens(tokens: Iterable): string[] { unique.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)) return unique } - -/** - * The ACL of a members-mode document: the sorted subject tokens of its active - * observers. Rejects anything that is not a well-formed token so a malformed - * value fails here rather than denying access silently. - */ diff --git a/apps/sim/lib/knowledge/api/internal-route.test.ts b/apps/sim/lib/knowledge/api/internal-route.test.ts index 4d69fea1e32..de40cb0269e 100644 --- a/apps/sim/lib/knowledge/api/internal-route.test.ts +++ b/apps/sim/lib/knowledge/api/internal-route.test.ts @@ -11,6 +11,7 @@ import { import { internalKnowledgeProvenanceUserId, resolveInternalKnowledgeBillingAttribution, + toInternalKnowledgeConnector, } from '@/lib/knowledge/api/internal-route' import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing' @@ -127,3 +128,42 @@ describe('internal Knowledge execution attribution', () => { ).rejects.toThrow('does not match the authenticated request scope') }) }) + +describe('toInternalKnowledgeConnector', () => { + const row = { + id: 'connector-1', + knowledgeBaseId: 'kb-1', + connectorType: 'google_drive', + credentialId: null, + sourceConfig: {}, + syncMode: null, + syncIntervalMinutes: 60, + status: 'active' as const, + lastSyncAt: null, + lastSyncError: null, + lastSyncDocCount: null, + nextSyncAt: null, + consecutiveFailures: 0, + accessMode: 'members' as const, + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + memberSyncStatus: 'idle' as const, + lastMemberSyncAt: null, + nextMemberSyncAt: null, + lastMemberSyncError: null, + memberSyncConsecutiveFailures: 0, + accessRewritePending: false, + createdAt: new Date('2026-09-01T00:00:00Z'), + updatedAt: new Date('2026-09-01T00:00:00Z'), + } + + it('presents a mutation result, which carries no viewer membership, as null', () => { + expect(toInternalKnowledgeConnector(row).viewerMembership).toBeNull() + }) + + it('keeps the membership a read resolved for the viewer', () => { + expect( + toInternalKnowledgeConnector({ ...row, viewerMembership: 'invited' }).viewerMembership + ).toBe('invited') + }) +}) diff --git a/apps/sim/lib/knowledge/api/internal-route.ts b/apps/sim/lib/knowledge/api/internal-route.ts index 9377289cbe9..43d91dafd40 100644 --- a/apps/sim/lib/knowledge/api/internal-route.ts +++ b/apps/sim/lib/knowledge/api/internal-route.ts @@ -135,9 +135,12 @@ export function toInternalKnowledgeConnector< nextSyncAt: Date | string | null lastMemberSyncAt: Date | string | null nextMemberSyncAt: Date | string | null + viewerMembership?: ConnectorData['viewerMembership'] }, >(connector: T): ConnectorData { return connectorDataSchema.parse({ + /** A mutation answers with the row alone; only a viewer's read carries their membership. */ + viewerMembership: null, ...connector, createdAt: serializeDate(connector.createdAt), updatedAt: serializeDate(connector.updatedAt), diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index f6c5a4287ca..bd515ea31cd 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -13,7 +13,7 @@ import { import { and, asc, count, desc, eq, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { decryptApiKey } from '@/lib/api-key/crypto' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' -import { requireWorkspaceRole } from '@/lib/core/application' +import { requireCurrentHumanRole } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { @@ -34,7 +34,10 @@ import { resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { resolveViewerConnectorMemberships } from '@/lib/knowledge/connectors/member-provisioning' +import { + resolveViewerConnectorMemberships, + type ViewerConnectorMembership, +} from '@/lib/knowledge/connectors/member-provisioning' import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' import { DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, @@ -326,7 +329,7 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ : await orderedQuery.limit(input.limit + 1).offset(offset) const hasMore = input.limit !== undefined && rows.length > input.limit const page = input.limit === undefined ? rows : rows.slice(0, input.limit) - const viewerUserId = resolvePrincipalSubjectUserId(principal) + const viewerUserId = principal.kind === 'session' ? principal.userId : null const memberships = viewerUserId && context.workspaceId ? await resolveViewerConnectorMemberships({ @@ -334,7 +337,7 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ workspaceId: context.workspaceId, connectors: page, }) - : new Map() + : new Map() return { connectors: page.map(({ encryptedApiKey: _encryptedApiKey, ...rest }) => ({ ...rest, @@ -396,7 +399,10 @@ export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ connectors: rows.flatMap((row) => { const viewerMembership = memberships.get(row.id) if (!isMemberSyncStatus(row.memberSyncStatus)) { - throw new Error(`Unexpected member sync status ${row.memberSyncStatus}`) + throw new OrchestrationError( + 'conflict', + `Unexpected member sync status ${row.memberSyncStatus}` + ) } return viewerMembership ? [ @@ -440,10 +446,12 @@ export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ .where(eq(knowledgeConnectorMemberSyncLog.connectorId, context.connectorId)) .orderBy(desc(knowledgeConnectorMemberSyncLog.startedAt)) .limit(10), - summarizeConnectorMembers(context.connectorId, connector.syncIntervalMinutes), + connector.accessMode === 'members' + ? summarizeConnectorMembers(context.connectorId, connector.syncIntervalMinutes) + : { active: 0, suspended: 0, stale: 0 }, ]) const { encryptedApiKey: _encryptedApiKey, ...connectorData } = connector - const viewerUserId = resolvePrincipalSubjectUserId(principal) + const viewerUserId = principal.kind === 'session' ? principal.userId : null const memberships = viewerUserId && context.workspaceId ? await resolveViewerConnectorMemberships({ @@ -451,7 +459,7 @@ export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ workspaceId: context.workspaceId, connectors: [connector], }) - : new Map() + : new Map() return { connector: { ...connectorData, @@ -530,7 +538,7 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ 'A members-mode connector needs a signed-in admin' ) } - await requireWorkspaceRole(subjectUserId, context, 'admin') + await requireCurrentHumanRole(subjectUserId, context, 'admin') const connectorMeta = getConnectorMeta(input.connectorType) if (!connectorMeta) { throw new OrchestrationError('validation', `Unknown connector type: ${input.connectorType}`) diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index 13c5cca04df..95d5f072452 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -57,6 +57,7 @@ import type { KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' import { + attachKnowledgeBaseConnectors, createAuthorizedKnowledgeBase, deleteKnowledgeBase, getKnowledgeBaseById, @@ -345,7 +346,7 @@ async function executeReadKnowledgeBase(args: { { maxRows: MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE } ) return { - knowledgeBase: args.context.knowledgeBase, + knowledgeBase: await attachKnowledgeBaseConnectors(args.context.knowledgeBase), folderPath: knowledgeFolderPathForId(index, args.context.knowledgeBase.folderId), } } @@ -397,25 +398,15 @@ async function executeDeleteKnowledgeBase(args: { export const listKnowledgeBases = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.list, - resolveContext: ({ - principal, - input, - }: { - principal: Principal - input: ListKnowledgeBasesInput - }) => resolveKnowledgeWorkspaceContext(input), + resolveContext: ({ input }: { input: ListKnowledgeBasesInput }) => + resolveKnowledgeWorkspaceContext(input), execute: executeListKnowledgeBases, }) export const listKnowledgeBaseCatalog = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.list, - resolveContext: ({ - principal, - input, - }: { - principal: Principal - input: ListKnowledgeBasesInput - }) => resolveKnowledgeWorkspaceContext(input), + resolveContext: ({ input }: { input: ListKnowledgeBasesInput }) => + resolveKnowledgeWorkspaceContext(input), async execute({ input, context }): Promise { const result = await executeListKnowledgeBases({ input, context }) const knowledgeBaseIds = result.knowledgeBases.map(({ knowledgeBase }) => knowledgeBase.id) @@ -555,13 +546,8 @@ export const listInternalKnowledgeBases = { export const createKnowledgeBase = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.create, - resolveContext: ({ - principal, - input, - }: { - principal: Principal - input: CreateKnowledgeBaseInput - }) => resolveKnowledgeWorkspaceContext(input), + resolveContext: ({ input }: { input: CreateKnowledgeBaseInput }) => + resolveKnowledgeWorkspaceContext(input), execute: executeCreateKnowledgeBase, projectAudit: ({ input, result }) => ({ action: AuditAction.KNOWLEDGE_BASE_CREATED, diff --git a/apps/sim/lib/knowledge/connectors/member-access.ts b/apps/sim/lib/knowledge/connectors/member-access.ts index 2b2cfd71e5d..6428a8c55d9 100644 --- a/apps/sim/lib/knowledge/connectors/member-access.ts +++ b/apps/sim/lib/knowledge/connectors/member-access.ts @@ -378,8 +378,9 @@ function isCapFieldSet(value: unknown): boolean { } /** - * The source config without its listing caps. A cap has no meaning once a - * connector syncs per member, so the switch clears it rather than refusing a + * The source config with its listing caps set to 0, which every connector + * reads as unlimited (an absent cap falls back to a connector's default). A + * cap has no meaning once a connector syncs per member, so the switch clears it rather than refusing a * connector the admin can no longer see the field on. */ export function stripListingCapFields( @@ -389,7 +390,7 @@ export function stripListingCapFields( const capFieldIds = connectorMeta.permissionScopedListing?.capFieldIds ?? [] if (capFieldIds.length === 0) return sourceConfig const stripped = { ...sourceConfig } - for (const fieldId of capFieldIds) delete stripped[fieldId] + for (const fieldId of capFieldIds) stripped[fieldId] = 0 return stripped } diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index 8684c847259..0bed21ab984 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -20,6 +20,7 @@ import { sql, } from 'drizzle-orm' import type { DbOrTx } from '@/lib/db/types' +import { textArrayLiteral } from '@/lib/knowledge/access/predicate' import { MEMBER_OBSERVATION_STALE_AFTER_HOURS, MEMBER_PURGE_MAX_PER_RUN, @@ -168,6 +169,44 @@ export async function listObservedDocumentIds( * document id that was detached or re-owned since it was collected is left * alone. */ +/** Documents rewritten per statement while a mode switch rewrites a connector's ACLs. */ +const ACCESS_REWRITE_BATCH_SIZE = 1000 + +/** + * Rewrites every document ACL of the connector to `target`, in bounded + * batches, until done or `deadlineAt` passes. Returns whether every row was + * rewritten. `beforeBatch` runs ahead of each statement, for a lease heartbeat. + */ +export async function rewriteConnectorAcls( + connectorId: string, + target: readonly string[], + options: { deadlineAt?: number; beforeBatch?: () => Promise } = {} +): Promise { + const mismatch = + target.length === 0 + ? sql`cardinality(${document.acl}) > 0` + : sql`${document.acl} <> ${textArrayLiteral(target)}` + for (;;) { + await options.beforeBatch?.() + const rewritten = await db + .update(document) + .set({ acl: [...target] }) + .where( + eq( + document.id, + sql`ANY(ARRAY( + SELECT ${document.id} FROM ${document} + WHERE ${document.connectorId} = ${connectorId} AND ${mismatch} + LIMIT ${ACCESS_REWRITE_BATCH_SIZE} + ))` + ) + ) + .returning({ id: document.id }) + if (rewritten.length < ACCESS_REWRITE_BATCH_SIZE) return true + if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) return false + } +} + export async function materializeDocumentAcls( connectorId: string, documentIds: Iterable @@ -273,20 +312,22 @@ export async function applyMemberDocumentLifecycle(input: { .returning({ id: document.id }) const purgeCutoff = new Date(now.getTime() - MEMBER_TOMBSTONE_PURGE_DAYS * 24 * 60 * 60 * 1000) - const purgeCandidates = await db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNotNull(document.deletedAt), - lt(document.deletedAt, purgeCutoff), - hasNoObservation() - ) - ) - .limit(MEMBER_PURGE_MAX_PER_RUN) + const purgeCandidates = input.allowRemoval + ? await db + .select({ id: document.id }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNotNull(document.deletedAt), + lt(document.deletedAt, purgeCutoff), + hasNoObservation() + ) + ) + .limit(MEMBER_PURGE_MAX_PER_RUN) + : [] const guard: ConnectorSyncDeletionGuard = { connectorId, @@ -296,7 +337,7 @@ export async function applyMemberDocumentLifecycle(input: { } let purged = 0 const purgeIds = purgeCandidates.map((row) => row.id) - for (let offset = 0; input.allowRemoval && offset < purgeIds.length; offset += PURGE_CHUNK_SIZE) { + for (let offset = 0; offset < purgeIds.length; offset += PURGE_CHUNK_SIZE) { await input.lease.beatIfDue() purged += await hardDeleteDocuments( purgeIds.slice(offset, offset + PURGE_CHUNK_SIZE), @@ -321,9 +362,11 @@ export interface StaleMemberSweepResult { * Removes the observations of members whose crawls have stopped, so the * documents only they observed go dark instead of staying readable forever. * - * Fail-closed but schedule-relative: an active member is swept only when both - * their last start and their last complete listing are older than - * `max(24 h, 2 × interval)`, so queue lag in a large group never trips it. A + * Fail-closed but schedule-relative: an active member is swept only when the + * connector itself completed a run inside `max(24 h, 2 × interval)` while both + * the member's last start and last complete listing are older than that, so + * queue lag in a large group, a deferred connector, or one on its failure + * ladder never trips it. A * suspended member is not swept at all: suspension already drops their token * from every ACL, and their observations are kept so a re-auth restores access * without a re-crawl until membership reconciliation purges the row after @@ -348,10 +391,17 @@ export async function sweepStaleMemberObservations(now: Date): Promise { }) }) + describe('memberNextAttemptAt', () => { + const now = new Date('2026-09-01T12:00:00Z') + + it('is exactly one interval on, with no jitter, so the connector run finds the member due', () => { + expect(memberNextAttemptAt(now, 60)).toEqual(new Date('2026-09-01T13:00:00Z')) + }) + + it('waits for the next manual run on a manual-only connector', () => { + expect(memberNextAttemptAt(now, 0)).toBeNull() + }) + }) + describe('memberFailureBackoffMs', () => { it('doubles on the connector interval and caps at a day', () => { expect(memberFailureBackoffMs(1, 60)).toBe(60 * 60 * 1000) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 906ef87fc87..42df625f531 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -1,5 +1,6 @@ import { db } from '@sim/db' import { + credential, document, knowledgeBase, knowledgeConnector, @@ -10,7 +11,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { randomInt } from '@sim/utils/random' -import { and, eq, inArray, isNull, lte, sql } from 'drizzle-orm' +import { and, eq, inArray, isNull, lte, notExists, sql } from 'drizzle-orm' +import { LRUCache } from 'lru-cache' import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, @@ -34,6 +36,7 @@ import { recordMemberObservations, removeMemberObservationsForDocuments, removeUnseenMemberObservations, + rewriteConnectorAcls, } from '@/lib/knowledge/connectors/member-observations' import { inviteWorkspaceMembersToCredentialGroup } from '@/lib/knowledge/connectors/member-provisioning' import { @@ -86,10 +89,10 @@ const logger = createLogger('ConnectorMemberSyncEngine') const HYDRATION_OBSERVER_ATTEMPTS = 3 /** A minted token is reused for this long before the member is re-minted. */ const MEMBER_TOKEN_REUSE_MS = 45 * 60 * 1000 +/** Members whose tokens one run keeps at once; a memory backstop, not a working-set limit. */ +const MEMBER_TOKEN_CACHE_MAX = 10_000 /** Overlap subtracted from a member's incremental watermark, covering source clock skew. */ const INCREMENTAL_OVERLAP_MS = 5 * 60 * 1000 -/** Rows rewritten per statement while finishing a mode switch's ACL rewrite. */ -const ACCESS_REWRITE_BATCH_SIZE = 1000 /** Member rows read per page while reconciling membership. */ const MEMBER_CREDENTIAL_PAGE_SIZE = 100 /** Backoff ceiling for one member's failure ladder. */ @@ -152,6 +155,11 @@ interface MemberListingOutcome { * retry cannot improve on, such as a capped or truncated source. */ resumable: boolean + /** + * The member was the run's only claim and still ran out of budget: a listing + * no run can finish alone, so it backs off instead of re-dispatching forever. + */ + exhaustedRunAlone: boolean suspect: boolean /** Cursor to store when this outcome lands: a value, null to close the feed, undefined to leave it. */ changeCursor: string | null | undefined @@ -283,6 +291,15 @@ export function buildMemberSyncFailureUpdate( } } +/** + * When a member who completed is next due: exactly one interval on, with no + * jitter, so they are due whenever the connector's own (jittered) run lands. + * Null on a manual-only connector: with its next manual run. + */ +export function memberNextAttemptAt(now: Date, syncIntervalMinutes: number): Date | null { + return syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60_000) : null +} + /** The next scheduled run: immediately while members remain due, else the interval plus jitter. */ export function nextMemberSyncTime( now: Date, @@ -351,14 +368,12 @@ function createMemberTokenCache(input: { connectorConfig: Pick credentialIdByMemberId: Map }): MemberTokenCache { - const tokens = new Map() const { auth } = input.connectorConfig if (auth.mode !== 'oauth') throw new Error('Members mode requires an OAuth connector') - return { - async get(memberId) { - const cached = tokens.get(memberId) - if (cached && Date.now() - cached.mintedAtMs < MEMBER_TOKEN_REUSE_MS) - return cached.accessToken + const tokens = new LRUCache({ + max: MEMBER_TOKEN_CACHE_MAX, + ttl: MEMBER_TOKEN_REUSE_MS, + fetchMethod: async (memberId) => { const credentialId = input.credentialIdByMemberId.get(memberId) if (!credentialId) throw new Error(`Member ${memberId} has no credential in this run`) const minted = await mintKnowledgeConnectorMemberToken({ @@ -370,9 +385,15 @@ function createMemberTokenCache(input: { runId: input.run.runId, }) input.run.result.credentialsAudited += 1 - tokens.set(memberId, { accessToken: minted.accessToken, mintedAtMs: Date.now() }) return minted.accessToken }, + }) + return { + async get(memberId) { + const accessToken = await tokens.fetch(memberId) + if (!accessToken) throw new Error(`No token could be minted for member ${memberId}`) + return accessToken + }, } } @@ -420,27 +441,7 @@ async function insertMemberSyncLog(runId: string, connectorId: string, startedAt * makes it visible again. */ async function finishPendingAccessRewrite(run: MemberSyncRun): Promise { - for (;;) { - await run.lease.beatIfDue() - const rewritten = await db - .update(document) - .set({ acl: [...EMPTY_ACL] }) - .where( - and( - eq( - document.id, - sql`ANY(ARRAY( - SELECT ${document.id} FROM ${document} - WHERE ${document.connectorId} = ${run.connectorId} - AND cardinality(${document.acl}) > 0 - LIMIT ${ACCESS_REWRITE_BATCH_SIZE} - ))` - ) - ) - ) - .returning({ id: document.id }) - if (rewritten.length < ACCESS_REWRITE_BATCH_SIZE) break - } + await rewriteConnectorAcls(run.connectorId, EMPTY_ACL, { beforeBatch: run.lease.beatIfDue }) await db .update(knowledgeConnector) .set({ accessRewritePending: false, updatedAt: new Date() }) @@ -649,8 +650,11 @@ async function claimNextMember(run: MemberSyncRun): Promise { * connector's next run" — a member that completed on a manual-only connector * — and must not keep the connector re-dispatching itself. */ -async function countDueMembers(run: MemberSyncRun): Promise { - const [row] = await db +async function countDueMembers( + run: MemberSyncRun, + binding: { credentialGroupOptionId: string } +): Promise { + const [due] = await db .select({ count: sql`count(*)::int` }) .from(knowledgeConnectorMember) .where( @@ -660,7 +664,30 @@ async function countDueMembers(run: MemberSyncRun): Promise { lte(knowledgeConnectorMember.nextAttemptAt, new Date()) ) ) - return row?.count ?? 0 + /** An account that connected while this run was listing has no member row yet. */ + const [unenrolled] = await db + .select({ count: sql`count(*)::int` }) + .from(credential) + .where( + and( + eq(credential.workspaceId, run.workspaceId), + eq(credential.credentialGroupOptionId, binding.credentialGroupOptionId), + eq(credential.type, 'managed_oauth'), + eq(credential.managedOauthStatus, 'active'), + notExists( + db + .select({ one: sql`1` }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + eq(knowledgeConnectorMember.credentialId, credential.id) + ) + ) + ) + ) + ) + return (due?.count ?? 0) + (unenrolled?.count ?? 0) } async function recordMemberFailure( @@ -865,6 +892,7 @@ async function applyMemberListing( if (documentId) seenDocumentIds.push(documentId) } const removesAllowed = outcome.mode === 'full' && outcome.complete && !outcome.suspect + const exhaustedFailures = (outcome.member.consecutiveFailures ?? 0) + 1 const now = new Date() await db.transaction(async (tx) => { @@ -898,12 +926,18 @@ async function applyMemberListing( await tx .update(knowledgeConnectorMember) .set({ - consecutiveFailures: 0, - lastError: null, + ...(outcome.exhaustedRunAlone + ? { + consecutiveFailures: exhaustedFailures, + lastError: 'Listing did not finish within one run', + } + : { consecutiveFailures: 0, lastError: null }), ...(outcome.mode === 'full' ? { lastListedCount: outcome.listedCount } : {}), - nextAttemptAt: outcome.resumable - ? now - : nextMemberSyncTime(now, syncIntervalMinutes, false), + nextAttemptAt: outcome.exhaustedRunAlone + ? new Date(now.getTime() + memberFailureBackoffMs(exhaustedFailures, syncIntervalMinutes)) + : outcome.resumable + ? now + : memberNextAttemptAt(now, syncIntervalMinutes), ...(removesAllowed ? { lastCompleteListingAt: now, memberSyncedThrough: outcome.listingStartedAt } : {}), @@ -1372,6 +1406,8 @@ export async function executeMemberSync( listedCount: admitted.seenExternalIds.size, complete: listed.complete, resumable: listed.resumable, + exhaustedRunAlone: + listed.resumable && result.membersClaimed === 1 && Date.now() >= run.deadlineAt, suspect, /** A doubted listing does not open the feed either: the next full listing decides. */ changeCursor: suspect ? undefined : listed.changeCursor, @@ -1476,7 +1512,7 @@ export async function executeMemberSync( lease: run.lease, }) - result.membersRemaining = (await countDueMembers(run)) > 0 + result.membersRemaining = (await countDueMembers(run, binding)) > 0 const landed = await completeMemberSync(run, connector.syncIntervalMinutes) if (!landed) { logger.warn( @@ -1500,7 +1536,12 @@ export async function executeMemberSync( } if (error instanceof ConnectorDeletedException) { logger.info('Connector deleted during member sync', { connectorId }) - await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch(() => undefined) + await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch((logError) => + logger.error('Failed to record member sync failure', { + connectorId, + error: getErrorMessage(logError), + }) + ) return skipped(result, 'connector_deleted_during_sync') } if (error instanceof MemberBindingGoneError) { diff --git a/apps/sim/lib/knowledge/connectors/queue.test.ts b/apps/sim/lib/knowledge/connectors/queue.test.ts index 816dd4184d4..f7e7d0ec807 100644 --- a/apps/sim/lib/knowledge/connectors/queue.test.ts +++ b/apps/sim/lib/knowledge/connectors/queue.test.ts @@ -82,6 +82,7 @@ describe('connector sync queue', () => { { knowledgeBaseId: 'knowledge-base-1', connectorStatus: 'active', + connectorAccessMode: 'workspace', connectorArchivedAt: null, connectorDeletedAt: null, connectorNextSyncAt: NEXT_SYNC_AT, @@ -197,6 +198,7 @@ describe('connector sync queue', () => { { knowledgeBaseId: 'knowledge-base-1', connectorStatus: 'paused', + connectorAccessMode: 'workspace', connectorArchivedAt: null, connectorDeletedAt: null, workspaceId: 'workspace-paid', @@ -235,6 +237,7 @@ describe('connector sync queue', () => { { knowledgeBaseId: 'knowledge-base-1', connectorStatus: 'paused', + connectorAccessMode: 'workspace', connectorArchivedAt: null, connectorDeletedAt: null, connectorNextSyncAt: NEXT_SYNC_AT, diff --git a/apps/sim/lib/knowledge/connectors/queue.ts b/apps/sim/lib/knowledge/connectors/queue.ts index 3adc175eac9..4d4de0e463c 100644 --- a/apps/sim/lib/knowledge/connectors/queue.ts +++ b/apps/sim/lib/knowledge/connectors/queue.ts @@ -161,6 +161,7 @@ async function markSyncPending(connectorId: string): Promise { .where( and( eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.accessMode, 'workspace'), inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), isNull(knowledgeConnector.syncLockToken), connectorIsLive() @@ -346,7 +347,7 @@ export async function dispatchSync( }) return { queued: false, reason: 'Connector has been archived or deleted' } } - if (row.connectorAccessMode !== undefined && row.connectorAccessMode !== 'workspace') { + if (row.connectorAccessMode !== 'workspace') { logger.info('Skipping sync dispatch: connector syncs per member', { connectorId, requestId }) return { queued: false, diff --git a/apps/sim/lib/knowledge/connectors/sync-lock.ts b/apps/sim/lib/knowledge/connectors/sync-lock.ts index f8ce65f3f34..b920124c0ee 100644 --- a/apps/sim/lib/knowledge/connectors/sync-lock.ts +++ b/apps/sim/lib/knowledge/connectors/sync-lock.ts @@ -183,7 +183,7 @@ export interface SyncRunLease { /** * The lease of the content sync engine, held through `syncLockToken`. The - * heartbeat clock is seeded at lock acquisition, which wrote `updatedAt` itself. + * heartbeat clock is seeded at lock acquisition, which opened `syncLockLeaseAt`. */ export function createContentSyncLease(connectorId: string, syncLogId: string): SyncRunLease { let lastHeartbeatAtMs = Date.now() diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 0d9b3b869cb..11813f3a1d6 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -6,6 +6,7 @@ import { generateId } from '@sim/utils/id' import { and, eq, exists, isNull, sql } from 'drizzle-orm' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import type { DbOrTx } from '@/lib/db/types' +import { textArrayLiteral } from '@/lib/knowledge/access/predicate' import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' import type { DocumentData } from '@/lib/knowledge/documents/service' @@ -46,16 +47,7 @@ export async function restoreWorkspaceDocumentAcls( executor: DbOrTx, connectorId: string ): Promise { - /** - * The comparison array is assembled from scalar binds: the shared pool runs - * with `fetch_types: false`, under which a JS array bound as one parameter - * fails at execution (see packages/db/db.ts). The write itself goes through - * the typed column, which encodes the array literal itself. - */ - const workspaceAcl = sql`ARRAY[${sql.join( - WORKSPACE_ACL.map((token) => sql`${token}`), - sql`, ` - )}]::text[]` + const workspaceAcl = textArrayLiteral(WORKSPACE_ACL) const restored = await executor .update(document) .set({ acl: [...WORKSPACE_ACL] }) @@ -407,8 +399,8 @@ export async function persistSkippedRetryHashes( } /** - * Upload content to storage as a .txt file, create a document record, - * and trigger processing via the existing pipeline. + * Stores the document's bytes (see {@link connectorStoredArtifact}) and inserts + * its `pending` row; the caller dispatches processing. */ export async function addDocument( knowledgeBaseId: string, @@ -488,10 +480,7 @@ export async function addDocument( } } -/** - * Update an existing connector-sourced document with new content. - * Updates in-place to avoid unique constraint violations on (connectorId, externalId). - */ +/** The row a connector-owned document write may target: live, connector-owned, and not user-excluded. */ export function connectorDocumentSyncTarget( documentId: string, knowledgeBaseId: string, @@ -506,6 +495,10 @@ export function connectorDocumentSyncTarget( ) } +/** + * Update an existing connector-sourced document with new content. + * Updates in-place to avoid unique constraint violations on (connectorId, externalId). + */ export async function updateDocument( existingDocId: string, knowledgeBaseId: string, @@ -602,7 +595,6 @@ export async function updateDocument( throw error } - // Clean up old storage file and its ownership binding if (oldFileUrl) { try { const urlPath = new URL(oldFileUrl, 'http://localhost').pathname diff --git a/apps/sim/lib/knowledge/connectors/sync-primitives.ts b/apps/sim/lib/knowledge/connectors/sync-primitives.ts index 248069bad1e..f12afdad5dd 100644 --- a/apps/sim/lib/knowledge/connectors/sync-primitives.ts +++ b/apps/sim/lib/knowledge/connectors/sync-primitives.ts @@ -686,8 +686,8 @@ export function classifySuspectListing( /** * Decides whether a suspect listing may still reconcile deletions. * - * A suspect listing is only acted on after a consecutive suspect observation, so a - * consecutive sync, so a single transient upstream fault can never remove + * A suspect listing is only acted on after a consecutive sync observes the same + * thing, so a single transient upstream fault can never remove * documents — not even reversibly, since a soft delete hides them from search * immediately. A genuinely emptied source keeps reconciling: its second sync * corroborates the first and tombstones everything, and a later sync — once the @@ -1666,7 +1666,6 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { } } - // Record all skipped (oversized) docs in this batch in one bulk insert. if (skipOps.length > 0) { try { const recorded = await persistSkippedDocuments( @@ -1808,13 +1807,6 @@ export async function reconcileDeletions(input: ReconcileDeletionsInput): Promis let reconcileDeletionsAllowed = shouldReconcileDeletions(isIncremental, syncContext, fullSync) - /** - * Backstop shared by every connector: a listing that reports (almost) - * nothing while this connector still owns a real corpus is treated as a - * fault, not as evidence of deletion, until a consecutive sync sees the - * same thing. Only evaluated when reconciliation would otherwise run, so - * healthy syncs pay nothing and no existing gate is loosened. - */ /** * Counted over deletion-eligible rows on both sides. The live read filters * excluded documents in SQL; the tombstoned read only projects the flag, so @@ -1828,6 +1820,13 @@ export async function reconcileDeletions(input: ReconcileDeletionsInput): Promis * are absent from the live read, so they must not inflate the numerator. */ const listedDocCount = countNonExcludedListed(seenExternalIds, excludedExternalIds) + /** + * Backstop shared by every connector: a listing that reports (almost) + * nothing while this connector still owns a real corpus is treated as a + * fault, not as evidence of deletion, until a consecutive sync sees the + * same thing. Only evaluated when reconciliation would otherwise run, so + * healthy syncs pay nothing and no existing gate is loosened. + */ if (reconcileDeletionsAllowed && classifySuspectListing(listedDocCount, ownedDocCount)) { const previousObservation = await loadPreviousListingObservation( connectorId, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 4995f4bb382..73e530f3b63 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -3010,7 +3010,10 @@ export async function bulkDocumentOperation( .where( and( eq(document.knowledgeBaseId, knowledgeBaseId), - inArray(document.id, documentIds), + inArray( + document.id, + documentsToUpdate.map((doc) => doc.id) + ), eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index 2be3ade31fe..90662f66a65 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -1,9 +1,9 @@ import { db } from '@sim/db' -import { document, knowledgeConnector, knowledgeConnectorMember } from '@sim/db/schema' +import { knowledgeConnector, knowledgeConnectorMember } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { and, eq, inArray, isNull } from 'drizzle-orm' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' @@ -16,6 +16,7 @@ import { stripListingCapFields, validateKnowledgeConnectorMembersBinding, } from '@/lib/knowledge/connectors/member-access' +import { rewriteConnectorAcls } from '@/lib/knowledge/connectors/member-observations' import { provisionKnowledgeConnectorMembersBinding } from '@/lib/knowledge/connectors/member-provisioning' import { type ConnectorWithoutSecret, @@ -40,8 +41,6 @@ class SwitchLeaseLostError extends Error { } } -/** Documents rewritten per statement while switching modes. */ -const ACCESS_REWRITE_BATCH_SIZE = 1000 /** Wall-clock the request spends rewriting before handing the rest to the member run. */ const ACCESS_REWRITE_REQUEST_BUDGET_MS = 20_000 /** Connector statuses a switch may start from; a running or queued sync owns the row. */ @@ -118,43 +117,6 @@ export async function resolveKnowledgeConnectorMembersBinding(input: { return { ...binding, sourceConfig } } -/** - * Rewrites the connector's document ACLs to `target` in bounded batches until - * done or the budget runs out. Returns whether every row was rewritten. - */ -async function rewriteConnectorAcls( - connectorId: string, - target: readonly string[], - deadlineAt: number -): Promise { - const targetArray = sql`ARRAY[${sql.join( - target.map((token) => sql`${token}`), - sql`, ` - )}]::text[]` - const mismatch = - target.length === 0 - ? sql`cardinality(${document.acl}) > 0` - : sql`${document.acl} <> ${targetArray}` - for (;;) { - const rewritten = await db - .update(document) - .set({ acl: [...target] }) - .where( - eq( - document.id, - sql`ANY(ARRAY( - SELECT ${document.id} FROM ${document} - WHERE ${document.connectorId} = ${connectorId} AND ${mismatch} - LIMIT ${ACCESS_REWRITE_BATCH_SIZE} - ))` - ) - ) - .returning({ id: document.id }) - if (rewritten.length < ACCESS_REWRITE_BATCH_SIZE) return true - if (Date.now() >= deadlineAt) return false - } -} - /** * Takes the connector's content lease for the switch, so no sync of either * engine can start while documents are being rewritten. Returns the row as it @@ -333,7 +295,9 @@ export async function performUpdateKnowledgeConnectorAccess( params.userId ) try { - const rewritten = await rewriteConnectorAcls(connectorId, EMPTY_ACL, deadlineAt) + const rewritten = await rewriteConnectorAcls(connectorId, EMPTY_ACL, { + deadlineAt: deadlineAt, + }) const now = new Date() const [updated] = await db .update(knowledgeConnector) @@ -459,7 +423,9 @@ export async function performUpdateKnowledgeConnectorAccess( .returning({ id: knowledgeConnector.id }) if (!row) throw new SwitchLeaseLostError() }) - const rewritten = await rewriteConnectorAcls(connectorId, WORKSPACE_ACL, deadlineAt) + const rewritten = await rewriteConnectorAcls(connectorId, WORKSPACE_ACL, { + deadlineAt: deadlineAt, + }) const now = new Date() const [updated] = await db .update(knowledgeConnector) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index e10fd8cd837..2df2febbeaa 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -19,6 +19,7 @@ import { findListingCapViolation, grantKnowledgeConnectorCredentialAccess, revokeKnowledgeConnectorCredentialAccess, + stripListingCapFields, } from '@/lib/knowledge/connectors/member-access' import { allocateTagSlots } from '@/lib/knowledge/constants' import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' @@ -602,6 +603,7 @@ export async function performUpdateKnowledgeConnector( } } + let sourceConfigToStore = updates.sourceConfig if (updates.sourceConfig !== undefined) { if (existing.accessMode === 'members') { /** @@ -615,6 +617,9 @@ export async function performUpdateKnowledgeConnector( ? findListingCapViolation(connectorConfig, updates.sourceConfig) : null if (capViolation) return fail(capViolation, 'validation') + if (connectorConfig) { + sourceConfigToStore = stripListingCapFields(connectorConfig, updates.sourceConfig) + } } else if (validateSourceConfig) { const rejection = await validateSourceConfig(existing, updates.sourceConfig) if (rejection) { @@ -646,8 +651,8 @@ export async function performUpdateKnowledgeConnector( const values: Partial = { updatedAt: updateTimestamp, } - if (updates.sourceConfig !== undefined) { - values.sourceConfig = updates.sourceConfig + if (sourceConfigToStore !== undefined) { + values.sourceConfig = sourceConfigToStore } if (updates.syncIntervalMinutes !== undefined) { values.syncIntervalMinutes = updates.syncIntervalMinutes @@ -1028,6 +1033,12 @@ export async function performSyncKnowledgeConnector( return classifyKnowledgeFailure(error, requestId, `Sync connector ${connectorId}`) } + if (rehydrate && connector.accessMode === 'members') { + return fail( + 'A connector that syncs per member re-hydrates through its members; run a sync instead', + 'validation' + ) + } logger.info( `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` ) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 60fbfd8700c..ef9f7d9c2bd 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -1,10 +1,10 @@ import { db } from '@sim/db' import { document, embedding } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' +import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import { and, eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import { type KnowledgeAccessScope, WORKSPACE_ACCESS_TOKENS } from '@/lib/knowledge/access/types' import { applyRecencyBoost, RRF_K } from '@/lib/knowledge/search/recency' import { coerceTagFilterValue, @@ -17,6 +17,8 @@ const logger = createLogger('KnowledgeSearchQueries') /** SQLSTATE for an unrecognised configuration parameter — pgvector older than 0.8. */ const UNDEFINED_OBJECT_SQLSTATE = '42704' +/** Tuples a relaxed-order scan may visit before giving up on filling the limit. */ +const HNSW_MAX_SCAN_TUPLES = '20000' /** How long to stop trying the iterative-scan settings after the server rejected them. */ const HNSW_SETTINGS_UNSUPPORTED_RETRY_MS = 10 * 60 * 1000 @@ -36,18 +38,24 @@ type SearchExecutor = Pick * and the attempt is retried after a while so an upgrade is picked up. */ async function withVectorScanSettings( + access: KnowledgeAccessScope, run: (executor: SearchExecutor) => Promise ): Promise { - if (Date.now() < hnswSettingsUnsupportedUntil) return run(db) + /** + * The workspace pair matches every row, so a plain index scan already fills + * the limit; only a personal token set is selective enough to need the + * iterative scan, and existing workspaces keep the query they had. + */ + if (!hasSubjectTokens(access) || Date.now() < hnswSettingsUnsupportedUntil) return run(db) try { return await db.transaction(async (tx) => { await tx.execute( - sql`SELECT set_config('hnsw.iterative_scan', 'relaxed_order', true), set_config('hnsw.max_scan_tuples', '20000', true)` + sql`SELECT set_config('hnsw.iterative_scan', 'relaxed_order', true), set_config('hnsw.max_scan_tuples', ${HNSW_MAX_SCAN_TUPLES}, true)` ) return run(tx) }) } catch (error) { - if ((error as { code?: unknown } | null)?.code !== UNDEFINED_OBJECT_SQLSTATE) throw error + if (getPostgresErrorCode(error) !== UNDEFINED_OBJECT_SQLSTATE) throw error hnswSettingsUnsupportedUntil = Date.now() + HNSW_SETTINGS_UNSUPPORTED_RETRY_MS logger.warn('pgvector iterative scan is unavailable; vector legs run without it', { error: getErrorMessage(error), @@ -56,6 +64,11 @@ async function withVectorScanSettings( } } +/** Whether the caller holds tokens beyond the workspace pair every document carries. */ +function hasSubjectTokens(access: KnowledgeAccessScope): boolean { + return access.kind === 'user' && access.tokens.length > WORKSPACE_ACCESS_TOKENS.length +} + export interface DocumentMetadata { filename: string sourceUrl: string | null @@ -421,7 +434,7 @@ async function executeVectorSearchOnIds( return [] } - const rows = await withVectorScanSettings((executor) => + const rows = await withVectorScanSettings(access, (executor) => executor .select( getSearchResultFields( @@ -520,7 +533,7 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise { + const allResults = await withVectorScanSettings(access, async (executor) => { const parallelResults = await Promise.all( knowledgeBaseIds.map((kbId) => vectorLeg(executor, eq(embedding.knowledgeBaseId, kbId), parallelLimit) @@ -530,7 +543,7 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise a.distance - b.distance).slice(0, topK) } - const rows = await withVectorScanSettings((executor) => + const rows = await withVectorScanSettings(access, (executor) => vectorLeg(executor, inArray(embedding.knowledgeBaseId, knowledgeBaseIds), topK) ) return rows.sort((a, b) => a.distance - b.distance) diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index c8d6d9cc4fc..0688c24755a 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -29,6 +29,7 @@ import { import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRestoreName } from '@/lib/core/utils/restore-name' import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import type { ChunkingConfig, CreateKnowledgeBaseData, @@ -250,6 +251,21 @@ async function attachConnectorTypes( connectorTypesByKb.set(row.knowledgeBaseId, types) if (row.accessMode === 'members') memberScopedKbIds.add(row.knowledgeBaseId) } + /** + * A members-mode connector only scopes documents where the feature is on; + * off, its documents read as workspace-visible and the base must say so. + */ + const memberScopedWorkspaceIds = new Set( + knowledgeBases + .filter((kb) => memberScopedKbIds.has(kb.id) && kb.workspaceId) + .map((kb) => kb.workspaceId as string) + ) + for (const workspaceId of memberScopedWorkspaceIds) { + if (await isKnowledgeMemberAccessAvailable({ workspaceId })) continue + for (const kb of knowledgeBases) { + if (kb.workspaceId === workspaceId) memberScopedKbIds.delete(kb.id) + } + } return knowledgeBases.map((kb) => ({ ...kb, @@ -1008,13 +1024,24 @@ export async function getKnowledgeBaseById( return null } - const [withConnectors] = await attachConnectorTypes([ - { - ...result[0], - chunkingConfig: result[0].chunkingConfig as ChunkingConfig, - docCount: Number(result[0].docCount), - }, - ]) + return { + ...result[0], + chunkingConfig: result[0].chunkingConfig as ChunkingConfig, + docCount: Number(result[0].docCount), + connectorTypes: [], + hasMemberScopedConnector: false, + } +} + +/** + * The knowledge base with its connector summary, for the surfaces that show + * it. Kept off {@link getKnowledgeBaseById} so every operation that only + * resolves its context does not pay for the connector read. + */ +export async function attachKnowledgeBaseConnectors( + knowledgeBase: KnowledgeBaseWithCounts +): Promise { + const [withConnectors] = await attachConnectorTypes([knowledgeBase]) return withConnectors } diff --git a/packages/db/migrations/0318_permission_aware_knowledge.sql b/packages/db/migrations/0318_permission_aware_knowledge.sql index 64c258333fc..43479568c2c 100644 --- a/packages/db/migrations/0318_permission_aware_knowledge.sql +++ b/packages/db/migrations/0318_permission_aware_knowledge.sql @@ -115,7 +115,7 @@ COMMIT;--> statement-breakpoint -- '{ws}', 'workspace', 'idle', and has no member lease — so validation cannot fail. DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM "pg_constraint" WHERE "conname" = 'doc_acl_token_shape_check' AND "conrelid" = '"document"'::regclass) THEN - ALTER TABLE "document" ADD CONSTRAINT "doc_acl_token_shape_check" CHECK (array_position("document"."acl", NULL) IS NULL AND (cardinality("document"."acl") = 0 OR array_to_string("document"."acl", E'\n') ~ '^((ws|pub|link|u:[^\nA-Z]+@[^\nA-Z]+|[gs]:[^\n:]+:[^\n:]+:[^\n]+)(\n(ws|pub|link|u:[^\nA-Z]+@[^\nA-Z]+|[gs]:[^\n:]+:[^\n:]+:[^\n]+))*)$')) NOT VALID; + ALTER TABLE "document" ADD CONSTRAINT "doc_acl_token_shape_check" CHECK (array_position("document"."acl", NULL) IS NULL AND (cardinality("document"."acl") = 0 OR (cardinality("document"."acl") = array_length(string_to_array(array_to_string("document"."acl", E'\n'), E'\n'), 1) AND array_to_string("document"."acl", E'\n') ~ '^((ws|pub|link|u:[^\nA-Z]+@[^\nA-Z]+|[gs]:[^\n:]+:[^\n:]+:[^\n]+)(\n(ws|pub|link|u:[^\nA-Z]+@[^\nA-Z]+|[gs]:[^\n:]+:[^\n:]+:[^\n]+))*)$'))) NOT VALID; END IF; END $$;--> statement-breakpoint ALTER TABLE "document" VALIDATE CONSTRAINT "doc_acl_token_shape_check";--> statement-breakpoint diff --git a/packages/db/migrations/meta/0318_snapshot.json b/packages/db/migrations/meta/0318_snapshot.json index e3ec6a6cd7b..42fce3acfd7 100644 --- a/packages/db/migrations/meta/0318_snapshot.json +++ b/packages/db/migrations/meta/0318_snapshot.json @@ -5454,7 +5454,7 @@ "checkConstraints": { "doc_acl_token_shape_check": { "name": "doc_acl_token_shape_check", - "value": "array_position(\"document\".\"acl\", NULL) IS NULL AND (cardinality(\"document\".\"acl\") = 0 OR array_to_string(\"document\".\"acl\", E'\\n') ~ '^((ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+)(\\n(ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+))*)$')" + "value": "array_position(\"document\".\"acl\", NULL) IS NULL AND (cardinality(\"document\".\"acl\") = 0 OR (cardinality(\"document\".\"acl\") = array_length(string_to_array(array_to_string(\"document\".\"acl\", E'\\n'), E'\\n'), 1) AND array_to_string(\"document\".\"acl\", E'\\n') ~ '^((ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+)(\\n(ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+))*)$'))" } }, "isRLSEnabled": false diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 4ab1f0d953d..497a0f55a84 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -2854,7 +2854,7 @@ export const document = pgTable( */ aclTokenShapeCheck: check( 'doc_acl_token_shape_check', - sql`array_position(${table.acl}, NULL) IS NULL AND (cardinality(${table.acl}) = 0 OR array_to_string(${table.acl}, E'\\n') ~ '^((ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+)(\\n(ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+))*)$')` + sql`array_position(${table.acl}, NULL) IS NULL AND (cardinality(${table.acl}) = 0 OR (cardinality(${table.acl}) = array_length(string_to_array(array_to_string(${table.acl}, E'\\n'), E'\\n'), 1) AND array_to_string(${table.acl}, E'\\n') ~ '^((ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+)(\\n(ws|pub|link|u:[^\\nA-Z]+@[^\\nA-Z]+|[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+))*)$'))` ), // Search by filename filenameIdx: index('doc_filename_idx').on(table.filename), @@ -4793,8 +4793,9 @@ export const knowledgeConnectorMember = pgTable( consecutiveFailures: integer('consecutive_failures').notNull().default(0), /** * When the member is next due. NULL means "with the connector's next run": - * a new member, or one that completed on a manual-only connector. An explicit - * time that has passed is what keeps a connector re-dispatching itself. + * a member that completed on a manual-only connector. A new member is due + * now. An explicit time that has passed is what keeps a connector + * re-dispatching itself. */ nextAttemptAt: timestamp('next_attempt_at'), lastStartedAt: timestamp('last_started_at'), From 88fbe59cf80df122c5b561c4049619b57f1c1259 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 23:49:51 -0700 Subject: [PATCH 25/76] feat(knowledge): search results a person can open, from Chat - The composer's Search mode searches every knowledge base as the signed-in person and lists what they may read as result cards: source icon, title linking back to the document, knowledge base, updated date, and the matching passage with the query terms in bold; Summarize hands a document to the agent in Build mode - The agent's knowledge tool returns each result's title, link, connector, and modified time and is told to cite with source tags carrying a snippet; a reply whose sources carry snippets ends with the same cards - A session route for the search, bound to the shared search use case, so the browser reads through the same access predicate as everything else - Search quality: hybrid legs over-fetch before fusion, the vector leg's iterative scan fills a limit past the default candidate pool, and the recency weight moves a fresh document a few places rather than the list - A manual member sync makes every active member due, so Sync members now lists everyone instead of nobody --- apps/sim/app/api/knowledge/search/route.ts | 47 +++++++ .../app/api/knowledge/search/utils.test.ts | 13 +- .../knowledge-search-results/index.ts | 1 + .../knowledge-search-results.tsx | 129 +++++++++++++++++ .../message-content/components/index.ts | 1 + .../message-sources/message-sources.tsx | 35 ++++- .../components/source-card/index.ts | 1 + .../components/source-card/source-card.tsx | 130 ++++++++++++++++++ .../components/source-chip/index.ts | 2 +- .../components/source-chip/source-chip.tsx | 2 +- .../components/special-tags/special-tags.tsx | 6 + .../message-content/message-content.tsx | 5 +- .../mothership-chat/mothership-chat.tsx | 8 ++ .../app/workspace/[workspaceId]/home/home.tsx | 43 +++++- apps/sim/hooks/queries/kb/knowledge.ts | 39 ++++++ .../sim/hooks/queries/utils/knowledge-keys.ts | 8 ++ .../sim/lib/api/contracts/knowledge/search.ts | 52 ++++++- .../server/knowledge/knowledge-base.test.ts | 2 +- .../tools/server/knowledge/knowledge-base.ts | 18 ++- apps/sim/lib/knowledge/application/search.ts | 8 ++ .../lib/knowledge/orchestration/connectors.ts | 17 +++ apps/sim/lib/knowledge/search/queries.ts | 60 ++++++-- apps/sim/lib/knowledge/search/recency.ts | 2 +- 23 files changed, 592 insertions(+), 37 deletions(-) create mode 100644 apps/sim/app/api/knowledge/search/route.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts new file mode 100644 index 00000000000..3f50b2f384c --- /dev/null +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -0,0 +1,47 @@ +import { searchWorkspaceKnowledgeContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { searchKnowledge } from '@/lib/knowledge/application/search' + +export const POST = defineInternalJsonRoute({ + contract: searchWorkspaceKnowledgeContract, + auth: internalSessionAuth, + operation: knowledgeOperations.search, + rateLimit: internalRateLimits.none({ + reason: 'A person typing queries; the embedding call is metered against their workspace', + }), + errorPolicy: internalKnowledgeErrorPolicies.search, + mapInput: ({ body }) => ({ + workspaceId: body.workspaceId, + knowledgeBaseIds: body.knowledgeBaseIds, + query: body.query, + topK: body.topK, + }), + useCase: searchKnowledge, + present: ({ results, knowledgeBases }, { input }) => { + const knowledgeBaseNames = new Map(knowledgeBases.map((kb) => [kb.id, kb.name])) + return { + success: true as const, + data: { + query: input.query ?? '', + results: results.map((result) => ({ + documentId: result.documentId, + knowledgeBaseId: result.knowledgeBaseId, + knowledgeBaseName: knowledgeBaseNames.get(result.knowledgeBaseId) ?? '', + documentName: result.documentName, + sourceUrl: result.sourceUrl, + connectorType: result.connectorType, + sourceModifiedAt: result.sourceModifiedAt?.toISOString() ?? null, + content: result.content, + chunkIndex: result.chunkIndex, + similarity: result.similarity, + })), + }, + } + }, +}) diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 7d97da6c79f..169aac67c7d 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -510,12 +510,13 @@ describe('Knowledge Search Utils', () => { it('runs both legs and fuses them in hybrid mode', async () => { /** - * Chains dequeue in creation order. A workspace-scoped vector leg selects - * directly (the iterative scan is reserved for a personal token set), so - * its select is built first, then the keyword ranking pass, then hydration. + * Chains dequeue in creation order. Hybrid legs over-fetch past the + * plain scan's candidate pool, so the vector leg opens its transaction + * and applies the scan settings before selecting: the keyword ranking + * pass is built first, then the vector select, then hydration. */ - queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) queueTableRows(schemaMock.embedding, [{ id: 'keyword-hit', keywordRank: 0.9 }]) + queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) queueTableRows(schemaMock.embedding, [makeResult('keyword-hit')]) const results = await executeKnowledgeSearch({ @@ -532,9 +533,9 @@ describe('Knowledge Search Utils', () => { }) it('falls back to vector results when the keyword leg fails', async () => { - queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) - /** The failing ranking chain is still built and takes the second queued set. */ + /** The failing ranking chain is still built first and takes the first queued set. */ queueTableRows(schemaMock.embedding, [{ id: 'never-ranked', keywordRank: 0 }]) + queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) /** * Both legs share one `orderBy` spy, so target the keyword leg by its diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts new file mode 100644 index 00000000000..d716af02066 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts @@ -0,0 +1 @@ +export { groupResultsByDocument, KnowledgeSearchResults } from './knowledge-search-results' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx new file mode 100644 index 00000000000..032623d712d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -0,0 +1,129 @@ +'use client' + +import { useMemo } from 'react' +import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' +import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' +import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { useKnowledgeBasesQuery, useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' + +/** A search spans at most this many knowledge bases. */ +const MAX_SEARCHED_KNOWLEDGE_BASES = 20 +/** Characters of the matching chunk shown under a result. */ +const SNIPPET_LENGTH = 280 + +function toSnippet(content: string): string { + const flat = content.replace(/\s+/g, ' ').trim() + return flat.length > SNIPPET_LENGTH ? `${flat.slice(0, SNIPPET_LENGTH).trimEnd()}…` : flat +} + +/** + * One card per document, keeping the best-ranked chunk of each: the list is + * already in rank order, so the first chunk seen for a document is its best. + */ +export function groupResultsByDocument( + results: readonly WorkspaceKnowledgeSearchResult[] +): WorkspaceKnowledgeSearchResult[] { + const seen = new Set() + const grouped: WorkspaceKnowledgeSearchResult[] = [] + for (const result of results) { + if (seen.has(result.documentId)) continue + seen.add(result.documentId) + grouped.push(result) + } + return grouped +} + +/** A result as the source card renders it; a document without a source URL cannot be opened. */ +function toSource(result: WorkspaceKnowledgeSearchResult): SourceTagData | null { + if (!result.sourceUrl) return null + return { + url: result.sourceUrl, + title: result.documentName ?? undefined, + siteName: result.knowledgeBaseName || undefined, + connectorType: result.connectorType ?? undefined, + snippet: toSnippet(result.content), + updatedAt: result.sourceModifiedAt ?? undefined, + } +} + +interface KnowledgeSearchResultsProps { + workspaceId: string + query: string + /** Asks the agent about one document; the prompt names it and links to it. */ + onSummarize: (prompt: string) => void +} + +/** + * The composer's Search mode: the documents the signed-in person may read that + * match their query, across every knowledge base in the workspace, as cards + * that open the source. Summarize hands one document to the agent. + */ +export function KnowledgeSearchResults({ + workspaceId, + query, + onSummarize, +}: KnowledgeSearchResultsProps) { + const { data: knowledgeBases = [], isPending: basesPending } = useKnowledgeBasesQuery(workspaceId) + const knowledgeBaseIds = useMemo( + () => knowledgeBases.slice(0, MAX_SEARCHED_KNOWLEDGE_BASES).map((kb) => kb.id), + [knowledgeBases] + ) + const { + data: results, + isPending, + isFetching, + error, + } = useWorkspaceKnowledgeSearch(workspaceId, knowledgeBaseIds, query) + const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) + + if (!basesPending && knowledgeBaseIds.length === 0) { + return ( +

+ No knowledge bases to search yet. Add one from the Knowledge tab. +

+ ) + } + if (error) { + return

{error.message}

+ } + if (isPending || (isFetching && !results)) { + return

Searching…

+ } + if (documents.length === 0) { + return ( +

+ No documents you can read match “{query}”. +

+ ) + } + + return ( +
+ {documents.map((result) => { + const source = toSource(result) + return source ? ( + + onSummarize(`Summarize "${cited.title ?? cited.url}" (${cited.url})`) + } + /> + ) : ( +
+

+ {result.documentName ?? 'Untitled document'} +

+

+ {result.knowledgeBaseName} +

+

+ {toSnippet(result.content)} +

+
+ ) + })} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/index.ts index 853c79b1eea..07f0139dc83 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/index.ts @@ -4,5 +4,6 @@ export { ChatContent } from './chat-content' export { MessageSources } from './message-sources' export { Options } from './options' export { QuestionDisplay } from './question' +export { highlightTerms, SourceCard } from './source-card' export { SourceChip, sourceLabel } from './source-chip' export { PendingTagIndicator, parseSpecialTags, SpecialTags } from './special-tags' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx index 8c72089296e..522a8837b11 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx @@ -1,6 +1,7 @@ 'use client' import { cn } from '@sim/emcn' +import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' import { SourceChip } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-chip' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' @@ -14,17 +15,41 @@ const STRIP_FADE_CLASSES = interface MessageSourcesProps { sources: readonly SourceTagData[] + /** The question the reply answers; its terms are bolded in result cards. */ + query?: string + /** Asks the agent about one cited document, when the surface can send a message. */ + onSummarize?: (prompt: string) => void } /** - * Footer strip listing every document a reply cited, once each: one - * horizontally scrolling row of {@link SourceChip}s that fades out at the right - * edge instead of wrapping, so a long list stays a single quiet line under the - * answer. + * Footer listing every document a reply cited, once each. Sources that carry + * a snippet — a search answer — are laid out as result cards; otherwise one + * horizontally scrolling row of {@link SourceChip}s that fades out at the + * right edge instead of wrapping, so a long list stays a single quiet line + * under the answer. */ -export function MessageSources({ sources }: MessageSourcesProps) { +export function MessageSources({ sources, query, onSummarize }: MessageSourcesProps) { if (sources.length === 0) return null + if (sources.some((source) => source.snippet)) { + return ( +
+ {sources.map((source) => ( + onSummarize(`Summarize "${cited.title ?? cited.url}" (${cited.url})`) + : undefined + } + /> + ))} +
+ ) + } + return (
term.trim()) + .filter((term) => term.length >= MIN_HIGHLIGHT_TERM_LENGTH) + ), + ] + if (terms.length === 0) return text + const pattern = new RegExp(`\\b(${terms.map(escapeRegExp).join('|')})\\b`, 'gi') + const parts = text.split(pattern) + return parts.map((part, index) => + index % 2 === 1 ? ( + + {part} + + ) : ( + part + ) + ) +} + +function parseUpdatedAt(value: string | undefined): Date | null { + if (!value) return null + const date = new Date(value) + return Number.isNaN(date.getTime()) ? null : date +} + +interface SourceCardProps { + source: SourceTagData + /** The query the document was found for; its terms are bolded in the snippet. */ + query?: string + /** Offers a Summarize action that asks the agent about this document. */ + onSummarize?: (source: SourceTagData) => void +} + +/** + * One document a search found, laid out to be scanned: the source's brand + * mark or favicon, the title as a link back to the document, where it lives + * and when it last changed, and the passage that matched with the query terms + * in bold. The same card serves the composer's search results and the + * footer of a reply that cited its sources with a snippet. + */ +export function SourceCard({ source, query, onSummarize }: SourceCardProps) { + const hostname = externalLinkHostname(source.url) + const ConnectorIcon = source.connectorType + ? BRAND_ICON_BY_BASE_TYPE.get(source.connectorType) + : undefined + const updatedAt = parseUpdatedAt(source.updatedAt) + const meta = [sourceLabel(source), updatedAt ? `Updated ${formatDate(updatedAt)}` : null].filter( + (part): part is string => Boolean(part) + ) + + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts index 4168fe2a584..329fa2848eb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts @@ -1 +1 @@ -export { SourceChip, sourceLabel } from './source-chip' +export { BRAND_ICON_BY_BASE_TYPE, SourceChip, sourceLabel } from './source-chip' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx index 4101ddffb17..03d00442aa1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx @@ -19,7 +19,7 @@ import { BrandIcon, type StyleableIcon } from '@/blocks/brand-icon' * which would drag seventy connector modules into every surface that renders * chat. Versioned catalog types (`gmail_v2`) collapse onto their base name. */ -const BRAND_ICON_BY_BASE_TYPE: ReadonlyMap = new Map( +export const BRAND_ICON_BY_BASE_TYPE: ReadonlyMap = new Map( Object.entries(blockTypeToIconMap).map(([type, icon]) => [stripVersionSuffix(type), icon]) ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 0328463259a..f44ac7e935b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -343,6 +343,10 @@ export interface SourceTagData { * without it the chip shows the site favicon. */ connectorType?: string + /** The passage the reply relied on; a reply whose sources carry one is listed as result cards. */ + snippet?: string + /** When the source last changed the document, as an ISO timestamp. */ + updatedAt?: string } export type ContentSegment = @@ -577,6 +581,8 @@ function isSourceTagData(value: unknown): value is SourceTagData { if (value.title !== undefined && typeof value.title !== 'string') return false if (value.siteName !== undefined && typeof value.siteName !== 'string') return false if (value.connectorType !== undefined && typeof value.connectorType !== 'string') return false + if (value.snippet !== undefined && typeof value.snippet !== 'string') return false + if (value.updatedAt !== undefined && typeof value.updatedAt !== 'string') return false return true } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx index 19ca71fc900..d08acaf60ac 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx @@ -834,6 +834,8 @@ interface MessageContentProps { onOptionSelect?: (id: string) => void onQuestionDismiss?: () => void onPhaseChange?: (phase: MessagePhase) => void + /** The user message this reply answers, for the result cards' highlighting. */ + userQuery?: string /** * The message's actions row (copy/thumbs). Rendered here, in the thinking * slot's position, so at settle the shimmer and the actions trade places in @@ -854,6 +856,7 @@ function MessageContentInner({ credentialSubmission, credentialAbandoned, onOptionSelect, + userQuery, onQuestionDismiss, onPhaseChange, actions, @@ -1029,7 +1032,7 @@ function MessageContentInner({ })} {sources.length > 0 && (
- +
)}
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx index f4f948f429f..9449cb13d7d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx @@ -2,6 +2,7 @@ import { memo, + type ReactNode, useCallback, useDeferredValue, useEffect, @@ -60,6 +61,8 @@ interface MothershipChatProps { workspaceId: string messages: ChatMessage[] isSending: boolean + /** The composer's Search-mode results, shown above the input. */ + searchResults?: ReactNode isReconnecting?: boolean isLoading?: boolean onSubmit: ( @@ -288,6 +291,7 @@ const AssistantMessageRow = memo(function AssistantMessageRow({ credentialSubmission={credentialSubmission} credentialAbandoned={credentialAbandoned} onOptionSelect={onOptionSelect} + userQuery={precedingUserContent} onQuestionDismiss={handleQuestionDismiss} onPhaseChange={setPhase} actions={ @@ -312,6 +316,7 @@ export function MothershipChat({ workspaceId, messages: messagesProp, isSending, + searchResults, isReconnecting = false, isLoading = false, onSubmit, @@ -817,6 +822,9 @@ export function MothershipChat({ onAnimationEnd={animateInput ? onInputAnimationEnd : undefined} >
+ {searchResults && ( +
{searchResults}
+ )} state.mode) const hasCheckedLandingStorageRef = useRef(false) const initialViewInputRef = useRef(null) const initialViewUserInputRef = useRef(null) @@ -440,6 +444,12 @@ export function Home({ chatId, userName, userId }: HomeProps) { is_new_task: !chatId, }) + /** Search mode answers with documents, not a turn of the agent. */ + if (useMothershipModeStore.getState().mode === 'search' && trimmed) { + setSearchQuery(trimmed) + return + } + if (initialViewInputRef.current) { setIsInputEntering(true) } @@ -450,6 +460,24 @@ export function Home({ chatId, userName, userId }: HomeProps) { [workspaceId, chatId, prepareResourceViewForAgentTurn, sendMessage] ) + /** Summarize on a result: hand the document to the agent in Build mode. */ + const handleSummarize = useCallback( + (prompt: string) => { + useMothershipModeStore.getState().setMode('build') + setSearchQuery('') + handleSubmit(prompt) + }, + [handleSubmit] + ) + const showSearchResults = composerMode === 'search' && searchQuery.length > 0 + const searchResults = showSearchResults ? ( + + ) : null + /** * Handles cross-surface send requests (terminal/console "Fix in Chat", the * log "Troubleshoot in Chat" action). `preventDefault` claims the event so a @@ -674,11 +702,13 @@ export function Home({ chatId, userName, userId }: HomeProps) { {/* Anchored out of flow so expanding/collapsing never shifts the centered input */}
- - initialViewUserInputRef.current?.populatePrompt(prompt) - } - /> + {searchResults ?? ( + + initialViewUserInputRef.current?.populatePrompt(prompt) + } + /> + )}
@@ -688,6 +718,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { workspaceId={workspaceId} messages={messages} isSending={isSending} + searchResults={searchResults} isReconnecting={isReconnecting} isLoading={showChatSkeleton} onSubmit={handleSubmit} diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 4c064e479c9..fe885869cfa 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -41,6 +41,7 @@ import { restoreKnowledgeBaseContract, type SaveDocumentTagDefinitionsResult, saveDocumentTagDefinitionsContract, + searchWorkspaceKnowledgeContract, type TagDefinitionData, type TagUsageData, type UpdateKnowledgeDocumentResponseData, @@ -48,6 +49,8 @@ import { updateKnowledgeChunkContract, updateKnowledgeDocumentContract, updateKnowledgeDocumentTagsContract, + type WorkspaceKnowledgeSearchBody, + type WorkspaceKnowledgeSearchResult, } from '@/lib/api/contracts/knowledge' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' @@ -74,6 +77,7 @@ export const KNOWLEDGE_DOCUMENT_DETAIL_STALE_TIME = 60 * 1000 export const KNOWLEDGE_DOCUMENT_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_SEARCH_STALE_TIME = 60 * 1000 +export const WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME = 60 * 1000 export const KNOWLEDGE_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_TAG_USAGE_STALE_TIME = 60 * 1000 export const KNOWLEDGE_DOCUMENT_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 @@ -1153,3 +1157,38 @@ export function useBulkDeleteKnowledgeBases(workspaceId: string) { }, }) } + +async function searchWorkspaceKnowledge( + body: WorkspaceKnowledgeSearchBody, + signal?: AbortSignal +): Promise { + const data = await requestJson(searchWorkspaceKnowledgeContract, { body, signal }) + return data.data.results +} + +/** + * What the signed-in person may read that matches `query`, across the given + * knowledge bases. Off until there is a query and a base to search. + */ +export function useWorkspaceKnowledgeSearch( + workspaceId: string | undefined, + knowledgeBaseIds: readonly string[], + query: string +) { + const trimmed = query.trim() + return useQuery({ + queryKey: knowledgeKeys.search(workspaceId, knowledgeBaseIds, trimmed), + queryFn: ({ signal }) => + searchWorkspaceKnowledge( + { + workspaceId: workspaceId as string, + knowledgeBaseIds: [...knowledgeBaseIds], + query: trimmed, + }, + signal + ), + enabled: Boolean(workspaceId) && knowledgeBaseIds.length > 0 && trimmed.length > 0, + staleTime: WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME, + placeholderData: keepPreviousData, + }) +} diff --git a/apps/sim/hooks/queries/utils/knowledge-keys.ts b/apps/sim/hooks/queries/utils/knowledge-keys.ts index e47704214d7..b4d2611d14a 100644 --- a/apps/sim/hooks/queries/utils/knowledge-keys.ts +++ b/apps/sim/hooks/queries/utils/knowledge-keys.ts @@ -30,6 +30,14 @@ export const knowledgeKeys = { details: () => [...knowledgeKeys.all, 'detail'] as const, detail: (knowledgeBaseId?: string) => [...knowledgeKeys.details(), knowledgeBaseId ?? ''] as const, + searches: () => [...knowledgeKeys.all, 'search'] as const, + search: (workspaceId: string | undefined, knowledgeBaseIds: readonly string[], query: string) => + [ + ...knowledgeKeys.searches(), + workspaceId ?? '', + [...knowledgeBaseIds].sort().join(','), + query, + ] as const, tagDefinitions: (knowledgeBaseId: string) => [...knowledgeKeys.detail(knowledgeBaseId), 'tagDefinitions'] as const, tagUsage: (knowledgeBaseId: string) => diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index a236923eeae..cd6abd94584 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -1,5 +1,8 @@ import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { + resolvedSecretTraceProvenanceSchema, + workspaceIdSchema, +} from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' import { DEFAULT_RERANKER_MODEL, rerankerModelSchema } from '@/lib/knowledge/reranker-models' @@ -151,3 +154,50 @@ export const internalKnowledgeSearchContract = defineRouteContract({ }), }, }) + +/** One document a workspace search matched, with the best chunk of it. */ +export const workspaceKnowledgeSearchResultSchema = z.object({ + documentId: z.string(), + knowledgeBaseId: z.string(), + knowledgeBaseName: z.string(), + documentName: z.string().nullable(), + sourceUrl: z.string().nullable(), + connectorType: z.string().nullable(), + sourceModifiedAt: z.string().nullable(), + content: z.string(), + chunkIndex: z.number(), + similarity: z.number(), +}) +export type WorkspaceKnowledgeSearchResult = z.output + +export const workspaceKnowledgeSearchBodySchema = z.object({ + workspaceId: workspaceIdSchema, + knowledgeBaseIds: z + .array(z.string().min(1, 'knowledgeBaseId cannot be empty')) + .min(1, 'At least one knowledge base is required') + .max(20, 'A search spans at most 20 knowledge bases'), + query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'), + topK: z.number().int().min(1).max(50).optional().default(20), +}) +export type WorkspaceKnowledgeSearchBody = z.input + +/** + * The search a signed-in person runs from the composer: what their own + * account may read across the workspace's knowledge bases, presented as + * documents to open rather than chunks to feed a model. + */ +export const searchWorkspaceKnowledgeContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/search', + body: workspaceKnowledgeSearchBodySchema, + response: { + mode: 'json', + schema: z.object({ + success: z.literal(true), + data: z.object({ + query: z.string(), + results: z.array(workspaceKnowledgeSearchResultSchema), + }), + }), + }, +}) diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts index 8c3757a4cb2..bf6cadf13c4 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts @@ -401,7 +401,7 @@ describe('manage_knowledge_base trusted application delegation', () => { workspaceId: 'workspace-paid', knowledgeBaseIds: [KNOWLEDGE_BASE.id], query: '{{KB_QUERY}}', - topK: 5, + topK: 10, resultSecretRegistry: registry, }) expect(mockReadKnowledgeBase).not.toHaveBeenCalled() diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index c65ad755b66..56d6c48151e 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -57,6 +57,16 @@ import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-sec const logger = createLogger('KnowledgeBaseServerTool') +/** Results a query returns unless the caller asks for a number. */ +const DEFAULT_QUERY_TOP_K = 10 +/** + * How the model cites a knowledge result in its reply. The `` tag is + * what the chat renders as a link back to the document, so a result without + * a source URL is quoted by name instead. + */ +const KNOWLEDGE_CITATION_INSTRUCTION = + 'Cite each result you use inline, right after the sentence it supports, as {"url":,"title":,"siteName":,"connectorType":,"snippet":,"updatedAt":}; omit the tag for a result whose sourceUrl is null and name the document instead.' + /** * Resolves an environment-variable reference passed as a connector API key. * @@ -405,7 +415,7 @@ export const knowledgeBaseServerTool: BaseServerTool ({ documentId: result.documentId, + documentName: result.documentName, + sourceUrl: result.sourceUrl, + sourceModifiedAt: result.sourceModifiedAt?.toISOString() ?? null, + connectorType: result.connectorType, content: result.content, chunkIndex: result.chunkIndex, similarity: result.similarity, diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 2b39c26da7f..e8d025cb919 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -117,6 +117,10 @@ export interface KnowledgeSearchItem { documentId: string documentName: string | null sourceUrl: string | null + /** When the source last changed the document; null for uploads and sources that do not say. */ + sourceModifiedAt: Date | null + /** The connector the document was synced through; null for an upload. */ + connectorType: string | null content: string chunkIndex: number metadata: Record @@ -509,6 +513,10 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ documentId: row.documentId, documentName: document?.filename ?? null, sourceUrl: document?.sourceUrl ?? null, + sourceModifiedAt: + document && 'sourceModifiedAt' in document ? (document.sourceModifiedAt ?? null) : null, + connectorType: + document && 'connectorType' in document ? (document.connectorType ?? null) : null, content: row.content, chunkIndex: row.chunkIndex, metadata, diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index 2df2febbeaa..c8e4bd60073 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -6,6 +6,7 @@ import { knowledgeBase, knowledgeBaseTagDefinitions, knowledgeConnector, + knowledgeConnectorMember, } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' @@ -1052,6 +1053,22 @@ export async function performSyncKnowledgeConnector( * outcome and both records describe what actually happened. */ try { + /** + * A manual run is meant to list everyone now, so every active member is + * made due; otherwise each waits out its own interval and the run claims + * nobody. + */ + if (connector.accessMode === 'members') { + await db + .update(knowledgeConnectorMember) + .set({ nextAttemptAt: new Date(), updatedAt: new Date() }) + .where( + and( + eq(knowledgeConnectorMember.connectorId, connectorId), + eq(knowledgeConnectorMember.status, 'active') + ) + ) + } const dispatch = connector.accessMode === 'members' ? await (await loadDispatchMemberSync())(connectorId, { billingAttribution, requestId }) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index ef9f7d9c2bd..57cece6f4c6 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import { document, embedding } from '@sim/db/schema' +import { document, embedding, knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import { and, eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' @@ -19,6 +19,8 @@ const logger = createLogger('KnowledgeSearchQueries') const UNDEFINED_OBJECT_SQLSTATE = '42704' /** Tuples a relaxed-order scan may visit before giving up on filling the limit. */ const HNSW_MAX_SCAN_TUPLES = '20000' +/** pgvector's default `hnsw.ef_search`: the candidates a plain scan yields before predicates. */ +const HNSW_DEFAULT_EF_SEARCH = 40 /** How long to stop trying the iterative-scan settings after the server rejected them. */ const HNSW_SETTINGS_UNSUPPORTED_RETRY_MS = 10 * 60 * 1000 @@ -39,14 +41,17 @@ type SearchExecutor = Pick */ async function withVectorScanSettings( access: KnowledgeAccessScope, + limit: number, run: (executor: SearchExecutor) => Promise ): Promise { /** - * The workspace pair matches every row, so a plain index scan already fills - * the limit; only a personal token set is selective enough to need the - * iterative scan, and existing workspaces keep the query they had. + * A plain index scan yields `hnsw.ef_search` candidates (40 by default) + * before the predicates apply. That fills a small limit for the workspace + * pair, which matches every row; a personal token set, or a limit past the + * pool, needs the iterative scan to keep going until the limit is met. */ - if (!hasSubjectTokens(access) || Date.now() < hnswSettingsUnsupportedUntil) return run(db) + const needsIterativeScan = hasSubjectTokens(access) || limit > HNSW_DEFAULT_EF_SEARCH + if (!needsIterativeScan || Date.now() < hnswSettingsUnsupportedUntil) return run(db) try { return await db.transaction(async (tx) => { await tx.execute( @@ -72,6 +77,10 @@ function hasSubjectTokens(access: KnowledgeAccessScope): boolean { export interface DocumentMetadata { filename: string sourceUrl: string | null + /** When the source last changed the document; null for uploads and sources that do not say. */ + sourceModifiedAt: Date | null + /** The connector the document was synced through; null for an upload. */ + connectorType: string | null } /** @@ -95,8 +104,11 @@ export async function getDocumentMetadataByIds( id: document.id, filename: document.filename, sourceUrl: document.sourceUrl, + sourceModifiedAt: document.sourceModifiedAt, + connectorType: knowledgeConnector.connectorType, }) .from(document) + .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) .where( and( inArray(document.id, uniqueIds), @@ -109,7 +121,12 @@ export async function getDocumentMetadataByIds( const map: Record = {} documents.forEach((doc) => { - map[doc.id] = { filename: doc.filename, sourceUrl: doc.sourceUrl ?? null } + map[doc.id] = { + filename: doc.filename, + sourceUrl: doc.sourceUrl ?? null, + sourceModifiedAt: doc.sourceModifiedAt ?? null, + connectorType: doc.connectorType ?? null, + } }) return map @@ -392,6 +409,13 @@ function getVisibilityConditions(access: KnowledgeAccessScope) { ] } +/** Candidates each hybrid leg retrieves before the fused list is trimmed to `topK`. */ +const HYBRID_CANDIDATE_MIN = 50 +const HYBRID_CANDIDATE_MAX = 200 +export function hybridCandidateCount(topK: number): number { + return Math.min(Math.max(topK * 3, HYBRID_CANDIDATE_MIN), HYBRID_CANDIDATE_MAX) +} + export function getQueryStrategy(kbCount: number, topK: number) { const useParallel = kbCount > 4 || (kbCount > 2 && topK > 50) const distanceThreshold = kbCount > 3 ? 0.8 : 1.0 @@ -434,7 +458,7 @@ async function executeVectorSearchOnIds( return [] } - const rows = await withVectorScanSettings(access, (executor) => + const rows = await withVectorScanSettings(access, topK, (executor) => executor .select( getSearchResultFields( @@ -533,7 +557,7 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise { + const allResults = await withVectorScanSettings(access, parallelLimit, async (executor) => { const parallelResults = await Promise.all( knowledgeBaseIds.map((kbId) => vectorLeg(executor, eq(embedding.knowledgeBaseId, kbId), parallelLimit) @@ -543,7 +567,7 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise a.distance - b.distance).slice(0, topK) } - const rows = await withVectorScanSettings(access, (executor) => + const rows = await withVectorScanSettings(access, topK, (executor) => vectorLeg(executor, inArray(embedding.knowledgeBaseId, knowledgeBaseIds), topK) ) return rows.sort((a, b) => a.distance - b.distance) @@ -810,17 +834,29 @@ export async function executeKnowledgeSearch( } const { distanceThreshold } = getQueryStrategy(knowledgeBaseIds.length, topK) + /** + * Hybrid fuses two rankings, so each leg retrieves more than the caller + * asked for: a chunk that both legs rank just below `topK` is a strong + * signal the fused list must be able to surface. + */ + const legTopK = searchMode === 'hybrid' ? hybridCandidateCount(topK) : topK const vectorSearch = hasFilters ? handleTagAndVectorSearch({ knowledgeBaseIds, - topK, + topK: legTopK, structuredFilters, queryVector, distanceThreshold, access, }) - : handleVectorOnlySearch({ knowledgeBaseIds, topK, queryVector, distanceThreshold, access }) + : handleVectorOnlySearch({ + knowledgeBaseIds, + topK: legTopK, + queryVector, + distanceThreshold, + access, + }) if (searchMode === 'vector') { const results = await vectorSearch @@ -833,7 +869,7 @@ export async function executeKnowledgeSearch( */ const keywordSearch = executeKeywordSearch({ knowledgeBaseIds, - topK, + topK: legTopK, query: query!, queryVector, structuredFilters, diff --git a/apps/sim/lib/knowledge/search/recency.ts b/apps/sim/lib/knowledge/search/recency.ts index 0fa7d515cc1..d06dd12bacf 100644 --- a/apps/sim/lib/knowledge/search/recency.ts +++ b/apps/sim/lib/knowledge/search/recency.ts @@ -9,7 +9,7 @@ export const RRF_K = 60 /** Age at which a document's recency boost has decayed to half. */ export const RECENCY_HALF_LIFE_DAYS = 90 /** The most a fully fresh document's rank score is raised, as a fraction. */ -export const RECENCY_WEIGHT = 0.15 +export const RECENCY_WEIGHT = 0.05 const DAY_MS = 24 * 60 * 60 * 1000 From caabe9b1db7c62d6ddc0720539fd56d8bb8bd439 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 23:57:51 -0700 Subject: [PATCH 26/76] fix(knowledge): land member observations and ACLs only under the lease A run that stalled past the lease TTL and resumed after a replacement took over could commit stale observations, membership rows, and document ACLs over the replacement's. Every such write now runs in a transaction that first proves the run still holds the connector's member lease, holding the connector row's lock so the scheduler cannot reclaim it mid-transaction; a run that lost it ends as superseded. --- .../connectors/member-observations.ts | 5 +- .../connectors/member-sync-engine.ts | 79 ++++++++++++++----- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index 0bed21ab984..57aa690c76f 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -209,13 +209,14 @@ export async function rewriteConnectorAcls( export async function materializeDocumentAcls( connectorId: string, - documentIds: Iterable + documentIds: Iterable, + executor: DbOrTx = db ): Promise { const ids = [...new Set(documentIds)] let updated = 0 for (let offset = 0; offset < ids.length; offset += MATERIALIZE_BATCH_SIZE) { const batch = ids.slice(offset, offset + MATERIALIZE_BATCH_SIZE) - const rows = await db + const rows = await executor .update(document) .set({ acl: observedAcl() }) .where( diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 42df625f531..8f2c6e498fb 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -22,6 +22,7 @@ import { isManagedCredentialGroupBindingLive, loadCredentialGroupCredentialListContext, } from '@/lib/credential-groups/credentials' +import type { DbOrTx } from '@/lib/db/types' import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { EMPTY_ACL, subjectToken } from '@/lib/knowledge/access/tokens' import { @@ -397,6 +398,28 @@ function createMemberTokenCache(input: { } } +/** + * Runs `fn` in a transaction that first proves this run still holds the + * connector's member lease, taking the connector row's lock so the scheduler + * cannot reclaim the lease mid-transaction. A run that stalled past the lease + * TTL and resumed after a replacement took over therefore never lands its + * observations or ACLs over the replacement's; it ends as superseded. + */ +async function withMemberLease( + run: Pick, + fn: (tx: DbOrTx) => Promise +): Promise { + return db.transaction(async (tx) => { + const [held] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(stillHoldsMemberSyncLock(run.connectorId, run.runId)) + .for('update') + if (!held) throw new SyncLockLostException(run.connectorId) + return fn(tx) + }) +} + async function acquireMemberSyncLock( connectorId: string, runId: string, @@ -525,6 +548,10 @@ async function reconcileMembership( const inserts: (typeof knowledgeConnectorMember.$inferInsert)[] = [] const affectedMemberIds: string[] = [] + const updates: Array<{ + id: string + values: Partial + }> = [] const deleteMemberIds: string[] = [] for (const snapshot of snapshots.values()) { @@ -558,17 +585,17 @@ async function reconcileMembership( const tokenChanged = row.subjectToken !== snapshot.subjectToken const statusChanged = row.status !== status if (!tokenChanged && !statusChanged) continue - await db - .update(knowledgeConnectorMember) - .set({ + updates.push({ + id: row.id, + values: { subjectToken: snapshot.subjectToken, status, suspendedAt: snapshot.active ? null : (row.suspendedAt ?? now), /** A reactivated member is due immediately; their observations may be stale. */ ...(statusChanged && snapshot.active ? { nextAttemptAt: now, consecutiveFailures: 0 } : {}), updatedAt: now, - }) - .where(eq(knowledgeConnectorMember.id, row.id)) + }, + }) affectedMemberIds.push(row.id) } @@ -585,18 +612,28 @@ async function reconcileMembership( affectedDocumentIds.add(documentId) } } - if (deleteMemberIds.length > 0) { - await db - .delete(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, run.connectorId), - inArray(knowledgeConnectorMember.id, deleteMemberIds) - ) - ) - } - if (inserts.length > 0) { - await db.insert(knowledgeConnectorMember).values(inserts).onConflictDoNothing() + if (updates.length > 0 || deleteMemberIds.length > 0 || inserts.length > 0) { + await withMemberLease(run, async (tx) => { + for (const update of updates) { + await tx + .update(knowledgeConnectorMember) + .set(update.values) + .where(eq(knowledgeConnectorMember.id, update.id)) + } + if (deleteMemberIds.length > 0) { + await tx + .delete(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + inArray(knowledgeConnectorMember.id, deleteMemberIds) + ) + ) + } + if (inserts.length > 0) { + await tx.insert(knowledgeConnectorMember).values(inserts).onConflictDoNothing() + } + }) } logger.info('Reconciled members-mode membership', { @@ -895,7 +932,7 @@ async function applyMemberListing( const exhaustedFailures = (outcome.member.consecutiveFailures ?? 0) + 1 const now = new Date() - await db.transaction(async (tx) => { + await withMemberLease(run, async (tx) => { const added = await recordMemberObservations(tx, outcome.member.id, seenDocumentIds, run.runId) run.result.observationsAdded += added /** @@ -1145,7 +1182,7 @@ async function disableMemberSync(run: MemberSyncRun, reason: string): Promise row.id) ) - await materializeDocumentAcls(run.connectorId, affected) + await withMemberLease(run, (tx) => materializeDocumentAcls(run.connectorId, affected, tx)) } await failMemberSyncLog(run.runId, run.result, reason) await db @@ -1472,7 +1509,9 @@ export async function executeMemberSync( for (const documentId of affected) affectedDocumentIds.add(documentId) } - await materializeDocumentAcls(connectorId, affectedDocumentIds) + await withMemberLease(run, (tx) => + materializeDocumentAcls(connectorId, affectedDocumentIds, tx) + ) /** * Nobody has completed a listing yet — a connector that just entered From 9ea67bab1fe4d8dd34284243f45e4e8b0c2efd64 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 00:03:55 -0700 Subject: [PATCH 27/76] fix(knowledge): disable a member sync only under the lease A run whose binding was removed suspended members and rewrote ACLs before proving it still held the lease; a run reclaimed meanwhile could suspend the replacement's members and then fail. Suspension, the ACLs it changes, and the disable now land in one lease-guarded transaction, and a reclaimed run ends as superseded. --- .../connectors/member-sync-engine.ts | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 8f2c6e498fb..8b9e0fc86f7 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -1167,35 +1167,38 @@ async function deferMemberSync(run: MemberSyncRun, syncIntervalMinutes: number): */ async function disableMemberSync(run: MemberSyncRun, reason: string): Promise { const now = new Date() - const suspended = await db - .update(knowledgeConnectorMember) - .set({ status: 'suspended', suspendedAt: now, updatedAt: now }) - .where( - and( - eq(knowledgeConnectorMember.connectorId, run.connectorId), - eq(knowledgeConnectorMember.status, 'active') + /** Suspension, the ACLs it changes, and the disable itself land together, and only under the lease. */ + await withMemberLease(run, async (tx) => { + const suspended = await tx + .update(knowledgeConnectorMember) + .set({ status: 'suspended', suspendedAt: now, updatedAt: now }) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + eq(knowledgeConnectorMember.status, 'active') + ) ) - ) - .returning({ id: knowledgeConnectorMember.id }) - if (suspended.length > 0) { - const affected = await listObservedDocumentIds( - db, - suspended.map((row) => row.id) - ) - await withMemberLease(run, (tx) => materializeDocumentAcls(run.connectorId, affected, tx)) - } + .returning({ id: knowledgeConnectorMember.id }) + if (suspended.length > 0) { + const affected = await listObservedDocumentIds( + tx, + suspended.map((row) => row.id) + ) + await materializeDocumentAcls(run.connectorId, affected, tx) + } + await tx + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'disabled', + lastMemberSyncError: reason, + nextMemberSyncAt: null, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where(holdsMemberSyncLockToken(run.connectorId, run.runId)) + }) await failMemberSyncLog(run.runId, run.result, reason) - await db - .update(knowledgeConnector) - .set({ - memberSyncStatus: 'disabled', - lastMemberSyncError: reason, - nextMemberSyncAt: null, - memberSyncLockToken: null, - memberSyncLockLeaseAt: null, - updatedAt: now, - }) - .where(holdsMemberSyncLockToken(run.connectorId, run.runId)) logger.warn('Member sync disabled', { connectorId: run.connectorId, reason }) } From 087d6d2b68fd0f0551d51874bc07ff60a2284afa Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 00:10:03 -0700 Subject: [PATCH 28/76] fix(knowledge): record a member failure only under the lease A listing that ran past the lease TTL and then failed wrote its backoff over the replacement run's counters and schedule. The failure ladder now lands under the same lease guard as every other member write. --- .../connectors/member-sync-engine.ts | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 8b9e0fc86f7..028c08d44a5 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -728,20 +728,23 @@ async function countDueMembers( } async function recordMemberFailure( + run: MemberSyncRun, member: MemberRow, error: unknown, syncIntervalMinutes: number ): Promise { const failures = member.consecutiveFailures + 1 - await db - .update(knowledgeConnectorMember) - .set({ - consecutiveFailures: failures, - nextAttemptAt: new Date(Date.now() + memberFailureBackoffMs(failures, syncIntervalMinutes)), - lastError: getErrorMessage(error), - updatedAt: new Date(), - }) - .where(eq(knowledgeConnectorMember.id, member.id)) + await withMemberLease(run, (tx) => + tx + .update(knowledgeConnectorMember) + .set({ + consecutiveFailures: failures, + nextAttemptAt: new Date(Date.now() + memberFailureBackoffMs(failures, syncIntervalMinutes)), + lastError: getErrorMessage(error), + updatedAt: new Date(), + }) + .where(eq(knowledgeConnectorMember.id, member.id)) + ) } /** @@ -904,7 +907,7 @@ async function listForMember(input: { memberId: member.id, error: getErrorMessage(error), }) - await recordMemberFailure(member, error, input.syncIntervalMinutes) + await recordMemberFailure(input.run, member, error, input.syncIntervalMinutes) run.result.membersFailed += 1 return { kind: 'failed' } } From 9549f17dfe1ecdb01ba84ed8bc0cfa74d2717dfa Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 00:22:27 -0700 Subject: [PATCH 29/76] fix(knowledge): tombstone and resurrect only under the lease The document lifecycle's tombstone and resurrection writes ran outside the lease guard, so a run reclaimed after its ACL transaction could hide a document the replacement restored or expose one it removed. They now run in the same lease-guarded transaction as every other member write, and a run whose lease is lost while it disables itself ends as superseded rather than rejecting. --- .../connectors/member-observations.ts | 77 ++++++++++--------- .../connectors/member-sync-engine.ts | 12 ++- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index 57aa690c76f..f4a2c549696 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -254,6 +254,8 @@ export async function applyMemberDocumentLifecycle(input: { knowledgeBaseId: string runId: string lease: Pick + /** Runs the tombstone and resurrection writes only while the run still holds its lease. */ + withLease: (fn: (tx: DbOrTx) => Promise) => Promise /** External ids whose refresh did not land this run; withheld from resurrection. */ failedExternalIds: ReadonlySet /** @@ -266,52 +268,55 @@ export async function applyMemberDocumentLifecycle(input: { const { connectorId, knowledgeBaseId, runId } = input const now = new Date() - const tombstoned = !input.allowRemoval - ? [] - : await db - .update(document) - .set({ deletedAt: now }) - .where( - and( - eq(document.connectorId, connectorId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - hasNoObservation() - ) - ) - .returning({ id: document.id }) - - const resurrectionCandidates = await db - .select({ id: document.id, externalId: document.externalId }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - isNotNull(document.deletedAt), - hasObservation() - ) - ) - const resurrectIds = resurrectionCandidates - .filter((row) => !row.externalId || !input.failedExternalIds.has(row.externalId)) - .map((row) => row.id) - const resurrected = - resurrectIds.length === 0 + const { tombstoned, resurrected } = await input.withLease(async (tx) => { + const tombstoned = !input.allowRemoval ? [] - : await db + : await tx .update(document) - .set({ deletedAt: null }) + .set({ deletedAt: now }) .where( and( - inArray(document.id, resurrectIds), eq(document.connectorId, connectorId), + eq(document.userExcluded, false), isNull(document.archivedAt), - isNotNull(document.deletedAt) + isNull(document.deletedAt), + hasNoObservation() ) ) .returning({ id: document.id }) + const resurrectionCandidates = await tx + .select({ id: document.id, externalId: document.externalId }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + isNotNull(document.deletedAt), + hasObservation() + ) + ) + const resurrectIds = resurrectionCandidates + .filter((row) => !row.externalId || !input.failedExternalIds.has(row.externalId)) + .map((row) => row.id) + const resurrected = + resurrectIds.length === 0 + ? [] + : await tx + .update(document) + .set({ deletedAt: null }) + .where( + and( + inArray(document.id, resurrectIds), + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + isNotNull(document.deletedAt) + ) + ) + .returning({ id: document.id }) + return { tombstoned, resurrected } + }) + const purgeCutoff = new Date(now.getTime() - MEMBER_TOMBSTONE_PURGE_DAYS * 24 * 60 * 60 * 1000) const purgeCandidates = input.allowRemoval ? await db diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 028c08d44a5..b26cc7e853a 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -1539,6 +1539,7 @@ export async function executeMemberSync( knowledgeBaseId: connector.knowledgeBaseId, runId, lease: run.lease, + withLease: (fn) => withMemberLease(run, fn), failedExternalIds: state.failedExternalIds, allowRemoval: (listed?.count ?? 0) > 0, }) @@ -1590,7 +1591,16 @@ export async function executeMemberSync( return skipped(result, 'connector_deleted_during_sync') } if (error instanceof MemberBindingGoneError) { - await disableMemberSync(run, error.message) + try { + await disableMemberSync(run, error.message) + } catch (disableError) { + if (!(disableError instanceof SyncLockLostException)) throw disableError + logger.warn('Member sync abandoned — lock was reclaimed before it could be disabled', { + connectorId, + runId, + }) + return skipped(result, 'sync_superseded') + } return { ...skipped(result, 'connector_not_syncable'), error: error.message } } From b7f638bfc55740269a6fb45744b1e11441344035 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 00:29:35 -0700 Subject: [PATCH 30/76] fix(knowledge): prove the lease before a batch's document writes Hydrating a batch can outlast the lease. processDocOps now takes the strict lease probe before persisting anything from a batch, so a run replaced during hydration cannot land stale content or queue processing over the replacement's; it ends as superseded. Both engines share the guard. --- apps/sim/lib/knowledge/connectors/sync-primitives.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/connectors/sync-primitives.ts b/apps/sim/lib/knowledge/connectors/sync-primitives.ts index f12afdad5dd..dabd90c9031 100644 --- a/apps/sim/lib/knowledge/connectors/sync-primitives.ts +++ b/apps/sim/lib/knowledge/connectors/sync-primitives.ts @@ -1495,7 +1495,7 @@ export interface ProcessDocOpsInput { forceRehydrate: boolean state: SyncRunState hydration: DocOpHydration - lease: Pick + lease: Pick /** Who may read the documents this pass writes. */ documentAccess: SyncDocumentAccess } @@ -1643,6 +1643,14 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { } } + /** + * Hydration above may have outlasted the lease. Nothing from this batch is + * written until the run proves it still owns the connector, so a run that + * was replaced meanwhile cannot land stale content or queue processing + * over the replacement's. + */ + await input.lease.beatLive() + if (skippedRetryHashUpdates.length > 0) { try { const missedExternalIds = await persistSkippedRetryHashes( From 75df3e51f5b5299012ffc1e6b8118051983aca6e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 00:51:15 -0700 Subject: [PATCH 31/76] feat(knowledge): one click connects a Sim Search source and indexes it for the person Connecting a source on the Search tab, or from the composer's Search-mode suggestions, no longer creates a bare credential that indexes nothing. It finds or creates the workspace's Sim Search knowledge base and a per-member connector for the source, then enrolls the person; the OAuth completion queues their member run, so indexing starts on its own and the row counts their documents up as they land. Sources that need a site or space are set up from a knowledge base instead. The credential-only detail page goes. --- .../api/knowledge/sim-search/connect/route.ts | 20 ++ .../connector-actions.test.ts | 7 +- .../suggested-actions/connector-actions.ts | 19 +- .../suggested-actions.test.tsx | 43 ++- .../suggested-actions/suggested-actions.tsx | 50 +++- .../components/suggested-actions/types.ts | 8 +- .../search/connected/[credentialId]/page.tsx | 15 -- .../search-credential-detail.tsx | 240 ----------------- .../search/hooks/use-search-credentials.ts | 48 ---- .../[workspaceId]/search/search.test.tsx | 171 ++++++------ .../workspace/[workspaceId]/search/search.tsx | 246 ++++++++++-------- apps/sim/hooks/queries/kb/connectors.ts | 23 ++ apps/sim/hooks/use-member-enrollment.ts | 75 ++++-- .../lib/api/contracts/knowledge/connectors.ts | 30 +++ .../lib/knowledge/application/connectors.ts | 41 ++- .../knowledge/application/operations.test.ts | 1 + .../lib/knowledge/application/operations.ts | 12 + .../lib/knowledge/application/search.test.ts | 5 + .../lib/knowledge/application/sim-search.ts | 131 ++++++++++ apps/sim/lib/sim-search/connectors.test.ts | 35 ++- apps/sim/lib/sim-search/connectors.ts | 29 +-- 21 files changed, 630 insertions(+), 619 deletions(-) create mode 100644 apps/sim/app/api/knowledge/sim-search/connect/route.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/page.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts create mode 100644 apps/sim/lib/knowledge/application/sim-search.ts diff --git a/apps/sim/app/api/knowledge/sim-search/connect/route.ts b/apps/sim/app/api/knowledge/sim-search/connect/route.ts new file mode 100644 index 00000000000..cc842ee90a0 --- /dev/null +++ b/apps/sim/app/api/knowledge/sim-search/connect/route.ts @@ -0,0 +1,20 @@ +import { connectSimSearchConnectorContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { connectSimSearchConnector } from '@/lib/knowledge/application/sim-search' + +export const POST = defineInternalJsonRoute({ + contract: connectSimSearchConnectorContract, + auth: internalSessionAuth, + operation: knowledgeOperations.simSearchConnect, + rateLimit: internalRateLimits.none({ reason: 'One click per source; mints a single-use link' }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ body }) => ({ workspaceId: body.workspaceId, connectorType: body.connectorType }), + useCase: connectSimSearchConnector, + present: (result) => ({ success: true as const, data: result }), +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts index e1290578c36..edfba05fbe8 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts @@ -22,10 +22,6 @@ vi.mock('@/lib/sim-search/connectors', () => { blockType: type, }) return { - isSearchConnectorConnected: ( - candidate: { providerIds: string[] }, - connected: ReadonlySet - ) => candidate.providerIds.some((providerId) => connected.has(providerId)), SEARCH_CONNECTORS: [ connector('airtable', 'Airtable', 'airtable'), connector('confluence', 'Confluence', 'confluence'), @@ -63,11 +59,12 @@ describe('computeConnectorActions', () => { }) }) - it('drops every connector on a connected provider and refills from the rotation', () => { + it('drops every connected source and refills from the rotation', () => { const actions = computeConnectorActions(new Set(['jira', 'airtable']), ALL_AVAILABLE) expect(actions.map((action) => action.id)).toEqual([ 'connect-confluence', + 'connect-jsm', 'connect-notion', 'connect-slack', ]) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts index dfb30c4765b..6123834245c 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts @@ -1,8 +1,4 @@ -import { - isSearchConnectorConnected, - SEARCH_CONNECTORS, - type SearchConnector, -} from '@/lib/sim-search/connectors' +import { SEARCH_CONNECTORS, type SearchConnector } from '@/lib/sim-search/connectors' import type { Action } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/types' import { weightedSample } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample' @@ -40,18 +36,17 @@ function toConnectorAction(connector: SearchConnector): Action { /** * Builds the Search-mode rows: the pinned connectors first, then a uniform - * sample of the rest to fill four slots. A connector whose provider the viewer - * has already connected is dropped from both halves — so Jira and Jira Service - * Management, which share one provider, leave together — and a pinned slot - * freed that way is taken by the rotation. Connectors this deployment cannot - * connect are dropped the same way, so a row never opens a modal that fails. + * sample of the rest to fill four slots. A source the viewer has already + * connected is dropped from both halves, and a pinned slot freed that way is + * taken by the rotation. Sources this deployment cannot connect are dropped + * the same way, so a row never starts a connection that fails. */ export function computeConnectorActions( - connectedProviderIds: ReadonlySet, + connectedTypes: ReadonlySet, isAvailable: (connector: SearchConnector) => boolean ): Action[] { const offered = (connector: SearchConnector) => - isAvailable(connector) && !isSearchConnectorConnected(connector, connectedProviderIds) + isAvailable(connector) && !connectedTypes.has(connector.type) const pinned = PINNED.filter(offered) const pool = ROTATING.filter(offered) const rotating = weightedSample(pool, CONNECTOR_ACTION_COUNT - pinned.length, () => 1) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx index f997520eaf3..b79be3ee8e5 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx @@ -5,10 +5,13 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCaptureEvent, mockUseSearchCredentials } = vi.hoisted(() => ({ - mockCaptureEvent: vi.fn(), - mockUseSearchCredentials: vi.fn(), -})) +const { mockCaptureEvent, mockUseWorkspaceMemberConnectors, mockConnectSource } = vi.hoisted( + () => ({ + mockCaptureEvent: vi.fn(), + mockUseWorkspaceMemberConnectors: vi.fn(), + mockConnectSource: vi.fn(), + }) +) vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), @@ -29,8 +32,12 @@ vi.mock('@/hooks/queries/tables', () => ({ vi.mock('@/hooks/queries/kb/knowledge', () => ({ useKnowledgeBasesQuery: () => ({ data: [] }), })) -vi.mock('@/app/workspace/[workspaceId]/search/hooks/use-search-credentials', () => ({ - useSearchCredentials: mockUseSearchCredentials, +vi.mock('@/hooks/queries/kb/connectors', () => ({ + memberConnectorKeys: { list: (workspaceId?: string) => ['member-connectors', workspaceId] }, + useWorkspaceMemberConnectors: mockUseWorkspaceMemberConnectors, +})) +vi.mock('@/hooks/use-member-enrollment', () => ({ + useMemberEnrollment: () => ({ connectSource: mockConnectSource }), })) vi.mock('@/hooks/use-permission-config', () => ({ usePermissionConfig: () => ({ @@ -54,10 +61,6 @@ vi.mock('@/lib/sim-search/connectors', () => { blockType: type, }) return { - isSearchConnectorConnected: ( - candidate: { providerIds: string[] }, - connected: ReadonlySet - ) => candidate.providerIds.some((providerId) => connected.has(providerId)), isSearchConnectorAvailable: ( candidate: { blockType: string }, availability: ReadonlyMap @@ -110,9 +113,19 @@ function connectModal(): string | null { beforeEach(() => { onSelectPrompt.mockClear() mockCaptureEvent.mockClear() - mockUseSearchCredentials.mockReturnValue({ - credentials: [{ id: 'cred-jira', providerId: 'jira' }], + mockUseWorkspaceMemberConnectors.mockReturnValue({ isPending: false, + data: [ + { + knowledgeBaseId: 'kb-search', + knowledgeBaseName: 'Sim Search', + connectorId: 'conn-jira', + connectorType: 'jira', + memberSyncStatus: 'idle', + viewerMembership: 'connected', + viewerDocumentCount: 3, + }, + ], }) useMothershipModeStore.getState().reset() }) @@ -140,12 +153,13 @@ describe('SuggestedActions', () => { expect(heading()).toBe('Connect Sim Search') expect(rows().map((row) => row.textContent)).toEqual([ 'Connect Confluence', + 'Connect Jira Service Management', 'Connect Airtable', 'Connect Slack', ]) }) - it('opens the OAuth connect modal for a connector row instead of populating the input', () => { + it('connects a source through its per-member connector instead of populating the input', () => { mount() act(() => useMothershipModeStore.getState().setMode('search')) expect(connectModal()).toBeNull() @@ -154,7 +168,8 @@ describe('SuggestedActions', () => { rows()[0].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) }) - expect(connectModal()).toBe('confluence') + expect(connectModal()).toBeNull() + expect(mockConnectSource).toHaveBeenCalledWith('workspace-1', 'confluence') expect(onSelectPrompt).not.toHaveBeenCalled() expect(mockCaptureEvent).toHaveBeenCalledWith( null, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx index c1f47260b06..7959f50dcd0 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx @@ -22,14 +22,19 @@ import type { OAuthConnectTarget, } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/types' import { weightedSample } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample' -import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' import { BrandIcon } from '@/blocks/brand-icon' import { getAllBlockMeta } from '@/blocks/registry' import type { ModuleTag } from '@/blocks/types' import { useWorkspaceCredentials } from '@/hooks/queries/credentials' +import { + memberConnectorKeys, + useWorkspaceMemberConnectors, + type WorkspaceMemberConnector, +} from '@/hooks/queries/kb/connectors' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useOAuthConnections } from '@/hooks/queries/oauth/oauth-connections' import { useTablesList } from '@/hooks/queries/tables' +import { useMemberEnrollment } from '@/hooks/use-member-enrollment' import { usePermissionConfig } from '@/hooks/use-permission-config' import { type MothershipMode, useMothershipModeStore } from '@/stores/mothership-mode/store' @@ -151,6 +156,7 @@ function scoreCandidate(c: Candidate, signals: Signals): number { } const EMPTY_CREDENTIALS: NonNullable['data']> = [] +const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] const EMPTY_SERVICES: NonNullable['data']> = [] type ServiceInfo = NonNullable['data']>[number] @@ -258,8 +264,8 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { const { data: knowledgeBases = [] } = useKnowledgeBasesQuery(workspaceId, { enabled: Boolean(workspaceId), }) - const { credentials: searchCredentials, isPending: searchCredentialsPending } = - useSearchCredentials(workspaceId) + const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS, isPending: connectionsPending } = + useWorkspaceMemberConnectors(workspaceId) const [expanded, setExpanded] = useState(true) /** @@ -296,15 +302,27 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { [connectedProviders, tables.length, knowledgeBases.length] ) - const connectedSearchProviders = useMemo( + /** Sources the viewer has already connected, by connector type. */ + const connectedSearchTypes = useMemo( () => new Set( - searchCredentials - .map((credential) => credential.providerId) - .filter((providerId): providerId is string => Boolean(providerId)) + memberConnectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.connectorType) ), - [searchCredentials] + [memberConnectors] ) + const connectedConnectorIds = useMemo( + () => + new Set( + memberConnectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.connectorId) + ), + [memberConnectors] + ) + const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) + const { connectSource } = useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds }) /** * Each mode's list is memoized on its own inputs alone, so switching modes — @@ -321,12 +339,12 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { */ const searchActions = useMemo( () => - searchCredentialsPending + connectionsPending ? [] - : computeConnectorActions(connectedSearchProviders, (connector) => + : computeConnectorActions(connectedSearchTypes, (connector) => isSearchConnectorAvailable(connector, integrationAvailability) ), - [searchCredentialsPending, connectedSearchProviders, integrationAvailability] + [connectionsPending, connectedSearchTypes, integrationAvailability] ) const buildActions = useMemo(() => { const personalized = services.length > 0 && connectedProviders.size > 0 @@ -343,14 +361,18 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { label: action.label, position, connected_provider_count: - action.kind === 'connector' ? connectedSearchProviders.size : connectedProviders.size, + action.kind === 'connector' ? connectedSearchTypes.size : connectedProviders.size, }) if (action.kind === 'prompt') { onSelectPrompt(action.prompt) return } - const target = - action.kind === 'connector' ? action.target : resolveOAuthServiceForSlug(action.slug) + /** A Sim Search source connects through its per-member connector, not a bare credential. */ + if (action.kind === 'connector') { + if (workspaceId) connectSource(workspaceId, action.target.type) + return + } + const target = resolveOAuthServiceForSlug(action.slug) if (target) setOAuthTarget(target) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts index 98c51898c9c..2165b1c8eef 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts @@ -1,4 +1,5 @@ import type { ComponentType, CSSProperties } from 'react' +import type { SearchConnector } from '@/lib/sim-search/connectors' export type ActionIcon = ComponentType<{ className?: string; style?: CSSProperties }> @@ -13,10 +14,11 @@ export interface OAuthConnectTarget { /** * One suggested-action row. `prompt` rows populate the input with a curated * prompt; `integration` rows resolve their OAuth service from the catalog slug - * on click; `connector` rows — the Search-mode "Connect X" rows — carry their - * connect target directly. Both connecting kinds open the OAuth connect modal. + * on click and open the OAuth connect modal; `connector` rows — the Search-mode + * "Connect X" rows — carry the Sim Search source, which connects through its + * per-member connector. */ export type Action = | { kind: 'prompt'; id: string; label: string; icon: ActionIcon; prompt: string } | { kind: 'integration'; id: string; label: string; icon: ActionIcon; slug: string } - | { kind: 'connector'; id: string; label: string; icon: ActionIcon; target: OAuthConnectTarget } + | { kind: 'connector'; id: string; label: string; icon: ActionIcon; target: SearchConnector } diff --git a/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/page.tsx b/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/page.tsx deleted file mode 100644 index 3c4d3ddbc94..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/page.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import type { Metadata } from 'next' -import { SearchCredentialDetail } from '@/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail' - -export const metadata: Metadata = { - title: 'Connected Sim Search Connector', -} - -export default async function SearchCredentialPage({ - params, -}: { - params: Promise<{ workspaceId: string; credentialId: string }> -}) { - const { workspaceId, credentialId } = await params - return -} diff --git a/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx b/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx deleted file mode 100644 index 8f00ee19c60..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/search/connected/[credentialId]/search-credential-detail.tsx +++ /dev/null @@ -1,240 +0,0 @@ -'use client' - -import { useState } from 'react' -import { - Chip, - ChipConfirmModal, - ChipCopyInput, - ChipInput, - ChipLink, - ChipTextarea, - cn, - toast, -} from '@sim/emcn' -import { ArrowLeft } from '@sim/emcn/icons' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { useRouter } from 'next/navigation' -import { SaveDiscardChips } from '@/components/settings/save-discard-actions' -import { writeOAuthReturnContext } from '@/lib/credentials/client-state' -import { resolveCredentialDisplay } from '@/lib/integrations' -import { - CredentialDetailHeading, - CredentialDetailLayout, - DetailSection, - UnsavedChangesModal, - useCredentialDetailForm, -} from '@/app/workspace/[workspaceId]/components/credential-detail' -import { - RESOURCE_TILE_BASE, - RESOURCE_TILE_PLAIN, -} from '@/app/workspace/[workspaceId]/components/resource-tile' -import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' -import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' -import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' -import { useCreateCredentialDraft, useDeleteWorkspaceCredential } from '@/hooks/queries/credentials' -import { useConnectOAuthService } from '@/hooks/queries/oauth/oauth-connections' -import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' - -const logger = createLogger('SearchCredentialDetail') - -interface SearchCredentialDetailProps { - workspaceId: string - credentialId: string -} - -/** - * A connected Sim Search credential: the integrations connected-credential page - * without its sharing. Connections here are personal, so there is no Share - * action and no members section — the credential resolves only when the viewer - * created it, and anyone else lands on "not found". Reconnect and Disconnect - * go through the same credential draft and delete paths as integrations. - */ -export function SearchCredentialDetail({ workspaceId, credentialId }: SearchCredentialDetailProps) { - const router = useRouter() - const searchHref = `/workspace/${workspaceId}/search` - - useOAuthReturnRouter() - - const { credentials, isPending: credentialsLoading } = useSearchCredentials(workspaceId) - const connectOAuthService = useConnectOAuthService() - const createDraft = useCreateCredentialDraft() - const deleteCredential = useDeleteWorkspaceCredential() - - const credential = credentials.find((c) => c.id === credentialId) ?? null - const isAdmin = credential?.role === 'admin' - - const [showDeleteConfirmDialog, setShowDeleteConfirmDialog] = useState(false) - - const form = useCredentialDetailForm({ credential, isAdmin, backHref: searchHref }) - - const display = credential ? resolveCredentialDisplay(credential) : null - - const handleReconnect = async () => { - if (!credential?.providerId) return - try { - const draft = await createDraft.mutateAsync({ - workspaceId, - providerId: credential.providerId, - displayName: credential.displayName, - description: credential.description || undefined, - credentialId: credential.id, - }) - writeOAuthReturnContext({ - origin: 'integrations', - displayName: credential.displayName, - providerId: credential.providerId, - preCount: credentials.filter((c) => c.providerId === credential.providerId).length, - workspaceId, - reconnect: true, - requestedAt: Date.now(), - }) - await connectOAuthService.mutateAsync({ - providerId: credential.providerId, - callbackURL: window.location.href, - draftId: draft.draftId, - }) - } catch (error: unknown) { - toast.error("Couldn't start reconnect", { - description: getErrorMessage(error, 'Please try again in a moment.'), - }) - logger.error('Failed to reconnect Sim Search credential', error) - } - } - - const handleConfirmDelete = async () => { - if (!credential) return - try { - await deleteCredential.mutateAsync(credential.id) - setShowDeleteConfirmDialog(false) - router.push(searchHref) - } catch (error) { - toast.error("Couldn't disconnect", { - description: getErrorMessage(error, 'Please try again in a moment.'), - }) - logger.error('Failed to disconnect Sim Search credential', error) - } - } - - const back = ( - - Search - - ) - - const actions = - credential && isAdmin ? ( - <> - - Reconnect - - setShowDeleteConfirmDialog(true)} - disabled={deleteCredential.isPending} - > - Disconnect - - - - ) : null - - if (credentialsLoading && !credential) { - return ( - - Loading… - - ) - } - - if (!credential) { - return ( - - Credential not found. - - ) - } - - return ( - <> - - - ) : ( -
- - {credential.displayName.slice(0, 1) || '?'} - -
- ) - } - title={display?.detailTitle ?? credential.displayName} - subtitle={display?.detailSubtitle ?? 'Connected service'} - /> - - - - - - - form.setDisplayNameDraft(event.target.value)} - autoComplete='off' - data-lpignore='true' - disabled={!isAdmin} - /> - - - - form.setDescriptionDraft(event.target.value)} - placeholder='Add a description...' - maxLength={500} - autoComplete='off' - data-lpignore='true' - disabled={!isAdmin} - /> - -
- - - - - - ) -} diff --git a/apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts b/apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts deleted file mode 100644 index c298ca4e737..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/search/hooks/use-search-credentials.ts +++ /dev/null @@ -1,48 +0,0 @@ -'use client' - -import { useMemo } from 'react' -import { useSession } from '@/lib/auth/auth-client' -import { isSearchConnectorProvider } from '@/lib/sim-search/connectors' -import { useWorkspaceCredentials, type WorkspaceCredential } from '@/hooks/queries/credentials' - -const EMPTY_CREDENTIALS: readonly WorkspaceCredential[] = [] - -interface UseSearchCredentialsResult { - /** The viewer's own OAuth credentials for Sim Search connector providers. */ - credentials: readonly WorkspaceCredential[] - isPending: boolean -} - -/** - * The credentials the Sim Search surface shows as connected. Sim Search - * connections are personal: the surface lists only credentials the viewer - * created, so a workspace admin — who can see every shared credential — still - * sees just their own here. Service accounts are excluded, as they are from the - * knowledge-base connector picker: no connector can authenticate with one. - * - * Reads the same unfiltered workspace credential query the integrations pages - * use, so the two surfaces share one cache and one invalidation path. - */ -export function useSearchCredentials(workspaceId: string): UseSearchCredentialsResult { - const { data: session, isPending: sessionPending } = useSession() - const userId = session?.user?.id - const { data: allCredentials, isPending: credentialsPending } = useWorkspaceCredentials({ - workspaceId, - enabled: Boolean(workspaceId), - }) - - const credentials = useMemo(() => { - if (!userId || !allCredentials) return EMPTY_CREDENTIALS - return allCredentials.filter( - (credential) => - credential.type === 'oauth' && - credential.createdBy === userId && - isSearchConnectorProvider(credential.providerId) - ) - }, [allCredentials, userId]) - - return { - credentials, - isPending: sessionPending || (Boolean(workspaceId) && credentialsPending), - } -} diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx index 73a8e2570e7..e7b33c51e54 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx @@ -5,6 +5,11 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, describe, expect, it, vi } from 'vitest' +const { mockConnect, mockConnectSource } = vi.hoisted(() => ({ + mockConnect: vi.fn(), + mockConnectSource: vi.fn(), +})) + vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), })) @@ -14,41 +19,39 @@ vi.mock('nuqs', () => ({ vi.mock('@/hooks/use-debounced-search-setter', () => ({ useDebouncedSearchSetter: (write: (value: string) => void) => write, })) -vi.mock('@/lib/auth/auth-client', () => ({ - useSession: () => ({ data: { user: { id: 'user-1' } }, isPending: false }), -})) vi.mock('@/hooks/use-permission-config', () => ({ usePermissionConfig: () => ({ integrationAvailability: new Map([ - ['confluence', { state: 'ready', oauthAvailable: true }], - /* A service-account-only deployment: the block is usable, the OAuth path is not. */ ['slack', { state: 'limited', oauthAvailable: false }], + ['jira', { state: 'available', oauthAvailable: true }], ]), }), })) vi.mock('@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration', () => ({ - useScrollRestoration: () => {}, -})) -vi.mock('@/hooks/use-oauth-return', () => ({ - useOAuthReturnRouter: () => {}, + useScrollRestoration: () => undefined, })) vi.mock('@/app/workspace/[workspaceId]/components', () => ({ IntegrationTabsHeader: () => null, })) -vi.mock('@/app/workspace/[workspaceId]/components/connect-oauth-modal', () => ({ - ConnectOAuthModal: ({ open, providerId }: { open: boolean; providerId: string }) => - open ?
{providerId}
: null, -})) vi.mock('@/blocks', () => ({ getBlock: () => undefined })) vi.mock('@/lib/integrations', () => ({ + blockTypeToIconMap: {}, resolveCredentialDisplay: () => ({ icon: () => null, blockType: 'confluence', subtitle: 'Sub' }), })) vi.mock('@/lib/sim-search/connectors', () => { const icon = () => null - const connector = (type: string, name: string, description: string) => ({ + const connector = (type: string, name: string, description: string, personal: boolean) => ({ type, - meta: { id: type, name, description, icon }, + meta: { + id: type, + name, + description, + icon, + auth: { mode: 'oauth', provider: type }, + permissionScopedListing: personal ? { capFieldIds: [] } : undefined, + configFields: personal ? [] : [{ id: 'domain', required: true }], + }, providerId: type, providerIds: [type], requiredScopes: [], @@ -56,68 +59,67 @@ vi.mock('@/lib/sim-search/connectors', () => { serviceIcon: icon, blockType: type, }) - const providers = new Set(['confluence', 'jira', 'slack']) return { + SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', + canConnectPersonally: (meta: { permissionScopedListing?: unknown }) => + Boolean(meta.permissionScopedListing), isSearchConnectorAvailable: ( candidate: { blockType: string }, availability: ReadonlyMap ) => availability.get(candidate.blockType)?.oauthAvailable ?? true, SEARCH_CONNECTORS: [ - connector('confluence', 'Confluence', 'Sync Confluence pages'), - connector('jira', 'Jira', 'Sync Jira issues'), - connector('slack', 'Slack', 'Sync Slack messages'), + connector('google_drive', 'Google Drive', 'Sync Drive files', true), + connector('confluence', 'Confluence', 'Sync Confluence pages', false), + connector('slack', 'Slack', 'Sync Slack messages', true), ], - isSearchConnectorProvider: (providerId: string | null) => - providerId !== null && providers.has(providerId), } }) -const credential = (overrides: Record) => ({ - id: 'cred', - workspaceId: 'workspace-1', - type: 'oauth', - displayName: 'Credential', - description: null, - unredacted: false, - providerId: 'confluence', - accountId: null, - envKey: null, - envOwnerUserId: null, - createdBy: 'user-1', - createdAt: '', - updatedAt: '', - role: 'admin', - ...overrides, -}) - -vi.mock('@/hooks/queries/credentials', () => ({ - useWorkspaceCredentials: () => ({ +vi.mock('@/hooks/queries/kb/connectors', () => ({ + memberConnectorKeys: { list: (workspaceId?: string) => ['member-connectors', workspaceId] }, + useWorkspaceMemberConnectors: () => ({ isPending: false, data: [ - credential({ id: 'cred-mine', displayName: 'My Confluence' }), - credential({ - id: 'cred-theirs', - displayName: 'Teammate Jira', - providerId: 'jira', - createdBy: 'user-2', - }), - credential({ id: 'cred-sa', displayName: 'Service Account', type: 'service_account' }), - credential({ id: 'cred-github', displayName: 'My GitHub', providerId: 'github' }), + { + knowledgeBaseId: 'kb-search', + knowledgeBaseName: 'Sim Search', + connectorId: 'conn-drive', + connectorType: 'google_drive', + memberSyncStatus: 'idle', + viewerMembership: 'connected', + viewerDocumentCount: 12, + }, + { + knowledgeBaseId: 'kb-sales', + knowledgeBaseName: 'Sales', + connectorId: 'conn-sales-drive', + connectorType: 'google_drive', + memberSyncStatus: 'idle', + viewerMembership: 'invited', + viewerDocumentCount: 0, + }, ], }), })) - -vi.mock('@/hooks/queries/kb/connectors', () => ({ - memberConnectorKeys: { list: (workspaceId?: string) => ['member-connectors', workspaceId] }, - useWorkspaceMemberConnectors: () => ({ data: [] }), -})) -vi.mock('@/hooks/use-member-enrollment', () => ({ - useMemberEnrollment: () => ({ - connect: vi.fn(), - isAwaiting: () => false, - isPending: false, - error: null, - }), +vi.mock('@/hooks/use-member-enrollment', async () => { + const actual = await vi.importActual( + '@/hooks/use-member-enrollment' + ) + return { + CONNECTABLE_MEMBERSHIPS: actual.CONNECTABLE_MEMBERSHIPS, + describeMembership: actual.describeMembership, + enrollmentActionLabel: actual.enrollmentActionLabel, + useMemberEnrollment: () => ({ + connect: mockConnect, + connectSource: mockConnectSource, + isAwaiting: () => false, + isPending: false, + error: null, + }), + } +}) +vi.mock('@/connectors/registry', () => ({ + CONNECTOR_META_REGISTRY: { google_drive: { name: 'Google Drive', icon: () => null } }, })) import { Search } from '@/app/workspace/[workspaceId]/search/search' @@ -139,12 +141,8 @@ function sectionLabels(): string[] { ) } -function hrefs(): Array { - return Array.from(container?.querySelectorAll('a') ?? []).map((a) => a.getAttribute('href')) -} - -function connectButton(name: string): HTMLButtonElement | null { - return container?.querySelector(`button[aria-label="Connect ${name}"]`) ?? null +function buttons(): HTMLButtonElement[] { + return Array.from(container?.querySelectorAll('button') ?? []) } afterEach(() => { @@ -152,43 +150,32 @@ afterEach(() => { container?.remove() root = null container = null + mockConnect.mockReset() + mockConnectSource.mockReset() }) describe('Search', () => { - it('lists the viewer’s own search-connector credentials under Connected', () => { + it('shows each source with the viewer’s own connection state', () => { mount() - expect(sectionLabels()).toEqual(['Connected', 'Sim Search Connectors']) + expect(sectionLabels()).toEqual(['Sim Search Connectors', 'Shared with you']) const text = container?.textContent ?? '' - expect(text).toContain('My Confluence') - expect(text).not.toContain('Teammate Jira') - expect(text).not.toContain('Service Account') - expect(text).not.toContain('My GitHub') - expect(hrefs()).toContain('/workspace/workspace-1/search/connected/cred-mine') + expect(text).toContain('Connected · 12 documents') + expect(text).toContain('Needs a site or space; set it up from a knowledge base.') + expect(text).toContain('Unavailable in this deployment. Contact your administrator.') + expect(text).toContain('Sales') }) - it('opens the connect modal for a connector instead of navigating', () => { + it('connects a source nobody has connected yet through its per-member connector', () => { mount() - const connect = connectButton('Confluence') - expect(connect).not.toBeNull() - expect(hrefs()).not.toContain('/workspace/workspace-1/search/confluence') - expect(document.querySelector('[data-testid="connect-modal"]')).toBeNull() - + const connect = buttons().find((button) => button.textContent === 'Connect') + expect(connect).toBeDefined() act(() => { connect?.dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) }) - expect(document.querySelector('[data-testid="connect-modal"]')?.textContent).toBe('confluence') - }) - - it('disables a connector whose OAuth path is unavailable, even when the block is usable', () => { - mount() - - expect(connectButton('Jira')).not.toBeNull() - expect(connectButton('Slack')).toBeNull() - expect(container?.textContent).toContain( - 'Unavailable in this deployment. Contact your administrator.' - ) + expect(mockConnect).toHaveBeenCalledWith('kb-sales', 'conn-sales-drive') + expect(mockConnectSource).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx index 172a2c729a1..2fbad0bca8e 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -1,136 +1,174 @@ 'use client' -import { useRef, useState } from 'react' -import { ChipInput } from '@sim/emcn' +import { useMemo, useRef } from 'react' +import { Button, ChipInput } from '@sim/emcn' import { Search as SearchIcon } from '@sim/emcn/icons' import { useParams } from 'next/navigation' import { useQueryState } from 'nuqs' -import { resolveCredentialDisplay } from '@/lib/integrations' import { + canConnectPersonally, isSearchConnectorAvailable, SEARCH_CONNECTORS, type SearchConnector, + SIM_SEARCH_KNOWLEDGE_BASE_NAME, } from '@/lib/sim-search/connectors' import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/components' -import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' -import { CONNECTED_LABEL } from '@/app/workspace/[workspaceId]/integrations/search-params' import { MemberConnectorsSection, memberConnectorName, } from '@/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section' -import { useSearchCredentials } from '@/app/workspace/[workspaceId]/search/hooks/use-search-credentials' import { connectorSearchParam, connectorSearchUrlKeys, } from '@/app/workspace/[workspaceId]/search/search-params' import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' -import type { WorkspaceCredential } from '@/hooks/queries/credentials' import { + memberConnectorKeys, useWorkspaceMemberConnectors, type WorkspaceMemberConnector, } from '@/hooks/queries/kb/connectors' import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' -import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' +import { + CONNECTABLE_MEMBERSHIPS, + describeMembership, + enrollmentActionLabel, + useMemberEnrollment, +} from '@/hooks/use-member-enrollment' import { usePermissionConfig } from '@/hooks/use-permission-config' const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] const CONNECTORS_LABEL = 'Sim Search Connectors' +const NEEDS_KNOWLEDGE_BASE_SETUP = 'Needs a site or space; set it up from a knowledge base.' +const UNAVAILABLE = 'Unavailable in this deployment. Contact your administrator.' + +/** What a source row says once the viewer's own indexing has settled. */ +function connectedDescription(connector: WorkspaceMemberConnector): string { + const count = connector.viewerDocumentCount + return count === 1 ? 'Connected · 1 document' : `Connected · ${count} documents` +} -interface ConnectorItemProps { +interface SourceRowProps { connector: SearchConnector + /** The Sim Search per-member connector for this source, once anyone has connected it. */ + connection: WorkspaceMemberConnector | undefined unavailable: boolean - onConnect: (connector: SearchConnector) => void + waiting: boolean + isPending: boolean + onConnect: () => void } /** - * A connector row acts in place — it opens the connect modal — so it carries no - * navigation chevron; only a connected credential leads to a page of its own. + * One Sim Search source: what the viewer's connection is doing (indexing, + * how many documents they can read, what to do next) and the one action open + * to them. A source nobody has connected yet offers Connect, which creates its + * connector and enrolls the viewer in one step. */ -function ConnectorItem({ connector, unavailable, onConnect }: ConnectorItemProps) { +function SourceRow({ + connector, + connection, + unavailable, + waiting, + isPending, + onConnect, +}: SourceRowProps) { + const personal = canConnectPersonally(connector.meta) + const membership = connection?.viewerMembership + const state = connection + ? (describeMembership({ + membership: connection.viewerMembership, + memberSyncStatus: connection.memberSyncStatus, + waiting, + name: connector.meta.name, + }) ?? connectedDescription(connection)) + : waiting + ? `Finish connecting your ${connector.meta.name} account in the other tab.` + : connector.meta.description + const description = unavailable ? UNAVAILABLE : personal ? state : NEEDS_KNOWLEDGE_BASE_SETUP + const connectable = + !unavailable && personal && (!membership || CONNECTABLE_MEMBERSHIPS.has(membership)) return ( } title={connector.meta.name} - description={ - unavailable - ? 'Unavailable in this deployment. Contact your administrator.' - : connector.meta.description + description={description} + disabled={unavailable || !personal} + trailing={ + connectable ? ( + + ) : undefined } - onClick={unavailable ? undefined : () => onConnect(connector)} - clickLabel={`Connect ${connector.meta.name}`} - disabled={unavailable} - /> - ) -} - -interface ConnectedItemProps { - credential: WorkspaceCredential - workspaceId: string -} - -function ConnectedItem({ credential, workspaceId }: ConnectedItemProps) { - const display = resolveCredentialDisplay(credential) - if (!display.icon) return null - return ( - } - title={credential.displayName} - description={credential.description || display.subtitle} - href={`/workspace/${workspaceId}/search/connected/${credential.id}`} - clickLabel={`Open ${credential.displayName}`} - navigable /> ) } /** - * The Sim Search connector catalog: the viewer's own connections first, then - * every connector a personal OAuth connection can power. Same shell and rows - * as the Integrations page, minus its showcase and category filter — the - * connector set is small enough that the search box alone narrows it. A - * connector row opens the OAuth connect modal right here; the OAuth redirect - * lands back on this page, where the return router reports the outcome. + * The Sim Search catalog: every source a person can connect with one click, + * each row showing where the viewer's own connection stands. Connecting opens + * the enrollment for the workspace's Sim Search knowledge base, and indexing + * starts on its own once the account is linked; documents count up here as + * they land. Per-member connectors in other knowledge bases are listed below + * under Shared with you, with the same actions. */ export function Search() { const scrollContainerRef = useRef(null) - useOAuthReturnRouter() const params = useParams() const workspaceId = (params?.workspaceId as string) || '' const { integrationAvailability } = usePermissionConfig() - const [connectTarget, setConnectTarget] = useState(null) const [searchTerm, setSearchTermParam] = useQueryState(connectorSearchParam.key, { ...connectorSearchParam.parser, ...connectorSearchUrlKeys, }) /** - * The input is controlled directly by the instant nuqs value; only the URL - * write is debounced. Filtering below is cheap in-memory over a static list, - * so it reads the instant value too. + * The input binds to the instant nuqs value; only the URL write is debounced. + * Filtering reads the same instant value: it is a cheap in-memory pass over a + * small static list, which is exactly the case the url-state rule permits. */ const setSearchTerm = useDebouncedSearchSetter(setSearchTermParam) - const { credentials, isPending: credentialsLoading } = useSearchCredentials(workspaceId) - - useScrollRestoration(scrollContainerRef, { ready: !credentialsLoading }) + const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS, isPending: connectionsPending } = + useWorkspaceMemberConnectors(workspaceId) + useScrollRestoration(scrollContainerRef, { ready: !connectionsPending }) + + /** The Sim Search connection per source; other knowledge bases' connectors keep their own section. */ + const { connectionByType, sharedConnectors } = useMemo(() => { + const connectionByType = new Map() + const sharedConnectors: WorkspaceMemberConnector[] = [] + for (const connector of memberConnectors) { + if ( + connector.knowledgeBaseName === SIM_SEARCH_KNOWLEDGE_BASE_NAME && + !connectionByType.has(connector.connectorType) + ) { + connectionByType.set(connector.connectorType, connector) + } else { + sharedConnectors.push(connector) + } + } + return { connectionByType, sharedConnectors } + }, [memberConnectors]) + const connectedConnectorIds = useMemo( + () => + new Set( + memberConnectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.connectorId) + ), + [memberConnectors] + ) + const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) + const { connect, connectSource, isAwaiting, isPending, error } = useMemberEnrollment({ + membershipQueryKeys, + connectedConnectorIds, + }) const normalizedSearch = searchTerm.trim().toLowerCase() - - const visibleCredentials = normalizedSearch - ? credentials.filter((credential) => { - const display = resolveCredentialDisplay(credential) - return [credential.displayName, credential.description ?? '', display.subtitle].some( - (text) => text.toLowerCase().includes(normalizedSearch) - ) - }) - : credentials - const visibleConnectors = normalizedSearch ? SEARCH_CONNECTORS.filter( (connector) => @@ -138,22 +176,18 @@ export function Search() { connector.meta.description.toLowerCase().includes(normalizedSearch) ) : SEARCH_CONNECTORS - - const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS } = - useWorkspaceMemberConnectors(workspaceId) - const visibleMemberConnectors = normalizedSearch - ? memberConnectors.filter((connector) => + const visibleSharedConnectors = normalizedSearch + ? sharedConnectors.filter((connector) => [memberConnectorName(connector), connector.knowledgeBaseName].some((text) => text.toLowerCase().includes(normalizedSearch) ) ) - : memberConnectors + : sharedConnectors const showNoResults = Boolean(normalizedSearch) && - visibleCredentials.length === 0 && visibleConnectors.length === 0 && - visibleMemberConnectors.length === 0 + visibleSharedConnectors.length === 0 return (
@@ -168,40 +202,39 @@ export function Search() { placeholder='Search connectors...' value={searchTerm} onChange={(e) => setSearchTerm(e.target.value)} - disabled={credentialsLoading} />
- - - {visibleCredentials.length > 0 && ( - - {visibleCredentials.map((credential) => ( - - ))} - - )} - {visibleConnectors.length > 0 && ( - {visibleConnectors.map((connector) => ( - - ))} + {visibleConnectors.map((connector) => { + const connection = connectionByType.get(connector.type) + return ( + + connection + ? connect(connection.knowledgeBaseId, connection.connectorId) + : connectSource(workspaceId, connector.type) + } + /> + ) + })} )} + + + {error &&

{error}

} + {showNoResults && ( No connectors found matching “{searchTerm}” @@ -210,21 +243,6 @@ export function Search() {
- {connectTarget && workspaceId && ( - { - if (!open) setConnectTarget(null) - }} - workspaceId={workspaceId} - providerId={connectTarget.providerId} - requiredScopes={connectTarget.requiredScopes} - serviceName={connectTarget.serviceName} - serviceIcon={connectTarget.serviceIcon} - /> - )}
) } diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 2f97da4cc0e..995c82316c1 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -12,6 +12,8 @@ import { type ConnectorDetailData, type ConnectorDocumentsData, type ConnectorMemberSummary, + type ConnectSimSearchConnectorBody, + connectSimSearchConnectorContract, createKnowledgeConnectorContract, deleteKnowledgeConnectorContract, getKnowledgeConnectorContract, @@ -657,3 +659,24 @@ export function useRestoreConnectorDocument() { invalidateConnectorDocumentChange(queryClient, variables), }) } + +async function connectSimSearchConnector(body: ConnectSimSearchConnectorBody) { + const result = await requestJson(connectSimSearchConnectorContract, { body }) + return result.data +} + +/** + * One click on a Sim Search source: the source's per-member connector exists + * afterwards and the viewer has their enrollment link. The member list and the + * base list both gain a row on a first connect. + */ +export function useConnectSimSearchConnector() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: connectSimSearchConnector, + onSettled: () => { + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) + }, + }) +} diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index cb299be8200..c4795b19752 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -6,6 +6,7 @@ import { type QueryKey, useQueryClient } from '@tanstack/react-query' import type { MemberSyncStatus } from '@/lib/knowledge/types' import { memberConnectorKeys, + useConnectSimSearchConnector, useStartConnectorMemberEnrollment, type ViewerConnectorMembership, } from '@/hooks/queries/kb/connectors' @@ -80,11 +81,11 @@ interface UseMemberEnrollmentProps { } /** - * Lets the viewer connect their own account to a per-member connector. - * Enrollment opens in a new tab, and the membership queries are polled - * meanwhile so the surface that started it updates on its own once the - * account is connected; the workspace-wide membership list is refreshed too, - * so the other surface catches up as well. + * Lets the viewer connect their own account to a per-member connector, by + * connector or by Sim Search source. Enrollment opens in a new tab, and the + * membership queries are polled meanwhile so the surface that started it + * updates on its own once the account is connected; the workspace-wide + * membership list is refreshed too, so the other surface catches up as well. * * The tab is opened in the click itself, before the enrollment link is * minted, because a tab opened after a network round trip is outside the @@ -96,7 +97,8 @@ export function useMemberEnrollment({ }: UseMemberEnrollmentProps) { const connectedRef = useRef(connectedConnectorIds) const queryClient = useQueryClient() - const { mutate: startEnrollment, isPending, error } = useStartConnectorMemberEnrollment() + const enrollment = useStartConnectorMemberEnrollment() + const sourceConnection = useConnectSimSearchConnector() const [awaitingSince, setAwaitingSince] = useState>(() => new Map()) const [popupBlocked, setPopupBlocked] = useState(false) @@ -126,7 +128,13 @@ export function useMemberEnrollment({ return () => clearInterval(timer) }, [awaiting, membershipQueryKeys, queryClient]) - const connect = (knowledgeBaseId: string, connectorId: string) => { + /** Opens the tab inside the click, then sends it wherever `start` mints. */ + const openEnrollment = ( + start: (handlers: { + onSuccess: (url: string, connectorId: string) => void + onError: () => void + }) => void + ) => { const tab = window.open('about:blank', '_blank') if (!tab) { setPopupBlocked(true) @@ -134,28 +142,53 @@ export function useMemberEnrollment({ } tab.opener = null setPopupBlocked(false) - startEnrollment( - { knowledgeBaseId, connectorId }, - { - onSuccess: ({ url }) => { - tab.location.href = url - setAwaitingSince((current) => new Map(current).set(connectorId, Date.now())) - }, - onError: (err) => { - tab.close() - logger.error('Failed to start member enrollment', { error: err.message }) - }, - } - ) + start({ + onSuccess: (url, connectorId) => { + tab.location.href = url + setAwaitingSince((current) => new Map(current).set(connectorId, Date.now())) + }, + onError: () => tab.close(), + }) } + const connect = (knowledgeBaseId: string, connectorId: string) => + openEnrollment(({ onSuccess, onError }) => + enrollment.mutate( + { knowledgeBaseId, connectorId }, + { + onSuccess: ({ url }) => onSuccess(url, connectorId), + onError: (err) => { + onError() + logger.error('Failed to start member enrollment', { error: err.message }) + }, + } + ) + ) + + /** Connects a Sim Search source: its per-member connector exists afterwards, and the viewer enrolls. */ + const connectSource = (workspaceId: string, connectorType: string) => + openEnrollment(({ onSuccess, onError }) => + sourceConnection.mutate( + { workspaceId, connectorType }, + { + onSuccess: ({ url, connectorId }) => onSuccess(url, connectorId), + onError: (err) => { + onError() + logger.error('Failed to connect a Sim Search source', { error: err.message }) + }, + } + ) + ) + const isAwaiting = (connectorId: string) => awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId) + const error = enrollment.error ?? sourceConnection.error return { connect, + connectSource, isAwaiting, - isPending, + isPending: enrollment.isPending || sourceConnection.isPending, error: popupBlocked ? POPUP_BLOCKED_MESSAGE : (error?.message ?? null), } } diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index c5cfacf43bc..dd48f20f57d 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -357,9 +357,39 @@ export const workspaceMemberConnectorSchema = z.object({ connectorType: z.string(), memberSyncStatus: z.enum(MEMBER_SYNC_STATUSES), viewerMembership: viewerConnectorMembershipSchema, + /** Documents of this connector the viewer may read right now. */ + viewerDocumentCount: z.number().int().nonnegative(), }) export type WorkspaceMemberConnector = z.output +export const connectSimSearchConnectorBodySchema = z.object({ + workspaceId: workspaceIdSchema, + connectorType: z.string().min(1, 'connectorType cannot be empty').max(100), +}) +export type ConnectSimSearchConnectorBody = z.input + +/** + * One click on a Sim Search source: the workspace's Sim Search knowledge base + * and per-member connector exist afterwards, and the caller gets the link that + * connects their own account. + */ +export const connectSimSearchConnectorContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/sim-search/connect', + body: connectSimSearchConnectorBodySchema, + response: { + mode: 'json', + schema: z.object({ + success: z.literal(true), + data: z.object({ + knowledgeBaseId: z.string(), + connectorId: z.string(), + url: z.string().url(), + }), + }), + }, +}) + export const listWorkspaceMemberConnectorsContract = defineRouteContract({ method: 'GET', path: '/api/knowledge/member-connectors', diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index bd515ea31cd..7891dd941b3 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -22,6 +22,8 @@ import { resolveCredentialTokenIdentity, } from '@/lib/credentials/access' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' +import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeAttributedUserId, @@ -360,6 +362,28 @@ export interface ListWorkspaceMemberConnectorsInput { * them to connect. Only connectors the viewer could actually read documents * from are listed: the knowledge base must be live and in the workspace. */ +/** Live documents per connector that the viewer's tokens match, for the Search tab's counts. */ +async function countViewerDocuments( + connectorIds: readonly string[], + access: KnowledgeAccessScope +): Promise> { + if (connectorIds.length === 0) return new Map() + const rows = await db + .select({ connectorId: document.connectorId, count: sql`count(*)::int` }) + .from(document) + .where( + and( + inArray(document.connectorId, [...connectorIds]), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + knowledgeAccessCondition(access) + ) + ) + .groupBy(document.connectorId) + return new Map(rows.flatMap((row) => (row.connectorId ? [[row.connectorId, row.count]] : []))) +} + export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listWorkspaceMemberConnectors, resolveContext: ({ input }: { input: ListWorkspaceMemberConnectorsInput }) => @@ -390,11 +414,17 @@ export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ ) ) .orderBy(asc(knowledgeBase.name), asc(knowledgeConnector.createdAt)) - const memberships = await resolveViewerConnectorMemberships({ - userId: viewerUserId, - workspaceId: context.workspaceId, - connectors: rows, - }) + const [memberships, documentCounts] = await Promise.all([ + resolveViewerConnectorMemberships({ + userId: viewerUserId, + workspaceId: context.workspaceId, + connectors: rows, + }), + countViewerDocuments( + rows.map((row) => row.id), + await createKnowledgeAccessProvider(principal, { workspaceId: context.workspaceId }).get() + ), + ]) return { connectors: rows.flatMap((row) => { const viewerMembership = memberships.get(row.id) @@ -413,6 +443,7 @@ export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ connectorType: row.connectorType, memberSyncStatus: row.memberSyncStatus, viewerMembership, + viewerDocumentCount: documentCounts.get(row.id) ?? 0, }, ] : [] diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index 747332b284e..53cdac8308f 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -58,6 +58,7 @@ describe('knowledge operation registry', () => { 'knowledge.connectors.access.update', 'knowledge.connectors.members.list', 'knowledge.connectors.members.enroll', + 'knowledge.simSearch.connect', 'knowledge.connectors.delete', 'knowledge.connectors.sync', 'knowledge.connectors.documents.list', diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index eccb44423e4..adea8d1349c 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -417,6 +417,18 @@ export const knowledgeOperations = { capability: 'knowledge.use', principalKinds: ['session'], }), + /** + * Connecting a Sim Search source: any reader may connect their own account; + * the first connect of a source also creates its knowledge base and + * connector, which those operations reserve for an admin. + */ + simSearchConnect: defineWorkspaceOperation({ + id: 'knowledge.simSearch.connect', + minimumRole: 'read', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }), deleteConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.delete', minimumRole: 'write', diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index d7949ab6735..6070b096137 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -40,6 +40,11 @@ vi.mock('@/lib/billing/core/billing-attribution', () => ({ checkAttributedUsageLimits: mocks.checkUsage, })) +/** Retrieval defaults are the flag's concern; here the flag is off so the search stays as configured. */ +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: async () => false, +})) + vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ checkActorUsageLimits: mocks.checkActorUsage, })) diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts new file mode 100644 index 00000000000..409d10c1ade --- /dev/null +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -0,0 +1,131 @@ +import { db } from '@sim/db' +import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { and, asc, eq, isNull } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' +import { createKnowledgeConnector } from '@/lib/knowledge/application/connectors' +import { resolveKnowledgeWorkspaceContext } from '@/lib/knowledge/application/contexts' +import { createKnowledgeBase } from '@/lib/knowledge/application/knowledge-bases' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { canConnectPersonally, SIM_SEARCH_KNOWLEDGE_BASE_NAME } from '@/lib/sim-search/connectors' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' + +const SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION = + 'What each person can open in the sources they connected, searched as them.' +/** Between runs the change feeds keep deletions and unshares fresh; the hourly run fills the rest. */ +const SIM_SEARCH_SYNC_INTERVAL_MINUTES = 60 + +export interface ConnectSimSearchConnectorInput { + workspaceId: string + /** `CONNECTOR_META_REGISTRY` key of the source to connect. */ + connectorType: string +} + +export interface ConnectSimSearchConnectorResult { + knowledgeBaseId: string + connectorId: string + /** The enrollment link that connects the caller's own account. */ + url: string +} + +async function findSimSearchConnector(workspaceId: string, connectorType: string) { + const [row] = await db + .select({ knowledgeBaseId: knowledgeBase.id, connectorId: knowledgeConnector.id }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeBase.name, SIM_SEARCH_KNOWLEDGE_BASE_NAME), + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.connectorType, connectorType), + eq(knowledgeConnector.accessMode, 'members'), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .orderBy(asc(knowledgeConnector.createdAt)) + .limit(1) + return row ?? null +} + +async function findSimSearchKnowledgeBase(workspaceId: string) { + const [row] = await db + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeBase.name, SIM_SEARCH_KNOWLEDGE_BASE_NAME), + isNull(knowledgeBase.deletedAt) + ) + ) + .orderBy(asc(knowledgeBase.createdAt)) + .limit(1) + return row ?? null +} + +/** + * One click on a Sim Search source: the workspace's Sim Search knowledge base + * and a per-member connector for that source exist after this (the first + * connect creates them, which the connector operation reserves for an admin), + * and the caller gets the link that connects their own account. The OAuth + * completion queues their member run, so indexing starts on its own. + */ +export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.simSearchConnect, + resolveContext: ({ input }: { input: ConnectSimSearchConnectorInput }) => + resolveKnowledgeWorkspaceContext(input), + async execute({ principal, input, context, request }): Promise { + const meta = CONNECTOR_META_REGISTRY[input.connectorType] + if (!meta || !canConnectPersonally(meta)) { + throw new OrchestrationError( + 'validation', + 'This source needs a site or space to be set up from a knowledge base first' + ) + } + const workspaceId = context.workspaceId + let target = await findSimSearchConnector(workspaceId, input.connectorType) + if (!target) { + const knowledgeBaseId = + (await findSimSearchKnowledgeBase(workspaceId))?.id ?? + ( + await createKnowledgeBase.execute({ + principal, + input: { + workspaceId, + name: SIM_SEARCH_KNOWLEDGE_BASE_NAME, + description: SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION, + source: 'ui', + }, + request, + }) + ).knowledgeBase.id + const created = await createKnowledgeConnector.execute({ + principal, + input: { + knowledgeBaseId, + assertedWorkspaceId: workspaceId, + connectorType: input.connectorType, + sourceConfig: {}, + syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, + accessMode: 'members', + source: 'ui', + }, + request, + }) + target = { knowledgeBaseId, connectorId: created.connector.id } + } + const { url } = await startKnowledgeConnectorMemberEnrollment.execute({ + principal, + input: { + knowledgeBaseId: target.knowledgeBaseId, + connectorId: target.connectorId, + assertedWorkspaceId: workspaceId, + }, + request, + }) + return { ...target, url } + }, +}) diff --git a/apps/sim/lib/sim-search/connectors.test.ts b/apps/sim/lib/sim-search/connectors.test.ts index df5ffbd7ef1..264d77cf56f 100644 --- a/apps/sim/lib/sim-search/connectors.test.ts +++ b/apps/sim/lib/sim-search/connectors.test.ts @@ -14,11 +14,20 @@ vi.mock('@/connectors/registry', () => { auth: { mode: 'oauth', provider: 'jira' }, icon, }, - jira: { id: 'jira', name: 'Jira', auth: { mode: 'oauth', provider: 'jira' }, icon }, + jira: { + id: 'jira', + name: 'Jira', + auth: { mode: 'oauth', provider: 'jira' }, + permissionScopedListing: { capFieldIds: ['maxIssues'] }, + configFields: [{ id: 'domain', required: true }], + icon, + }, google_drive: { id: 'google_drive', name: 'Google Drive', auth: { mode: 'oauth', provider: 'google-drive' }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, + configFields: [{ id: 'maxFiles', required: false }], icon, }, gmail: { @@ -70,9 +79,8 @@ vi.mock('@/lib/integrations/credential-display', () => ({ })) import { + canConnectPersonally, isSearchConnectorAvailable, - isSearchConnectorConnected, - isSearchConnectorProvider, SEARCH_CONNECTORS, } from '@/lib/sim-search/connectors' @@ -103,21 +111,12 @@ describe('SEARCH_CONNECTORS', () => { }) }) -describe('isSearchConnectorProvider', () => { - it('matches credentials by provider, including shared and additional providers', () => { - expect(isSearchConnectorProvider('jira')).toBe(true) - expect(isSearchConnectorProvider('google-drive')).toBe(true) - expect(isSearchConnectorProvider('salesforce-sandbox')).toBe(true) - expect(isSearchConnectorProvider('slack')).toBe(false) - expect(isSearchConnectorProvider(null)).toBe(false) - }) -}) - -describe('isSearchConnectorConnected', () => { - it('counts a credential under any of the service’s provider ids', () => { - const salesforce = SEARCH_CONNECTORS.find((connector) => connector.type === 'salesforce')! - expect(isSearchConnectorConnected(salesforce, new Set(['salesforce-sandbox']))).toBe(true) - expect(isSearchConnectorConnected(salesforce, new Set(['jira']))).toBe(false) +describe('canConnectPersonally', () => { + it('offers one-click connection only to per-member sources with no required setup', () => { + const drive = SEARCH_CONNECTORS.find((connector) => connector.type === 'google_drive')! + const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + expect(canConnectPersonally(drive.meta)).toBe(true) + expect(canConnectPersonally(jira.meta)).toBe(false) }) }) diff --git a/apps/sim/lib/sim-search/connectors.ts b/apps/sim/lib/sim-search/connectors.ts index fda056d4d27..58827ad5d4b 100644 --- a/apps/sim/lib/sim-search/connectors.ts +++ b/apps/sim/lib/sim-search/connectors.ts @@ -8,6 +8,9 @@ import { import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import type { ConnectorMeta } from '@/connectors/types' +/** The workspace knowledge base Sim Search indexes into, one per workspace, created on first connect. */ +export const SIM_SEARCH_KNOWLEDGE_BASE_NAME = 'Sim Search' + /** * A knowledge-base connector offered on the Sim Search surface: the connector's * client-safe meta paired with the OAuth service a user connects it through. @@ -72,25 +75,15 @@ export const SEARCH_CONNECTORS: readonly SearchConnector[] = Object.entries(CONN .sort((a, b) => a.meta.name.localeCompare(b.meta.name)) /** - * Provider ids some Sim Search connector connects through. Several connectors - * share one (Jira and Jira Service Management both use `jira`), so a credential - * is matched to the surface by provider rather than to a single connector. + * Whether a source connects with one click on Sim Search: it crawls per + * member, and nothing in its config is required beyond the listing caps + * members mode clears. A source that needs a site or space (Confluence, + * Jira) is set up from a knowledge base, where an admin can name it. */ -const SEARCH_PROVIDER_IDS: ReadonlySet = new Set( - SEARCH_CONNECTORS.flatMap((connector) => connector.providerIds) -) - -/** Whether a stored credential's provider powers a Sim Search connector. */ -export function isSearchConnectorProvider(providerId: string | null): boolean { - return providerId !== null && SEARCH_PROVIDER_IDS.has(providerId) -} - -/** Whether the viewer has connected this connector's service under any of its provider ids. */ -export function isSearchConnectorConnected( - connector: SearchConnector, - connectedProviderIds: ReadonlySet -): boolean { - return connector.providerIds.some((providerId) => connectedProviderIds.has(providerId)) +export function canConnectPersonally(meta: ConnectorMeta): boolean { + if (meta.auth.mode !== 'oauth' || !meta.permissionScopedListing) return false + const capFieldIds = new Set(meta.permissionScopedListing.capFieldIds) + return meta.configFields.every((field) => !field.required || capFieldIds.has(field.id)) } /** From ea8563f78a305eef66e6fd90f5f9eb6aa7e5520b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 00:57:49 -0700 Subject: [PATCH 32/76] feat(knowledge): the composer's Search mode shows every source and reads like a search - Under the input, Search mode lists every Sim Search source as a chip with the person's own state: connected with a document count, indexing, reconnect, or one click to connect; sources that need a site link to Knowledge. The sampled four-row list goes, and with it the rows that offered sources the server had to refuse - Results carry a header (how many documents, searched as you, and which source is still indexing), hover actions to copy the link or summarize, an Answer with Sim action for a prose answer, and source and recency filters once a list is long and mixed enough to need them - An existing chat opens in Build; a new chat keeps the last mode. Results never join a transcript --- .../knowledge-search-results.tsx | 175 +++++++++++--- .../components/source-card/source-card.tsx | 70 ++++-- .../home/components/search-sources/index.ts | 1 + .../search-sources/search-sources.tsx | 213 ++++++++++++++++++ .../connector-actions.test.ts | 96 -------- .../suggested-actions/connector-actions.ts | 54 ----- .../suggested-actions.test.tsx | 72 +----- .../suggested-actions/suggested-actions.tsx | 105 +++------ .../components/suggested-actions/types.ts | 6 +- .../app/workspace/[workspaceId]/home/home.tsx | 9 + 10 files changed, 458 insertions(+), 343 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 032623d712d..14ad1ada9e0 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -1,15 +1,32 @@ 'use client' -import { useMemo } from 'react' +import { useMemo, useState } from 'react' +import { Button, Chip } from '@sim/emcn' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { + isIndexing, + simSearchConnectionsByType, +} from '@/app/workspace/[workspaceId]/home/components/search-sources' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import { useWorkspaceMemberConnectors } from '@/hooks/queries/kb/connectors' import { useKnowledgeBasesQuery, useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' /** A search spans at most this many knowledge bases. */ const MAX_SEARCHED_KNOWLEDGE_BASES = 20 /** Characters of the matching chunk shown under a result. */ const SNIPPET_LENGTH = 280 +/** Filters appear only once a list is long and mixed enough for them to help. */ +const FILTERS_MIN_RESULTS = 10 +const DAY_MS = 24 * 60 * 60 * 1000 + +const UPDATED_WINDOWS = [ + { id: 'any', label: 'Any time', days: null }, + { id: '7d', label: 'Past week', days: 7 }, + { id: '30d', label: 'Past month', days: 30 }, +] as const +type UpdatedWindow = (typeof UPDATED_WINDOWS)[number]['id'] function toSnippet(content: string): string { const flat = content.replace(/\s+/g, ' ').trim() @@ -46,22 +63,32 @@ function toSource(result: WorkspaceKnowledgeSearchResult): SourceTagData | null } } +function connectorName(connectorType: string): string { + return CONNECTOR_META_REGISTRY[connectorType]?.name ?? connectorType +} + interface KnowledgeSearchResultsProps { workspaceId: string query: string /** Asks the agent about one document; the prompt names it and links to it. */ onSummarize: (prompt: string) => void + /** Asks the agent the query itself, for a prose answer with citations. */ + onAnswer: (query: string) => void } /** * The composer's Search mode: the documents the signed-in person may read that - * match their query, across every knowledge base in the workspace, as cards - * that open the source. Summarize hands one document to the agent. + * match their query, across every knowledge base in the workspace, as rows + * that open the source. A header says how many and that the search ran as + * them; while a connected source is still indexing it says so, and the list + * grows as documents land. Filters by source and recency appear only once the + * list is long and mixed enough to need them. */ export function KnowledgeSearchResults({ workspaceId, query, onSummarize, + onAnswer, }: KnowledgeSearchResultsProps) { const { data: knowledgeBases = [], isPending: basesPending } = useKnowledgeBasesQuery(workspaceId) const knowledgeBaseIds = useMemo( @@ -74,12 +101,40 @@ export function KnowledgeSearchResults({ isFetching, error, } = useWorkspaceKnowledgeSearch(workspaceId, knowledgeBaseIds, query) + const { data: memberConnectors = [] } = useWorkspaceMemberConnectors(workspaceId) + const indexing = useMemo( + () => + [...simSearchConnectionsByType(memberConnectors).values()] + .filter(isIndexing) + .map((connection) => connectorName(connection.connectorType)), + [memberConnectors] + ) const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) + const sourceTypes = useMemo( + () => [...new Set(documents.map((result) => result.connectorType ?? 'upload'))], + [documents] + ) + const [sourceFilter, setSourceFilter] = useState(null) + const [updatedFilter, setUpdatedFilter] = useState('any') + const showFilters = documents.length >= FILTERS_MIN_RESULTS && sourceTypes.length > 1 + const visible = useMemo(() => { + if (!showFilters) return documents + const window = UPDATED_WINDOWS.find((entry) => entry.id === updatedFilter) + const cutoff = window?.days ? Date.now() - window.days * DAY_MS : null + return documents.filter((result) => { + if (sourceFilter && (result.connectorType ?? 'upload') !== sourceFilter) return false + if (cutoff !== null) { + const modified = result.sourceModifiedAt ? Date.parse(result.sourceModifiedAt) : Number.NaN + if (Number.isNaN(modified) || modified < cutoff) return false + } + return true + }) + }, [documents, showFilters, sourceFilter, updatedFilter]) if (!basesPending && knowledgeBaseIds.length === 0) { return (

- No knowledge bases to search yet. Add one from the Knowledge tab. + Nothing to search yet. Connect a source above to index what you can open.

) } @@ -89,41 +144,87 @@ export function KnowledgeSearchResults({ if (isPending || (isFetching && !results)) { return

Searching…

} - if (documents.length === 0) { - return ( -

- No documents you can read match “{query}”. -

- ) - } + + const indexingNote = + indexing.length > 0 + ? `Still indexing ${indexing.join(', ')}; results grow as documents land.` + : null return ( -
- {documents.map((result) => { - const source = toSource(result) - return source ? ( - - onSummarize(`Summarize "${cited.title ?? cited.url}" (${cited.url})`) - } - /> - ) : ( -
-

- {result.documentName ?? 'Untitled document'} -

-

- {result.knowledgeBaseName} -

-

- {toSnippet(result.content)} -

-
- ) - })} +
+
+ + {documents.length === 1 ? '1 document' : `${documents.length} documents`} · searched as + you + {indexingNote ? ` · ${indexingNote}` : ''} + + +
+ {showFilters && ( +
+ setSourceFilter(null)}> + All sources + + {sourceTypes.map((type) => ( + setSourceFilter(sourceFilter === type ? null : type)} + > + {type === 'upload' ? 'Uploads' : connectorName(type)} + + ))} + · + {UPDATED_WINDOWS.map((window) => ( + setUpdatedFilter(window.id)} + > + {window.label} + + ))} +
+ )} + {visible.length === 0 ? ( +

+ {documents.length === 0 + ? `No documents you can read match “${query}”.` + : 'No documents match these filters.'} +

+ ) : ( +
+ {visible.map((result) => { + const source = toSource(result) + return source ? ( + + onSummarize(`Summarize "${cited.title ?? cited.url}" (${cited.url})`) + } + /> + ) : ( +
+

+ {result.documentName ?? 'Untitled document'} +

+

+ {result.knowledgeBaseName} +

+

+ {toSnippet(result.content)} +

+
+ ) + })} +
+ )}
) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index 23631adc5f1..d2dff4cb4f0 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -1,7 +1,8 @@ 'use client' -import type { ReactNode } from 'react' -import { Button, cn } from '@sim/emcn' +import { type ReactNode, useState } from 'react' +import { Button, cn, Tooltip } from '@sim/emcn' +import { Check, Link as LinkIcon } from '@sim/emcn/icons' import { formatDate } from '@sim/utils/formatting' import { faviconUrl } from '@/lib/core/utils/favicon' import { @@ -18,6 +19,8 @@ import { BrandIcon } from '@/blocks/brand-icon' /** Query terms shorter than this are too common to bold. */ const MIN_HIGHLIGHT_TERM_LENGTH = 3 +/** How long the copied state shows on the copy-link action. */ +const COPIED_FEEDBACK_MS = 1_500 function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -56,6 +59,39 @@ function parseUpdatedAt(value: string | undefined): Date | null { return Number.isNaN(date.getTime()) ? null : date } +interface CopyLinkActionProps { + url: string +} + +/** Copies the document's link; confirms with a check for a moment. */ +function CopyLinkAction({ url }: CopyLinkActionProps) { + const [copied, setCopied] = useState(false) + return ( + + + + + {copied ? 'Copied' : 'Copy link'} + + ) +} + interface SourceCardProps { source: SourceTagData /** The query the document was found for; its terms are bolded in the snippet. */ @@ -68,8 +104,9 @@ interface SourceCardProps { * One document a search found, laid out to be scanned: the source's brand * mark or favicon, the title as a link back to the document, where it lives * and when it last changed, and the passage that matched with the query terms - * in bold. The same card serves the composer's search results and the - * footer of a reply that cited its sources with a snippet. + * in bold. Actions stay out of the way until the row is hovered or focused. + * The same row serves the composer's search results and the footer of a reply + * that cited its sources with a snippet. */ export function SourceCard({ source, query, onSummarize }: SourceCardProps) { const hostname = externalLinkHostname(source.url) @@ -82,7 +119,7 @@ export function SourceCard({ source, query, onSummarize }: SourceCardProps) { ) return ( -
+
{ConnectorIcon ? ( @@ -115,16 +152,19 @@ export function SourceCard({ source, query, onSummarize }: SourceCardProps) {

)}
- {onSummarize && ( - - )} +
+ + {onSummarize && ( + + )} +
) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts new file mode 100644 index 00000000000..9ff66d4c312 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts @@ -0,0 +1 @@ +export { isIndexing, SearchSources, simSearchConnectionsByType } from './search-sources' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx new file mode 100644 index 00000000000..1f0c57a2124 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx @@ -0,0 +1,213 @@ +'use client' + +import { useMemo } from 'react' +import { Chip, cn } from '@sim/emcn' +import { Loader, Plus } from '@sim/emcn/icons' +import Link from 'next/link' +import { + canConnectPersonally, + isSearchConnectorAvailable, + SEARCH_CONNECTORS, + type SearchConnector, + SIM_SEARCH_KNOWLEDGE_BASE_NAME, +} from '@/lib/sim-search/connectors' +import { BrandIcon } from '@/blocks/brand-icon' +import { + memberConnectorKeys, + useWorkspaceMemberConnectors, + type WorkspaceMemberConnector, +} from '@/hooks/queries/kb/connectors' +import { CONNECTABLE_MEMBERSHIPS, useMemberEnrollment } from '@/hooks/use-member-enrollment' +import { usePermissionConfig } from '@/hooks/use-permission-config' + +const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] + +/** The Sim Search connection per source, keyed by connector type. */ +export function simSearchConnectionsByType( + connectors: readonly WorkspaceMemberConnector[] +): Map { + const byType = new Map() + for (const connector of connectors) { + if (connector.knowledgeBaseName !== SIM_SEARCH_KNOWLEDGE_BASE_NAME) continue + if (!byType.has(connector.connectorType)) byType.set(connector.connectorType, connector) + } + return byType +} + +/** Whether a connected source is still indexing for the viewer. */ +export function isIndexing(connection: WorkspaceMemberConnector | undefined): boolean { + return ( + connection?.viewerMembership === 'connected' && + (connection.memberSyncStatus === 'pending' || connection.memberSyncStatus === 'running') + ) +} + +/** The chip's trailing state text for one source. */ +function sourceState( + connection: WorkspaceMemberConnector | undefined, + waiting: boolean +): string | null { + if (waiting) return 'Connecting…' + if (!connection) return null + switch (connection.viewerMembership) { + case 'connected': + return isIndexing(connection) + ? 'Indexing' + : connection.viewerDocumentCount === 1 + ? '1 document' + : `${connection.viewerDocumentCount} documents` + case 'needs_reauth': + return 'Reconnect' + case 'unverified_email': + return 'Verify email' + case 'revoked': + return 'Access removed' + default: + return null + } +} + +interface SourceChipProps { + connector: SearchConnector + connection: WorkspaceMemberConnector | undefined + unavailable: boolean + waiting: boolean + disabled: boolean + onConnect: () => void +} + +function SourceChip({ + connector, + connection, + unavailable, + waiting, + disabled, + onConnect, +}: SourceChipProps) { + const state = sourceState(connection, waiting) + const connected = connection?.viewerMembership === 'connected' + const actionable = + !unavailable && + !waiting && + (!connection || CONNECTABLE_MEMBERSHIPS.has(connection.viewerMembership)) + const title = unavailable + ? `${connector.meta.name} is unavailable in this deployment` + : connected + ? `${connector.meta.name}: ${state}` + : `Connect ${connector.meta.name}` + return ( + } + rightIcon={waiting || isIndexing(connection) ? Loader : actionable ? Plus : undefined} + > + + {connector.meta.name} + {state && {state}} + + + ) +} + +interface SearchSourcesProps { + workspaceId: string +} + +/** + * Every source Sim Search can index for the person, as chips under the + * composer: connected ones show how many documents they can read (or that + * indexing is still running), the rest connect with one click. Sources that + * need a site or space link to Knowledge, where an admin can name it. This is + * the whole catalog, not a sample, so nothing connectable stays hidden. + */ +export function SearchSources({ workspaceId }: SearchSourcesProps) { + const { integrationAvailability } = usePermissionConfig() + const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS } = + useWorkspaceMemberConnectors(workspaceId) + const connectionByType = useMemo( + () => simSearchConnectionsByType(memberConnectors), + [memberConnectors] + ) + const connectedConnectorIds = useMemo( + () => + new Set( + memberConnectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.connectorId) + ), + [memberConnectors] + ) + const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) + const { connect, connectSource, isAwaiting, isPending, error } = useMemberEnrollment({ + membershipQueryKeys, + connectedConnectorIds, + }) + + /** Connected first, then one-click sources, then those that need a knowledge base. */ + const ordered = useMemo(() => { + const rank = (connector: SearchConnector) => { + const connection = connectionByType.get(connector.type) + if (connection?.viewerMembership === 'connected') return 0 + if (connection) return 1 + return canConnectPersonally(connector.meta) ? 2 : 3 + } + return [...SEARCH_CONNECTORS].sort( + (a, b) => rank(a) - rank(b) || a.meta.name.localeCompare(b.meta.name) + ) + }, [connectionByType]) + + return ( +
+
+ {ordered.map((connector) => { + const connection = connectionByType.get(connector.type) + if (!canConnectPersonally(connector.meta) && !connection) { + return ( + + + } + > + + {connector.meta.name} + + Set up in Knowledge + + + + + ) + } + return ( + + connection + ? connect(connection.knowledgeBaseId, connection.connectorId) + : connectSource(workspaceId, connector.type) + } + /> + ) + })} +
+ {error &&

{error}

} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts deleted file mode 100644 index edfba05fbe8..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockRandomFloat } = vi.hoisted(() => ({ - mockRandomFloat: vi.fn<() => number>(), -})) - -vi.mock('@sim/utils/random', () => ({ randomFloat: mockRandomFloat })) - -vi.mock('@/lib/sim-search/connectors', () => { - const icon = () => null - const connector = (type: string, name: string, providerId: string) => ({ - type, - meta: { id: type, name, description: `Sync ${name}`, icon }, - providerId, - providerIds: [providerId], - requiredScopes: [], - serviceName: name, - serviceIcon: icon, - blockType: type, - }) - return { - SEARCH_CONNECTORS: [ - connector('airtable', 'Airtable', 'airtable'), - connector('confluence', 'Confluence', 'confluence'), - connector('jira', 'Jira', 'jira'), - connector('jsm', 'Jira Service Management', 'jira'), - connector('notion', 'Notion', 'notion'), - connector('slack', 'Slack', 'slack'), - ], - } -}) - -import { computeConnectorActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions' - -const ALL_AVAILABLE = () => true - -describe('computeConnectorActions', () => { - beforeEach(() => { - /** A zero roll always samples the first remaining candidate, so the rotation is catalog order. */ - mockRandomFloat.mockReturnValue(0) - }) - - it('pins Confluence, Jira, and JSM first and fills the last slot from the rotation', () => { - const actions = computeConnectorActions(new Set(), ALL_AVAILABLE) - - expect(actions.map((action) => action.id)).toEqual([ - 'connect-confluence', - 'connect-jira', - 'connect-jsm', - 'connect-airtable', - ]) - expect(actions[0]).toMatchObject({ - kind: 'connector', - label: 'Connect Confluence', - target: { providerId: 'confluence', serviceName: 'Confluence' }, - }) - }) - - it('drops every connected source and refills from the rotation', () => { - const actions = computeConnectorActions(new Set(['jira', 'airtable']), ALL_AVAILABLE) - - expect(actions.map((action) => action.id)).toEqual([ - 'connect-confluence', - 'connect-jsm', - 'connect-notion', - 'connect-slack', - ]) - }) - - it('drops connectors this deployment cannot connect, pinned or not', () => { - const actions = computeConnectorActions( - new Set(), - (connector) => connector.type !== 'jira' && connector.type !== 'airtable' - ) - - expect(actions.map((action) => action.id)).toEqual([ - 'connect-confluence', - 'connect-jsm', - 'connect-notion', - 'connect-slack', - ]) - }) - - it('returns fewer than four rows once the rotation is exhausted', () => { - const actions = computeConnectorActions(new Set(['airtable', 'notion', 'slack']), ALL_AVAILABLE) - - expect(actions.map((action) => action.id)).toEqual([ - 'connect-confluence', - 'connect-jira', - 'connect-jsm', - ]) - }) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts deleted file mode 100644 index 6123834245c..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { SEARCH_CONNECTORS, type SearchConnector } from '@/lib/sim-search/connectors' -import type { Action } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/types' -import { weightedSample } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample' - -/** Rows shown in Search mode — the same count as the Build-mode set. */ -const CONNECTOR_ACTION_COUNT = 4 - -/** - * Connectors pinned to the head of the Search-mode list, in this order. Keys - * are `CONNECTOR_META_REGISTRY` keys, as on every {@link SearchConnector}. - */ -export const PINNED_CONNECTOR_TYPES = ['confluence', 'jira', 'jsm'] as const - -const PINNED_TYPES: ReadonlySet = new Set(PINNED_CONNECTOR_TYPES) - -/** The pinned connectors, in pinned order. */ -const PINNED: readonly SearchConnector[] = PINNED_CONNECTOR_TYPES.flatMap((type) => { - const connector = SEARCH_CONNECTORS.find((candidate) => candidate.type === type) - return connector ? [connector] : [] -}) - -/** Every other Sim Search connector — the pool the remaining slots rotate through. */ -const ROTATING: readonly SearchConnector[] = SEARCH_CONNECTORS.filter( - (connector) => !PINNED_TYPES.has(connector.type) -) - -function toConnectorAction(connector: SearchConnector): Action { - return { - kind: 'connector', - id: `connect-${connector.type}`, - label: `Connect ${connector.meta.name}`, - icon: connector.meta.icon, - target: connector, - } -} - -/** - * Builds the Search-mode rows: the pinned connectors first, then a uniform - * sample of the rest to fill four slots. A source the viewer has already - * connected is dropped from both halves, and a pinned slot freed that way is - * taken by the rotation. Sources this deployment cannot connect are dropped - * the same way, so a row never starts a connection that fails. - */ -export function computeConnectorActions( - connectedTypes: ReadonlySet, - isAvailable: (connector: SearchConnector) => boolean -): Action[] { - const offered = (connector: SearchConnector) => - isAvailable(connector) && !connectedTypes.has(connector.type) - const pinned = PINNED.filter(offered) - const pool = ROTATING.filter(offered) - const rotating = weightedSample(pool, CONNECTOR_ACTION_COUNT - pinned.length, () => 1) - return [...pinned, ...rotating].map(toConnectorAction) -} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx index b79be3ee8e5..b0674f4774b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx @@ -5,13 +5,9 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCaptureEvent, mockUseWorkspaceMemberConnectors, mockConnectSource } = vi.hoisted( - () => ({ - mockCaptureEvent: vi.fn(), - mockUseWorkspaceMemberConnectors: vi.fn(), - mockConnectSource: vi.fn(), - }) -) +const { mockCaptureEvent } = vi.hoisted(() => ({ + mockCaptureEvent: vi.fn(), +})) vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), @@ -32,12 +28,8 @@ vi.mock('@/hooks/queries/tables', () => ({ vi.mock('@/hooks/queries/kb/knowledge', () => ({ useKnowledgeBasesQuery: () => ({ data: [] }), })) -vi.mock('@/hooks/queries/kb/connectors', () => ({ - memberConnectorKeys: { list: (workspaceId?: string) => ['member-connectors', workspaceId] }, - useWorkspaceMemberConnectors: mockUseWorkspaceMemberConnectors, -})) -vi.mock('@/hooks/use-member-enrollment', () => ({ - useMemberEnrollment: () => ({ connectSource: mockConnectSource }), +vi.mock('@/app/workspace/[workspaceId]/home/components/search-sources', () => ({ + SearchSources: () =>
, })) vi.mock('@/hooks/use-permission-config', () => ({ usePermissionConfig: () => ({ @@ -106,27 +98,9 @@ function rows(): HTMLButtonElement[] { ) } -function connectModal(): string | null { - return document.querySelector('[data-testid="connect-modal"]')?.textContent ?? null -} - beforeEach(() => { onSelectPrompt.mockClear() mockCaptureEvent.mockClear() - mockUseWorkspaceMemberConnectors.mockReturnValue({ - isPending: false, - data: [ - { - knowledgeBaseId: 'kb-search', - knowledgeBaseName: 'Sim Search', - connectorId: 'conn-jira', - connectorType: 'jira', - memberSyncStatus: 'idle', - viewerMembership: 'connected', - viewerDocumentCount: 3, - }, - ], - }) useMothershipModeStore.getState().reset() }) @@ -145,41 +119,13 @@ describe('SuggestedActions', () => { expect(rows().map((row) => row.textContent)).toContain('Integrate with Slack') }) - it('swaps to the connector list in Search mode, minus connected and unavailable connectors', () => { + it('shows every source in Search mode instead of the sampled suggestions', () => { mount() act(() => useMothershipModeStore.getState().setMode('search')) - expect(heading()).toBe('Connect Sim Search') - expect(rows().map((row) => row.textContent)).toEqual([ - 'Connect Confluence', - 'Connect Jira Service Management', - 'Connect Airtable', - 'Connect Slack', - ]) - }) - - it('connects a source through its per-member connector instead of populating the input', () => { - mount() - act(() => useMothershipModeStore.getState().setMode('search')) - expect(connectModal()).toBeNull() - - act(() => { - rows()[0].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) - }) - - expect(connectModal()).toBeNull() - expect(mockConnectSource).toHaveBeenCalledWith('workspace-1', 'confluence') - expect(onSelectPrompt).not.toHaveBeenCalled() - expect(mockCaptureEvent).toHaveBeenCalledWith( - null, - 'suggested_action_clicked', - expect.objectContaining({ - kind: 'connector', - action_id: 'connect-confluence', - position: 0, - connected_provider_count: 1, - }) - ) + expect(heading()).toBe('Sources') + expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() + expect(rows()).toHaveLength(0) }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx index 7959f50dcd0..889c988aef7 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx @@ -13,9 +13,8 @@ import { resolveOAuthServiceForSlug, } from '@/lib/integrations' import { captureEvent } from '@/lib/posthog/client' -import { isSearchConnectorAvailable } from '@/lib/sim-search/connectors' import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' -import { computeConnectorActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/connector-actions' +import { SearchSources } from '@/app/workspace/[workspaceId]/home/components/search-sources' import type { Action, ActionIcon, @@ -26,15 +25,9 @@ import { BrandIcon } from '@/blocks/brand-icon' import { getAllBlockMeta } from '@/blocks/registry' import type { ModuleTag } from '@/blocks/types' import { useWorkspaceCredentials } from '@/hooks/queries/credentials' -import { - memberConnectorKeys, - useWorkspaceMemberConnectors, - type WorkspaceMemberConnector, -} from '@/hooks/queries/kb/connectors' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useOAuthConnections } from '@/hooks/queries/oauth/oauth-connections' import { useTablesList } from '@/hooks/queries/tables' -import { useMemberEnrollment } from '@/hooks/use-member-enrollment' import { usePermissionConfig } from '@/hooks/use-permission-config' import { type MothershipMode, useMothershipModeStore } from '@/stores/mothership-mode/store' @@ -156,7 +149,6 @@ function scoreCandidate(c: Candidate, signals: Signals): number { } const EMPTY_CREDENTIALS: NonNullable['data']> = [] -const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] const EMPTY_SERVICES: NonNullable['data']> = [] type ServiceInfo = NonNullable['data']>[number] @@ -242,7 +234,7 @@ const INITIAL_ACTIONS: Action[] = [ /** Section heading per composer mode — Search reads as a connect-your-sources list. */ const HEADINGS: Record = { build: 'Suggested actions', - search: 'Connect Sim Search', + search: 'Sources', } interface SuggestedActionsProps { @@ -264,8 +256,6 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { const { data: knowledgeBases = [] } = useKnowledgeBasesQuery(workspaceId, { enabled: Boolean(workspaceId), }) - const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS, isPending: connectionsPending } = - useWorkspaceMemberConnectors(workspaceId) const [expanded, setExpanded] = useState(true) /** @@ -302,28 +292,6 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { [connectedProviders, tables.length, knowledgeBases.length] ) - /** Sources the viewer has already connected, by connector type. */ - const connectedSearchTypes = useMemo( - () => - new Set( - memberConnectors - .filter((connector) => connector.viewerMembership === 'connected') - .map((connector) => connector.connectorType) - ), - [memberConnectors] - ) - const connectedConnectorIds = useMemo( - () => - new Set( - memberConnectors - .filter((connector) => connector.viewerMembership === 'connected') - .map((connector) => connector.connectorId) - ), - [memberConnectors] - ) - const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) - const { connectSource } = useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds }) - /** * Each mode's list is memoized on its own inputs alone, so switching modes — * or the other mode's signals settling — never re-samples it. @@ -337,21 +305,12 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { * flashes. The store's default mode is Build, so the server render never * shows the sampled Search list. */ - const searchActions = useMemo( - () => - connectionsPending - ? [] - : computeConnectorActions(connectedSearchTypes, (connector) => - isSearchConnectorAvailable(connector, integrationAvailability) - ), - [connectionsPending, connectedSearchTypes, integrationAvailability] - ) const buildActions = useMemo(() => { const personalized = services.length > 0 && connectedProviders.size > 0 if (!personalized) return INITIAL_ACTIONS return computeActions(services, signals) }, [connectedProviders, services, signals]) - const actions = mode === 'search' ? searchActions : buildActions + const actions = buildActions const handleSelect = (action: Action, position: number) => { captureEvent(posthog, 'suggested_action_clicked', { @@ -360,18 +319,12 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { action_id: action.id, label: action.label, position, - connected_provider_count: - action.kind === 'connector' ? connectedSearchTypes.size : connectedProviders.size, + connected_provider_count: connectedProviders.size, }) if (action.kind === 'prompt') { onSelectPrompt(action.prompt) return } - /** A Sim Search source connects through its per-member connector, not a bare credential. */ - if (action.kind === 'connector') { - if (workspaceId) connectSource(workspaceId, action.target.type) - return - } const target = resolveOAuthServiceForSlug(action.slug) if (target) setOAuthTarget(target) } @@ -419,28 +372,34 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { `collapsible-up`/`-down` interpolate height alone, so a margin here would hold its full value through the close and then vanish on unmount, snapping the content below up. */} -
- {actions.map((action, i) => { - const Icon = action.icon - return ( - - ) - })} -
+ {mode === 'search' && workspaceId ? ( +
+ +
+ ) : ( +
+ {actions.map((action, i) => { + const Icon = action.icon + return ( + + ) + })} +
+ )} {oauthTarget && workspaceId && ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts index 2165b1c8eef..9e39d99dbc1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts @@ -1,5 +1,4 @@ import type { ComponentType, CSSProperties } from 'react' -import type { SearchConnector } from '@/lib/sim-search/connectors' export type ActionIcon = ComponentType<{ className?: string; style?: CSSProperties }> @@ -14,11 +13,8 @@ export interface OAuthConnectTarget { /** * One suggested-action row. `prompt` rows populate the input with a curated * prompt; `integration` rows resolve their OAuth service from the catalog slug - * on click and open the OAuth connect modal; `connector` rows — the Search-mode - * "Connect X" rows — carry the Sim Search source, which connects through its - * per-member connector. + * on click and open the OAuth connect modal. */ export type Action = | { kind: 'prompt'; id: string; label: string; icon: ActionIcon; prompt: string } | { kind: 'integration'; id: string; label: string; icon: ActionIcon; slug: string } - | { kind: 'connector'; id: string; label: string; icon: ActionIcon; target: SearchConnector } diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index b3e620cd2b7..32e2fec5c9d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -469,12 +469,21 @@ export function Home({ chatId, userName, userId }: HomeProps) { }, [handleSubmit] ) + /** + * A chat that already exists opens in Build: its transcript is a + * conversation, and search results never join it. A new chat keeps whatever + * mode the person used last. + */ + useEffect(() => { + if (chatId) useMothershipModeStore.getState().setMode('build') + }, [chatId]) const showSearchResults = composerMode === 'search' && searchQuery.length > 0 const searchResults = showSearchResults ? ( ) : null From dba3510978c9203d9379e6a65ebdc9a8073922c5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 01:17:22 -0700 Subject: [PATCH 33/76] fix(knowledge): process connector-owned documents as the system Members-mode documents are inserted hidden until the member sync materializes who observed them, so the processor's own source-file read, authorized as the actor with workspace scope, denied every document a per-member connector created. Connector-owned rows now read their source as the system; uploads keep the actor's authorization. --- apps/sim/app/api/files/authorization.ts | 16 ++-- .../document-processing-source.test.ts | 34 ++++++++ .../knowledge/documents/document-processor.ts | 81 +++++++++++-------- apps/sim/lib/knowledge/documents/service.ts | 17 +++- .../lib/uploads/utils/file-utils.server.ts | 21 ++++- 5 files changed, 122 insertions(+), 47 deletions(-) diff --git a/apps/sim/app/api/files/authorization.ts b/apps/sim/app/api/files/authorization.ts index c3eb0f50f7e..db10d3b7820 100644 --- a/apps/sim/app/api/files/authorization.ts +++ b/apps/sim/app/api/files/authorization.ts @@ -9,7 +9,7 @@ import { resolveUserKnowledgeAccessScope, WORKSPACE_ACCESS_SCOPE, } from '@/lib/knowledge/access/scope' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' import { getFileMetadata } from '@/lib/uploads' import type { StorageContext } from '@/lib/uploads/config' import type { StorageConfig } from '@/lib/uploads/core/storage-client' @@ -494,7 +494,7 @@ async function verifyCopilotFileAccess( async function hasActiveKbDocumentForKey( cloudKey: string, workspaceId: string, - access: KnowledgeAccessScope + access: KnowledgeAccessScope | SystemAccessScope ): Promise { const rows = await db .select({ id: document.id }) @@ -519,17 +519,19 @@ async function hasActiveKbDocumentForKey( /** * How a KB file read identifies the reader for document access. `'user'` is * for a session-authenticated person; a resolved scope is for a caller that - * already holds one (an execution with a principal). Anything else — an - * internal token, a tool running with the workflow owner's id — reads as the - * workspace, never as the person whose id it happens to carry. + * already holds one (an execution with a principal). The system scope is for + * a background job reading a connector-owned row it is processing, which in + * members mode is hidden until the sync materializes its readers. Anything + * else — an internal token, a tool running with the workflow owner's id — + * reads as the workspace, never as the person whose id it happens to carry. */ -export type KnowledgeFileAccess = 'user' | KnowledgeAccessScope +export type KnowledgeFileAccess = 'user' | KnowledgeAccessScope | SystemAccessScope async function resolveKnowledgeFileAccess( knowledgeAccess: KnowledgeFileAccess | undefined, userId: string, workspaceId: string -): Promise { +): Promise { if (knowledgeAccess === 'user') return resolveUserKnowledgeAccessScope(userId, workspaceId) return knowledgeAccess ?? WORKSPACE_ACCESS_SCOPE } diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 96f2aee25ac..9d1886eb7ef 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -78,6 +78,7 @@ import { EMBEDDING_QUOTA_EXHAUSTED_MESSAGE, } from '@/lib/embeddings' import { EmbeddingAPIError, EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { PermanentDocumentProcessingError, UsageLimitDocumentProcessingError, @@ -100,6 +101,7 @@ const PERSISTED_CONTEXT = { fileUrl: PERSISTED_URL, fileSize: 512, mimeType: 'application/pdf', + connectorId: null, tag1: null, tag2: null, tag3: null, @@ -220,11 +222,42 @@ describe('knowledge document processing source', () => { PERSISTED_CONTEXT.uploadedBy, null, undefined, + undefined, undefined ) expect(mockGenerateEmbeddings).not.toHaveBeenCalled() }) + it('reads a connector-owned source file as the system, not as the actor', async () => { + resetDbChainMock() + dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }]) + dbChainMockFns.limit + .mockResolvedValueOnce([{ ...PERSISTED_CONTEXT, connectorId: 'connector-1' }]) + .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW]) + .mockResolvedValueOnce([{ id: 'document-1' }]) + + await processDocumentAsync('knowledge-base-1', 'document-1', { + filename: PERSISTED_CONTEXT.filename, + fileUrl: PERSISTED_CONTEXT.fileUrl, + fileSize: PERSISTED_CONTEXT.fileSize, + mimeType: PERSISTED_CONTEXT.mimeType, + }) + + expect(mockProcessDocument).toHaveBeenCalledWith( + PERSISTED_CONTEXT.fileUrl, + PERSISTED_CONTEXT.filename, + PERSISTED_CONTEXT.mimeType, + 1024, + 200, + 100, + PERSISTED_CONTEXT.uploadedBy, + null, + undefined, + undefined, + SYSTEM_ACCESS_SCOPE + ) + }) + it('processes a legacy document when its workspace metadata row no longer exists', async () => { mockGetFileMetadataByKeys.mockResolvedValue([]) mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(new Map()) @@ -246,6 +279,7 @@ describe('knowledge document processing source', () => { PERSISTED_CONTEXT.uploadedBy, null, undefined, + undefined, undefined ) }) diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index ed486c519a6..8172f9e755f 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -60,7 +60,10 @@ import { import { StorageService } from '@/lib/uploads' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' import { getFileExtension, isInternalFileUrl } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' +import { + type DownloadFileFromUrlOptions, + downloadFileFromUrl, +} from '@/lib/uploads/utils/file-utils.server' import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' import { mistralParserTool } from '@/tools/mistral/parser' @@ -194,6 +197,12 @@ async function applyStrategy( } } +/** + * Who a source-file read runs as: the actor for authorization and OCR + * attribution, plus how a knowledge-base file identifies its reader. + */ +type SourceFileAccess = Pick + export async function processDocument( fileUrl: string, filename: string, @@ -204,7 +213,8 @@ export async function processDocument( userId?: string, workspaceId?: string | null, strategy?: ChunkingStrategy, - strategyOptions?: StrategyOptions + strategyOptions?: StrategyOptions, + knowledgeAccess?: DownloadFileFromUrlOptions['knowledgeAccess'] ): Promise<{ chunks: Chunk[] metadata: { @@ -219,9 +229,10 @@ export async function processDocument( } }> { logger.info('Processing document', { mimeType }) + const access: SourceFileAccess = { userId, knowledgeAccess } try { - const parseResult = await parseDocument(fileUrl, filename, mimeType, userId, workspaceId) + const parseResult = await parseDocument(fileUrl, filename, mimeType, access, workspaceId) const { content, processingMethod } = parseResult const cloudUrl = 'cloudUrl' in parseResult ? parseResult.cloudUrl : undefined @@ -355,7 +366,7 @@ async function readEmbeddedPdfText( fileUrl: string, filename: string, mimeType: string, - userId?: string + access: SourceFileAccess ): Promise< | { content: string @@ -366,7 +377,7 @@ async function readEmbeddedPdfText( | undefined > { try { - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) const parsed = await parseBuffer(buffer, 'pdf') /** @@ -408,7 +419,7 @@ async function parseDocument( fileUrl: string, filename: string, mimeType: string, - userId?: string, + access: SourceFileAccess, workspaceId?: string | null ): Promise<{ content: string @@ -435,30 +446,30 @@ async function parseDocument( * documents that actually need it — which also means everything else stops * depending on that service being reachable. */ - const embedded = await readEmbeddedPdfText(fileUrl, filename, mimeType, userId) + const embedded = await readEmbeddedPdfText(fileUrl, filename, mimeType, access) if (embedded) return embedded assertKnowledgeOpaqueModelInputSafe() if (ocrProvider === 'azure-mistral') { logger.info('Using Azure Mistral OCR') - return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId) + return parseWithAzureMistralOCR(fileUrl, filename, mimeType, access) } logger.info('Using Mistral OCR') - return parseWithMistralOCR(fileUrl, filename, mimeType, userId, workspaceId, mistralApiKey) + return parseWithMistralOCR(fileUrl, filename, mimeType, access, workspaceId, mistralApiKey) } } logger.info('Using file parser') - return parseWithFileParser(fileUrl, filename, mimeType, userId) + return parseWithFileParser(fileUrl, filename, mimeType, access) } async function handleFileForOCR( fileUrl: string, filename: string, mimeType: string, - userId?: string, + access: SourceFileAccess, workspaceId?: string | null ) { const isExternalHttps = /^https:\/\//i.test(fileUrl) && !isInternalFileUrl(fileUrl) @@ -466,7 +477,7 @@ async function handleFileForOCR( if (isExternalHttps) { if (mimeType === 'application/pdf') { logger.info('handleFileForOCR: Downloading external PDF for OCR admission') - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) logger.info('handleFileForOCR: Downloaded external PDF', { bytes: buffer.length }) return { httpsUrl: fileUrl, buffer } } @@ -476,7 +487,7 @@ async function handleFileForOCR( logger.info('Uploading document to cloud storage for OCR') - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) logger.info('Downloaded document for OCR', { bytes: buffer.length }) @@ -485,7 +496,7 @@ async function handleFileForOCR( originalName: filename, uploadedAt: new Date().toISOString(), purpose: 'knowledge-base', - ...(userId && { userId }), + ...(access.userId && { userId: access.userId }), ...(workspaceId && { workspaceId }), } @@ -521,20 +532,20 @@ async function handleFileForOCR( * up front on an oversized `Content-Length`), so an attacker-controlled `fileUrl` * pointing at an unbounded body cannot exhaust the processing worker's memory. */ -async function downloadFileWithTimeout(fileUrl: string, userId?: string): Promise { +async function downloadFileWithTimeout(fileUrl: string, access: SourceFileAccess): Promise { return downloadFileFromUrl(fileUrl, { timeoutMs: TIMEOUTS.FILE_DOWNLOAD, maxBytes: MAX_FILE_SIZE, - userId, + ...access, }) } -async function downloadFileForBase64(fileUrl: string, userId?: string): Promise { +async function downloadFileForBase64(fileUrl: string, access: SourceFileAccess): Promise { if (/^data:/i.test(fileUrl)) { return decodeDataUriWithinLimit(fileUrl, MAX_FILE_SIZE).buffer } if (/^https?:\/\//i.test(fileUrl) || isInternalFileUrl(fileUrl)) { - return downloadFileWithTimeout(fileUrl, userId) + return downloadFileWithTimeout(fileUrl, access) } throw new Error( 'Unsupported fileUrl scheme: only data: URIs, http(s):// URLs, and internal /api/files/serve/ paths are allowed' @@ -678,7 +689,7 @@ async function parseWithAzureMistralOCR( fileUrl: string, filename: string, mimeType: string, - userId?: string + access: SourceFileAccess ) { validateOCRConfig( env.OCR_AZURE_API_KEY, @@ -687,7 +698,7 @@ async function parseWithAzureMistralOCR( 'Azure Mistral OCR' ) - const fileBuffer = await downloadFileForBase64(fileUrl, userId) + const fileBuffer = await downloadFileForBase64(fileUrl, access) const requestPolicy = getAzureMistralOcrRequestPolicy(env.OCR_AZURE_MODEL_NAME!) try { @@ -792,7 +803,7 @@ async function parseWithMistralOCR( fileUrl: string, filename: string, mimeType: string, - userId?: string, + access: SourceFileAccess, workspaceId?: string | null, mistralApiKey?: string | null ) { @@ -805,7 +816,7 @@ async function parseWithMistralOCR( fileUrl, filename, mimeType, - userId, + access, workspaceId ) @@ -829,13 +840,13 @@ async function parseWithMistralOCR( maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes, maxPages: MISTRAL_OCR_REQUEST_POLICY.maxPages, }) - return processMistralOCRInBatches(filename, apiKey, buffer, userId, cloudUrl) + return processMistralOCRInBatches(filename, apiKey, buffer, access, cloudUrl) } const params = { filePath: httpsUrl, apiKey, resultType: 'text' as const } try { - const response = await executeMistralOCRRequest(params, userId) + const response = await executeMistralOCRRequest(params, access) const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult const content = processOCRContent(result, filename, pageCount > 0 ? pageCount : undefined) @@ -850,7 +861,7 @@ async function parseWithMistralOCR( async function executeMistralOCRRequest( params: { filePath: string; apiKey: string; resultType: 'text' }, - userId?: string + access: SourceFileAccess ): Promise { return retryWithExponentialBackoff( async () => { @@ -876,7 +887,7 @@ async function executeMistralOCRRequest( requestId: generateId(), signal: controller.signal, trustedCaller: 'knowledge-ingestion', - userId, + userId: access.userId, }) return Response.json(result) } catch (error) { @@ -905,7 +916,7 @@ async function processChunk( chunkIndex: number, filename: string, apiKey: string, - userId?: string + access: SourceFileAccess ): Promise { const chunkPageCount = chunk.endPage - chunk.startPage + 1 @@ -955,7 +966,7 @@ async function processChunk( resultType: 'text' as const, } - const response = await executeMistralOCRRequest(params, userId) + const response = await executeMistralOCRRequest(params, access) const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult if (!result.success) { @@ -1192,7 +1203,7 @@ async function processMistralOCRInBatches( filename: string, apiKey: string, pdfBuffer: Buffer, - userId?: string, + access: SourceFileAccess, cloudUrl?: string ): Promise<{ content: string @@ -1204,7 +1215,7 @@ async function processMistralOCRInBatches( 'mistral', filename, MISTRAL_OCR_REQUEST_POLICY, - (chunk, index) => processChunk(chunk, index, filename, apiKey, userId) + (chunk, index) => processChunk(chunk, index, filename, apiKey, access) ) return { content, processingMethod: 'mistral-ocr', cloudUrl } @@ -1231,7 +1242,7 @@ async function parseWithFileParser( fileUrl: string, filename: string, mimeType: string, - userId?: string + access: SourceFileAccess ) { try { let content: string @@ -1245,7 +1256,7 @@ async function parseWithFileParser( // Internal URLs may arrive as an app-relative `/api/files/serve/...` path // (some ingestion callers store the relative path); downloadFileFromUrl // resolves it directly against storage without an absolute origin. - const result = await parseHttpFile(fileUrl, filename, mimeType, userId) + const result = await parseHttpFile(fileUrl, filename, mimeType, access) content = result.content metadata = result.metadata || {} } else { @@ -1275,10 +1286,10 @@ async function parseDataURI( async function parseHttpFile( fileUrl: string, filename: string, - mimeType?: string, - userId?: string + mimeType: string | undefined, + access: SourceFileAccess ): Promise<{ content: string; metadata?: FileParseMetadata }> { - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) /** Prefer what we actually downloaded over what the document is *called*. */ const extension = diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 73e530f3b63..9cedc591da5 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -77,7 +77,11 @@ import { mergeDurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' +import { + type KnowledgeAccessScope, + SYSTEM_ACCESS_SCOPE, + type SystemAccessScope, +} from '@/lib/knowledge/access/types' import { assertDocumentChunkCountWithinLimit, isPermanentDocumentProcessingError, @@ -1360,6 +1364,7 @@ export async function processDocumentAsync( embeddingModel: knowledgeBase.embeddingModel, billedAccountUserId: workspaceTable.billedAccountUserId, uploadedBy: document.uploadedBy, + connectorId: document.connectorId, filename: document.filename, fileUrl: document.fileUrl, fileSize: document.fileSize, @@ -1578,7 +1583,15 @@ export async function processDocumentAsync( documentActorUserId, ctx.workspaceId, rawConfig?.strategy, - rawConfig?.strategyOptions + rawConfig?.strategyOptions, + /** + * A connector-owned row was written by the sync from bytes it fetched, + * not from a caller-supplied URL, so the processor reads it as the + * system: in members mode the row stays hidden until the sync + * materializes who observed it, and the actor's own scope would deny + * the read. Uploads keep the actor's authorization above. + */ + ctx.connectorId ? SYSTEM_ACCESS_SCOPE : undefined ) assertDocumentChunkCountWithinLimit(processed.chunks.length) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index d3cb23db6c9..b78bff2b00b 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -37,7 +37,7 @@ import { } from '@/lib/uploads/utils/file-utils' import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' -import { verifyFileAccess } from '@/app/api/files/authorization' +import { type KnowledgeFileAccess, verifyFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' const logger = createLogger('FileUtilsServer') @@ -229,6 +229,13 @@ export interface DownloadFileFromUrlOptions { * be treated as implicitly trusted. */ userId?: string + /** + * How a knowledge-base file identifies its reader. Omitted, the read is + * authorized as the workspace, which is what a caller-supplied URL gets. A + * background job processing a connector-owned row passes the system scope, + * because that row is hidden until the sync materializes who may read it. + */ + knowledgeAccess?: KnowledgeFileAccess } /** @@ -248,7 +255,13 @@ export async function downloadFileFromUrl( fileUrl: string, options: DownloadFileFromUrlOptions = {} ): Promise { - const { timeoutMs = getMaxExecutionTimeout(), maxBytes, signal, userId } = options + const { + timeoutMs = getMaxExecutionTimeout(), + maxBytes, + signal, + userId, + knowledgeAccess, + } = options signal?.throwIfAborted() @@ -266,7 +279,9 @@ export async function downloadFileFromUrl( const context = inferContextFromKey(key) - const hasAccess = await verifyFileAccess(key, userId, undefined, context, false) + const hasAccess = await verifyFileAccess(key, userId, undefined, context, false, { + knowledgeAccess, + }) if (!hasAccess) { logger.warn('Internal file download denied: access check failed', { key, context, userId }) throw new Error('Access denied: file not found or insufficient permissions') From 662119a10314e009e62e3072a695496c1c22235d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 01:26:38 -0700 Subject: [PATCH 34/76] feat(search): connect every personal source in place The Sources strip lists only what a person can connect themselves and asks for a source's required setup (a site, a space) in a small modal on the first connect; sources an admin must set up as workspace connectors no longer appear as dead chips. The indexing loader spins, the query stays in the composer after a search, and the knowledge-base page drops the per-member sync banner. --- .../search-sources/search-sources.tsx | 99 ++++++++++--------- .../search-sources/source-setup-modal.tsx | 83 ++++++++++++++++ .../home/components/user-input/user-input.tsx | 30 ++++-- .../app/workspace/[workspaceId]/home/home.tsx | 1 + .../[workspaceId]/knowledge/[id]/base.tsx | 2 - .../knowledge/[id]/components/index.ts | 1 - .../member-connect-banner.tsx | 89 ----------------- .../[workspaceId]/search/search.test.tsx | 2 +- .../workspace/[workspaceId]/search/search.tsx | 23 ++++- apps/sim/hooks/use-member-enrollment.ts | 14 ++- .../lib/api/contracts/knowledge/connectors.ts | 2 + .../lib/knowledge/application/sim-search.ts | 27 +++-- apps/sim/lib/sim-search/connectors.test.ts | 24 ++++- apps/sim/lib/sim-search/connectors.ts | 38 +++++-- 14 files changed, 264 insertions(+), 171 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/member-connect-banner/member-connect-banner.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx index 1f0c57a2124..658f2e101c4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx @@ -1,16 +1,17 @@ 'use client' -import { useMemo } from 'react' -import { Chip, cn } from '@sim/emcn' +import { useMemo, useState } from 'react' +import { Chip } from '@sim/emcn' import { Loader, Plus } from '@sim/emcn/icons' -import Link from 'next/link' import { canConnectPersonally, isSearchConnectorAvailable, + personalSetupFields, SEARCH_CONNECTORS, type SearchConnector, SIM_SEARCH_KNOWLEDGE_BASE_NAME, } from '@/lib/sim-search/connectors' +import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' import { BrandIcon } from '@/blocks/brand-icon' import { memberConnectorKeys, @@ -22,6 +23,16 @@ import { usePermissionConfig } from '@/hooks/use-permission-config' const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] +/** The sources a person can connect themselves, alphabetical. */ +const PERSONAL_SEARCH_CONNECTORS = SEARCH_CONNECTORS.filter((connector) => + canConnectPersonally(connector.meta) +) + +/** A chip's trailing icon is a component, so the spinning loader needs a wrapper to carry `animate`. */ +function SpinningLoader({ className }: { className?: string }) { + return +} + /** The Sim Search connection per source, keyed by connector type. */ export function simSearchConnectionsByType( connectors: readonly WorkspaceMemberConnector[] @@ -103,7 +114,7 @@ function SourceChip({ onClick={actionable ? onConnect : undefined} title={title} leftAdornment={} - rightIcon={waiting || isIndexing(connection) ? Loader : actionable ? Plus : undefined} + rightIcon={waiting || isIndexing(connection) ? SpinningLoader : actionable ? Plus : undefined} > {connector.meta.name} @@ -118,11 +129,12 @@ interface SearchSourcesProps { } /** - * Every source Sim Search can index for the person, as chips under the - * composer: connected ones show how many documents they can read (or that - * indexing is still running), the rest connect with one click. Sources that - * need a site or space link to Knowledge, where an admin can name it. This is - * the whole catalog, not a sample, so nothing connectable stays hidden. + * Every source a person can connect themselves, as chips under the composer: + * connected ones show how many documents they can read (or that indexing is + * still running), the rest connect with one click. A source that needs a site + * or space asks for it once, in place, on the connect that creates it; + * everyone after that clicks straight through. Sources an admin must set up + * as workspace connectors do not appear here. */ export function SearchSources({ workspaceId }: SearchSourcesProps) { const { integrationAvailability } = usePermissionConfig() @@ -146,50 +158,35 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { membershipQueryKeys, connectedConnectorIds, }) + const [setupConnector, setSetupConnector] = useState(null) - /** Connected first, then one-click sources, then those that need a knowledge base. */ + /** Connected sources first, then the rest alphabetically. */ const ordered = useMemo(() => { - const rank = (connector: SearchConnector) => { - const connection = connectionByType.get(connector.type) - if (connection?.viewerMembership === 'connected') return 0 - if (connection) return 1 - return canConnectPersonally(connector.meta) ? 2 : 3 - } - return [...SEARCH_CONNECTORS].sort( + const rank = (connector: SearchConnector) => + connectionByType.get(connector.type)?.viewerMembership === 'connected' ? 0 : 1 + return [...PERSONAL_SEARCH_CONNECTORS].sort( (a, b) => rank(a) - rank(b) || a.meta.name.localeCompare(b.meta.name) ) }, [connectionByType]) + const startConnect = (connector: SearchConnector) => { + const connection = connectionByType.get(connector.type) + if (connection) { + connect(connection.knowledgeBaseId, connection.connectorId) + return + } + if (personalSetupFields(connector.meta).length > 0) { + setSetupConnector(connector) + return + } + connectSource(workspaceId, connector.type) + } + return (
{ordered.map((connector) => { const connection = connectionByType.get(connector.type) - if (!canConnectPersonally(connector.meta) && !connection) { - return ( - - - } - > - - {connector.meta.name} - - Set up in Knowledge - - - - - ) - } return ( - connection - ? connect(connection.knowledgeBaseId, connection.connectorId) - : connectSource(workspaceId, connector.type) - } + onConnect={() => startConnect(connector)} /> ) })}
{error &&

{error}

} + {setupConnector && ( + { + if (!open) setSetupConnector(null) + }} + onConnect={(sourceConfig) => + connectSource(workspaceId, setupConnector.type, sourceConfig) + } + /> + )}
) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx new file mode 100644 index 00000000000..71695341983 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx @@ -0,0 +1,83 @@ +'use client' + +import { useState } from 'react' +import { + ChipModal, + ChipModalBody, + ChipModalField, + ChipModalFooter, + ChipModalHeader, +} from '@sim/emcn' +import type { SearchConnector } from '@/lib/sim-search/connectors' +import type { ConnectorConfigField } from '@/connectors/types' + +interface SourceSetupModalProps { + connector: SearchConnector + fields: readonly ConnectorConfigField[] + onOpenChange: (open: boolean) => void + /** Connects the source with the filled-in fields; the caller opens the OAuth tab in this click. */ + onConnect: (sourceConfig: Record) => void +} + +/** + * The few fields a source needs before its first connect, such as a site and + * a space. Everyone after the first person clicks straight through. + */ +export function SourceSetupModal({ + connector, + fields, + onOpenChange, + onConnect, +}: SourceSetupModalProps) { + const [values, setValues] = useState>({}) + const complete = fields.every((field) => values[field.id]?.trim()) + + const submit = () => { + if (!complete) return + onConnect(Object.fromEntries(fields.map((field) => [field.id, values[field.id]?.trim() ?? '']))) + onOpenChange(false) + } + + return ( + + onOpenChange(false)}> + Connect {connector.meta.name} + + + {fields.map((field) => + field.type === 'dropdown' ? ( + setValues((current) => ({ ...current, [field.id]: value }))} + options={(field.options ?? []).map((option) => ({ + value: option.id, + label: option.label, + }))} + placeholder={field.placeholder} + required + /> + ) : ( + setValues((current) => ({ ...current, [field.id]: value }))} + placeholder={field.placeholder} + hint={field.description} + autoComplete='off' + required + /> + ) + )} + + onOpenChange(false)} + primaryAction={{ label: 'Connect', onClick: submit, disabled: !complete }} + /> + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 3b0bb976a7b..14d4983b586 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -70,6 +70,13 @@ interface UserInputProps { onStopGeneration: () => void isInitialView?: boolean onSendQueuedHead?: () => void + /** + * Whether the text is cleared once submitted. A search keeps its query in + * the box, the way a search bar does, so it can be read and refined against + * the results; a message to the agent clears, since it now lives in the + * transcript. Defaults to clearing. + */ + clearOnSubmit?: boolean onEditQueuedTail?: () => void } @@ -98,6 +105,7 @@ const UserInputImpl = forwardRef(function UserI isInitialView = true, onSendQueuedHead, onEditQueuedTail, + clearOnSubmit = true, }, ref ) { @@ -158,6 +166,8 @@ const UserInputImpl = forwardRef(function UserI const draftScopeKeyRef = useRef(draftScopeKey) draftScopeKeyRef.current = draftScopeKey + const clearOnSubmitRef = useRef(clearOnSubmit) + clearOnSubmitRef.current = clearOnSubmit const hasRestoredDraftRef = useRef(false) useEffect(() => { @@ -546,15 +556,17 @@ const UserInputImpl = forwardRef(function UserI fileAttachmentsForApi.length > 0 ? fileAttachmentsForApi : undefined, activeContexts.length > 0 ? activeContexts : undefined ) - currentEditor.clear() - sttPrefixRef.current = '' - if (draftSaveTimerRef.current !== null) { - window.clearTimeout(draftSaveTimerRef.current) - draftSaveTimerRef.current = null - } - pendingDraftRef.current = null - if (draftScopeKeyRef.current) { - useMothershipDraftsStore.getState().clearDraft(draftScopeKeyRef.current) + if (clearOnSubmitRef.current) { + currentEditor.clear() + sttPrefixRef.current = '' + if (draftSaveTimerRef.current !== null) { + window.clearTimeout(draftSaveTimerRef.current) + draftSaveTimerRef.current = null + } + pendingDraftRef.current = null + if (draftScopeKeyRef.current) { + useMothershipDraftsStore.getState().clearDraft(draftScopeKeyRef.current) + } } resetTranscript() currentFiles.clearAttachedFiles() diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 32e2fec5c9d..3479819cb78 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -705,6 +705,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { defaultValue={initialPrompt} draftScopeKey={draftScopeKey} onSubmit={handleSubmit} + clearOnSubmit={composerMode !== 'search'} isSending={isSending} onStopGeneration={handleStopGeneration} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index 4c393eecbba..1176e7250bb 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -91,7 +91,6 @@ import { BaseTagsModal, ConnectorsSection, DocumentContextMenu, - MemberConnectBanner, RenameDocumentModal, } from '@/app/workspace/[workspaceId]/knowledge/[id]/components' import { DOCUMENT_COLUMNS } from '@/app/workspace/[workspaceId]/knowledge/[id]/document-columns' @@ -1313,7 +1312,6 @@ export function KnowledgeBase({ breadcrumbs={breadcrumbs} actions={headerActions} /> - - new Set( - connectors - .filter((connector) => connector.viewerMembership === 'connected') - .map((connector) => connector.id) - ), - [connectors] - ) - const membershipQueryKeys = useMemo( - () => [connectorKeys.lists(knowledgeBaseId)], - [knowledgeBaseId] - ) - const { connect, isAwaiting, isPending, error } = useMemberEnrollment({ - membershipQueryKeys, - connectedConnectorIds, - }) - - const rows = connectors.flatMap((connector) => { - const membership = connector.viewerMembership - if (connector.accessMode !== 'members' || membership === null) return [] - const waiting = isAwaiting(connector.id) - const text = describeMembership({ - membership, - memberSyncStatus: connector.memberSyncStatus, - waiting, - name: connectorName(connector), - }) - return text ? [{ connector, membership, waiting, text }] : [] - }) - if (rows.length === 0) return null - - return ( -
- {rows.map(({ connector, membership, waiting, text }) => ( -
-

- {(membership === 'connected' || waiting) && ( - - )} - {text} -

- {CONNECTABLE_MEMBERSHIPS.has(membership) && ( - - )} -
- ))} - {error &&

{error}

} -
- ) -} diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx index e7b33c51e54..24a689f2210 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx @@ -161,7 +161,7 @@ describe('Search', () => { expect(sectionLabels()).toEqual(['Sim Search Connectors', 'Shared with you']) const text = container?.textContent ?? '' expect(text).toContain('Connected · 12 documents') - expect(text).toContain('Needs a site or space; set it up from a knowledge base.') + expect(text).toContain('Set up by a workspace admin from a knowledge base.') expect(text).toContain('Unavailable in this deployment. Contact your administrator.') expect(text).toContain('Sales') }) diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx index 2fbad0bca8e..1303e258856 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -1,6 +1,6 @@ 'use client' -import { useMemo, useRef } from 'react' +import { useMemo, useRef, useState } from 'react' import { Button, ChipInput } from '@sim/emcn' import { Search as SearchIcon } from '@sim/emcn/icons' import { useParams } from 'next/navigation' @@ -8,11 +8,13 @@ import { useQueryState } from 'nuqs' import { canConnectPersonally, isSearchConnectorAvailable, + personalSetupFields, SEARCH_CONNECTORS, type SearchConnector, SIM_SEARCH_KNOWLEDGE_BASE_NAME, } from '@/lib/sim-search/connectors' import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/components' +import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' @@ -42,7 +44,7 @@ import { usePermissionConfig } from '@/hooks/use-permission-config' const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] const CONNECTORS_LABEL = 'Sim Search Connectors' -const NEEDS_KNOWLEDGE_BASE_SETUP = 'Needs a site or space; set it up from a knowledge base.' +const NEEDS_KNOWLEDGE_BASE_SETUP = 'Set up by a workspace admin from a knowledge base.' const UNAVAILABLE = 'Unavailable in this deployment. Contact your administrator.' /** What a source row says once the viewer's own indexing has settled. */ @@ -163,6 +165,7 @@ export function Search() { [memberConnectors] ) const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) + const [setupConnector, setSetupConnector] = useState(null) const { connect, connectSource, isAwaiting, isPending, error } = useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds, @@ -220,7 +223,9 @@ export function Search() { onConnect={() => connection ? connect(connection.knowledgeBaseId, connection.connectorId) - : connectSource(workspaceId, connector.type) + : personalSetupFields(connector.meta).length > 0 + ? setSetupConnector(connector) + : connectSource(workspaceId, connector.type) } /> ) @@ -234,6 +239,18 @@ export function Search() { /> {error &&

{error}

} + {setupConnector && ( + { + if (!open) setSetupConnector(null) + }} + onConnect={(sourceConfig) => + connectSource(workspaceId, setupConnector.type, sourceConfig) + } + /> + )} {showNoResults && ( diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index c4795b19752..b8d36ea3eb4 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -165,11 +165,19 @@ export function useMemberEnrollment({ ) ) - /** Connects a Sim Search source: its per-member connector exists afterwards, and the viewer enrolls. */ - const connectSource = (workspaceId: string, connectorType: string) => + /** + * Connects a Sim Search source: its per-member connector exists afterwards, + * and the viewer enrolls. The setup fields are read only when this connect + * creates the connector. + */ + const connectSource = ( + workspaceId: string, + connectorType: string, + sourceConfig?: Record + ) => openEnrollment(({ onSuccess, onError }) => sourceConnection.mutate( - { workspaceId, connectorType }, + { workspaceId, connectorType, sourceConfig }, { onSuccess: ({ url, connectorId }) => onSuccess(url, connectorId), onError: (err) => { diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index dd48f20f57d..e35d9c10b5a 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -365,6 +365,8 @@ export type WorkspaceMemberConnector = z.output diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts index 409d10c1ade..e63de78a909 100644 --- a/apps/sim/lib/knowledge/application/sim-search.ts +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -8,7 +8,11 @@ import { createKnowledgeConnector } from '@/lib/knowledge/application/connectors import { resolveKnowledgeWorkspaceContext } from '@/lib/knowledge/application/contexts' import { createKnowledgeBase } from '@/lib/knowledge/application/knowledge-bases' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { canConnectPersonally, SIM_SEARCH_KNOWLEDGE_BASE_NAME } from '@/lib/sim-search/connectors' +import { + canConnectPersonally, + missingSetupFields, + SIM_SEARCH_KNOWLEDGE_BASE_NAME, +} from '@/lib/sim-search/connectors' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' const SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION = @@ -20,6 +24,8 @@ export interface ConnectSimSearchConnectorInput { workspaceId: string /** `CONNECTOR_META_REGISTRY` key of the source to connect. */ connectorType: string + /** The source's setup fields (a site, a space); read only when this connect creates the connector. */ + sourceConfig?: Record } export interface ConnectSimSearchConnectorResult { @@ -69,9 +75,10 @@ async function findSimSearchKnowledgeBase(workspaceId: string) { /** * One click on a Sim Search source: the workspace's Sim Search knowledge base * and a per-member connector for that source exist after this (the first - * connect creates them, which the connector operation reserves for an admin), - * and the caller gets the link that connects their own account. The OAuth - * completion queues their member run, so indexing starts on its own. + * connect creates them, which the connector operation reserves for an admin, + * and supplies the source's setup fields when it has any), and the caller + * gets the link that connects their own account. The OAuth completion queues + * their member run, so indexing starts on its own. */ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.simSearchConnect, @@ -82,12 +89,20 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ if (!meta || !canConnectPersonally(meta)) { throw new OrchestrationError( 'validation', - 'This source needs a site or space to be set up from a knowledge base first' + 'This source cannot be connected per person; a workspace admin sets it up from a knowledge base' ) } const workspaceId = context.workspaceId let target = await findSimSearchConnector(workspaceId, input.connectorType) if (!target) { + const sourceConfig = input.sourceConfig ?? {} + const missing = missingSetupFields(meta, sourceConfig) + if (missing.length > 0) { + throw new OrchestrationError( + 'validation', + `${meta.name} needs ${missing.map((field) => field.title).join(' and ')} to connect` + ) + } const knowledgeBaseId = (await findSimSearchKnowledgeBase(workspaceId))?.id ?? ( @@ -108,7 +123,7 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ knowledgeBaseId, assertedWorkspaceId: workspaceId, connectorType: input.connectorType, - sourceConfig: {}, + sourceConfig, syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, accessMode: 'members', source: 'ui', diff --git a/apps/sim/lib/sim-search/connectors.test.ts b/apps/sim/lib/sim-search/connectors.test.ts index 264d77cf56f..1f8dd35edd8 100644 --- a/apps/sim/lib/sim-search/connectors.test.ts +++ b/apps/sim/lib/sim-search/connectors.test.ts @@ -81,6 +81,8 @@ vi.mock('@/lib/integrations/credential-display', () => ({ import { canConnectPersonally, isSearchConnectorAvailable, + missingSetupFields, + personalSetupFields, SEARCH_CONNECTORS, } from '@/lib/sim-search/connectors' @@ -112,11 +114,29 @@ describe('SEARCH_CONNECTORS', () => { }) describe('canConnectPersonally', () => { - it('offers one-click connection only to per-member sources with no required setup', () => { + it('offers personal connection to OAuth sources whose listing is permission-scoped', () => { const drive = SEARCH_CONNECTORS.find((connector) => connector.type === 'google_drive')! const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + const gmail = SEARCH_CONNECTORS.find((connector) => connector.type === 'gmail')! expect(canConnectPersonally(drive.meta)).toBe(true) - expect(canConnectPersonally(jira.meta)).toBe(false) + expect(canConnectPersonally(jira.meta)).toBe(true) + expect(canConnectPersonally(gmail.meta)).toBe(false) + }) +}) + +describe('personalSetupFields', () => { + it('asks for required config beyond the listing caps, never a selector', () => { + const drive = SEARCH_CONNECTORS.find((connector) => connector.type === 'google_drive')! + const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + expect(personalSetupFields(drive.meta)).toEqual([]) + expect(personalSetupFields(jira.meta).map((field) => field.id)).toEqual(['domain']) + }) + + it('reports the setup fields a config leaves empty', () => { + const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + expect(missingSetupFields(jira.meta, {}).map((field) => field.id)).toEqual(['domain']) + expect(missingSetupFields(jira.meta, { domain: ' ' })).toHaveLength(1) + expect(missingSetupFields(jira.meta, { domain: 'acme.atlassian.net' })).toEqual([]) }) }) diff --git a/apps/sim/lib/sim-search/connectors.ts b/apps/sim/lib/sim-search/connectors.ts index 58827ad5d4b..716a9ee1c89 100644 --- a/apps/sim/lib/sim-search/connectors.ts +++ b/apps/sim/lib/sim-search/connectors.ts @@ -6,7 +6,7 @@ import { getServiceConfigByServiceId, } from '@/lib/oauth' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import type { ConnectorMeta } from '@/connectors/types' +import type { ConnectorConfigField, ConnectorMeta } from '@/connectors/types' /** The workspace knowledge base Sim Search indexes into, one per workspace, created on first connect. */ export const SIM_SEARCH_KNOWLEDGE_BASE_NAME = 'Sim Search' @@ -75,15 +75,37 @@ export const SEARCH_CONNECTORS: readonly SearchConnector[] = Object.entries(CONN .sort((a, b) => a.meta.name.localeCompare(b.meta.name)) /** - * Whether a source connects with one click on Sim Search: it crawls per - * member, and nothing in its config is required beyond the listing caps - * members mode clears. A source that needs a site or space (Confluence, - * Jira) is set up from a knowledge base, where an admin can name it. + * Whether a source connects per person on Sim Search: it authenticates with + * OAuth and its listing reflects who may read each document, so each member's + * own crawl is the permission check. A source that fails this is a workspace + * connector an admin sets up from a knowledge base. */ export function canConnectPersonally(meta: ConnectorMeta): boolean { - if (meta.auth.mode !== 'oauth' || !meta.permissionScopedListing) return false - const capFieldIds = new Set(meta.permissionScopedListing.capFieldIds) - return meta.configFields.every((field) => !field.required || capFieldIds.has(field.id)) + return meta.auth.mode === 'oauth' && meta.permissionScopedListing !== undefined +} + +/** + * The fields a person fills in before a source's first connect: its required + * config beyond the listing caps members mode clears. A selector needs a + * credential the source does not have yet, so a selector's typed twin stands + * in for it (Confluence's space key, Jira's project key). + */ +export function personalSetupFields(meta: ConnectorMeta): ConnectorConfigField[] { + const capFieldIds = new Set(meta.permissionScopedListing?.capFieldIds ?? []) + return meta.configFields.filter( + (field) => field.required && field.type !== 'selector' && !capFieldIds.has(field.id) + ) +} + +/** The setup fields a source config leaves empty. */ +export function missingSetupFields( + meta: ConnectorMeta, + sourceConfig: Record +): ConnectorConfigField[] { + return personalSetupFields(meta).filter((field) => { + const value = sourceConfig[field.id] + return typeof value !== 'string' || value.trim() === '' + }) } /** From 3bbcaa3878ca59926e31e239d3e326adf9201918 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 01:38:16 -0700 Subject: [PATCH 35/76] feat(search): keep the results familiar to a search page A result's second line names its source app rather than the knowledge base, emptying the search box returns to the sources, and the arrow keys walk the result links. --- .../knowledge-search-results.tsx | 28 +++++++++++++++++-- .../components/source-card/source-card.tsx | 1 + .../home/components/user-input/user-input.tsx | 12 ++++++++ .../app/workspace/[workspaceId]/home/home.tsx | 4 +++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 14ad1ada9e0..5f5b608dbcc 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -50,13 +50,19 @@ export function groupResultsByDocument( return grouped } -/** A result as the source card renders it; a document without a source URL cannot be opened. */ +/** + * A result as the source card renders it: the row's second line names the + * source app, or the knowledge base for an upload. A document without a + * source URL cannot be opened. + */ function toSource(result: WorkspaceKnowledgeSearchResult): SourceTagData | null { if (!result.sourceUrl) return null return { url: result.sourceUrl, title: result.documentName ?? undefined, - siteName: result.knowledgeBaseName || undefined, + siteName: result.connectorType + ? connectorName(result.connectorType) + : result.knowledgeBaseName || undefined, connectorType: result.connectorType ?? undefined, snippet: toSnippet(result.content), updatedAt: result.sourceModifiedAt ?? undefined, @@ -67,6 +73,22 @@ function connectorName(connectorType: string): string { return CONNECTOR_META_REGISTRY[connectorType]?.name ?? connectorType } +/** + * Arrow keys walk the result links, the way a search page does; Enter on a + * focused link opens it natively. Focus stops at either end. + */ +function handleResultsKeyDown(event: React.KeyboardEvent) { + if (event.key !== 'ArrowDown' && event.key !== 'ArrowUp') return + const links = [...event.currentTarget.querySelectorAll('a[data-source-link]')] + if (links.length === 0) return + const index = links.findIndex((link) => link === document.activeElement) + const next = + event.key === 'ArrowDown' ? Math.min(index + 1, links.length - 1) : Math.max(index - 1, 0) + if (next === index) return + event.preventDefault() + links[next].focus() +} + interface KnowledgeSearchResultsProps { workspaceId: string query: string @@ -197,7 +219,7 @@ export function KnowledgeSearchResults({ : 'No documents match these filters.'}

) : ( -
+
{visible.map((result) => { const source = toSource(result) return source ? ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index d2dff4cb4f0..1add016d1a3 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -137,6 +137,7 @@ export function SourceCard({ source, query, onSummarize }: SourceCardProps) { href={source.url} target='_blank' rel='noopener noreferrer' + data-source-link='' onClick={(event) => handleExternalLinkClick(event, source.url)} className={cn( 'truncate text-[var(--text-primary)] text-sm no-underline hover:underline', diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 14d4983b586..ba71adcec1a 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -77,6 +77,8 @@ interface UserInputProps { * transcript. Defaults to clearing. */ clearOnSubmit?: boolean + /** Called when the text becomes empty after having had content, such as a search being cleared. */ + onCleared?: () => void onEditQueuedTail?: () => void } @@ -106,6 +108,7 @@ const UserInputImpl = forwardRef(function UserI onSendQueuedHead, onEditQueuedTail, clearOnSubmit = true, + onCleared, }, ref ) { @@ -213,6 +216,15 @@ const UserInputImpl = forwardRef(function UserI } }, []) // eslint-disable-line react-hooks/exhaustive-deps -- intentional mount-only restore + const onClearedRef = useRef(onCleared) + onClearedRef.current = onCleared + const hadTextRef = useRef(false) + useEffect(() => { + const hasText = editor.value.trim().length > 0 + if (hadTextRef.current && !hasText) onClearedRef.current?.() + hadTextRef.current = hasText + }, [editor.value]) + const isFirstSaveRef = useRef(true) const draftSaveTimerRef = useRef(null) const pendingDraftRef = useRef<{ key: string; payload: DraftPayload } | null>(null) diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 3479819cb78..e9c5b35777e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -460,6 +460,9 @@ export function Home({ chatId, userName, userId }: HomeProps) { [workspaceId, chatId, prepareResourceViewForAgentTurn, sendMessage] ) + /** An emptied search box returns to the sources; nothing else reads the cleared query. */ + const clearSearch = useCallback(() => setSearchQuery(''), []) + /** Summarize on a result: hand the document to the agent in Build mode. */ const handleSummarize = useCallback( (prompt: string) => { @@ -706,6 +709,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { draftScopeKey={draftScopeKey} onSubmit={handleSubmit} clearOnSubmit={composerMode !== 'search'} + onCleared={clearSearch} isSending={isSending} onStopGeneration={handleStopGeneration} /> From 686236a0b77e1bd8c06ed5909287aaf9cca2c11f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 01:47:39 -0700 Subject: [PATCH 36/76] feat(knowledge): sync Google Chat, Meet, Sheets, Bitbucket, and Airtable per member Google Chat lists only the spaces the caller belongs to, Google Meet only the conferences they organized, and Google Sheets, Bitbucket, and Airtable one configured spreadsheet, repository, or table that a member either can read in full or cannot reach at all, so each now declares its listing caps and can sync per member. Sheets, Bitbucket, and Airtable report a 403 or 404 on their configured scope as a complete listing of nothing for that member; Bitbucket reads a zero cap as unlimited instead of refusing it. Google Chat, Meet, and Sheets join the Google managed OAuth providers, and Bitbucket gains a managed policy that takes its subject from account_id and its address from the confirmed primary email, requesting the email scope that needs. HubSpot stays a workspace connector: HubSpot documents that an OAuth token reflects the app's scopes, not the authorizing user's record permissions. Google Vault needs the Vault admin privilege and Trello authorizes through its own OAuth 1.0a flow rather than a managed OAuth 2.0 connector. --- apps/sim/connectors/airtable/airtable.ts | 19 +++- apps/sim/connectors/airtable/meta.ts | 5 + .../connectors/bitbucket/bitbucket.test.ts | 55 ++++++++++- apps/sim/connectors/bitbucket/bitbucket.ts | 30 +++++- apps/sim/connectors/bitbucket/meta.ts | 6 ++ apps/sim/connectors/google-chat/meta.ts | 6 ++ apps/sim/connectors/google-meet/meta.ts | 7 ++ .../google-sheets/google-sheets.test.ts | 35 ++++++- .../connectors/google-sheets/google-sheets.ts | 14 ++- apps/sim/connectors/google-sheets/meta.ts | 5 + .../permission-scoped-listing.test.ts | 5 + apps/sim/lib/auth/connectors/managed-oauth.ts | 92 ++++++++++++++++++- .../credential-groups/provider-registry.ts | 4 + apps/sim/lib/credential-groups/providers.ts | 24 +++++ 14 files changed, 295 insertions(+), 12 deletions(-) diff --git a/apps/sim/connectors/airtable/airtable.ts b/apps/sim/connectors/airtable/airtable.ts index cb955b91a21..fdddb724f41 100644 --- a/apps/sim/connectors/airtable/airtable.ts +++ b/apps/sim/connectors/airtable/airtable.ts @@ -3,7 +3,12 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { airtableConnectorMeta } from '@/connectors/airtable/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { computeContentHash, parseTagDate } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + computeContentHash, + isListingScopeUnavailableError, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('AirtableConnector') @@ -143,6 +148,8 @@ function readMaxRecords(sourceConfig: Record): number { export const airtableConnector: ConnectorConfig = { ...airtableConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + /** * Lists records from `GET /v0/{baseId}/{tableIdOrName}`. * @@ -217,7 +224,15 @@ export const airtableConnector: ConnectorConfig = { * Throwing aborts the sync before reconciliation, which is the safe * outcome — the next run restarts iteration from the beginning. */ - throw new Error(`Failed to list Airtable records: ${response.status}`) + const message = `Failed to list Airtable records: ${response.status}` + /** + * Airtable answers a base or table the caller cannot reach with 403 + * (INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND) or 404: the configured scope + * is out of this caller's reach. + */ + throw response.status === 403 || response.status === 404 + ? new ConnectorListingScopeUnavailableError(message, response.status) + : new Error(message) } const data = (await response.json()) as { diff --git a/apps/sim/connectors/airtable/meta.ts b/apps/sim/connectors/airtable/meta.ts index 3044ceca757..fd4dac57c2d 100644 --- a/apps/sim/connectors/airtable/meta.ts +++ b/apps/sim/connectors/airtable/meta.ts @@ -14,6 +14,11 @@ export const airtableConnectorMeta: ConnectorMeta = { requiredScopes: ['data.records:read', 'schema.bases:read'], }, + /** + * The listing is one configured table's records under the caller's own + * token, which reaches only the bases that member granted and can read. + */ + permissionScopedListing: { capFieldIds: ['maxRecords'] }, configFields: [ { id: 'baseSelector', diff --git a/apps/sim/connectors/bitbucket/bitbucket.test.ts b/apps/sim/connectors/bitbucket/bitbucket.test.ts index 8b184f8a1e8..e36b329a4f2 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.test.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.test.ts @@ -490,6 +490,21 @@ describe('bitbucket maxItems cap', () => { expect(syncContext.listingCapped).toBe(true) }) + it('reads a zero cap, which members mode writes, as unlimited', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: 0 }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBeUndefined() + }) + it('leaves the listing reconcilable when no cap is configured', async () => { mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]]) @@ -533,6 +548,42 @@ describe('bitbucket maxItems cap', () => { }) }) +describe('bitbucket listing scope', () => { + it.each([403, 404])( + 'reports a repository the token cannot reach (%i) as an unavailable listing scope', + async (status) => { + mockApi([[/\/repositories\/acme\/widgets$/, () => jsonResponse({ type: 'error' }, status)]]) + + const error = await bitbucketConnector + .listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + .catch((caught: unknown) => caught) + + expect(bitbucketConnector.isListingScopeUnavailableError?.(error)).toBe(true) + } + ) + + it('reports a pull request listing the token cannot reach as an unavailable listing scope', async () => { + mockApi([[/\/pullrequests/, () => jsonResponse({ type: 'error' }, 403)]]) + + const error = await bitbucketConnector + .listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {}) + .catch((caught: unknown) => caught) + + expect(bitbucketConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('keeps any other repository failure retryable', async () => { + mockApi([[/\/repositories\/acme\/widgets$/, () => jsonResponse({ type: 'error' }, 500)]]) + + const error = await bitbucketConnector + .listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(Error) + expect(bitbucketConnector.isListingScopeUnavailableError?.(error)).toBe(false) + }) +}) + describe('bitbucket pull request listing', () => { it('builds the documented collection query', async () => { mockApi([[/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })]]) @@ -844,13 +895,13 @@ describe('bitbucket validateConfig', () => { ).toEqual({ valid: true }) }) - it('rejects a non-positive maxItems before spending a request', async () => { + it('rejects a negative maxItems before spending a request', async () => { mockApi([]) expect( await bitbucketConnector.validateConfig(ACCESS_TOKEN, { ...CONFIG, - maxItems: '0', + maxItems: '-1', }) ).toEqual({ valid: false, error: 'Max items must be a positive integer' }) }) diff --git a/apps/sim/connectors/bitbucket/bitbucket.ts b/apps/sim/connectors/bitbucket/bitbucket.ts index 484fbdcaf47..b0d5239ee9a 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.ts @@ -5,6 +5,8 @@ import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, markSkipped, parseTagDate, readBodyWithLimit, @@ -223,12 +225,15 @@ function readSlug(value: unknown): string { return typeof value === 'string' ? value.trim() : '' } -/** Reads an optional positive, finite integer document cap. */ +/** + * Reads the optional document cap: a positive, finite integer, or 0 for + * unlimited, which is also how a members-mode sync clears the cap. + */ function readMaxItems(value: unknown): number { if (value === undefined || value === null) return 0 if (typeof value === 'string' && !value.trim()) return 0 const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed <= 0) { + if (!Number.isSafeInteger(parsed) || parsed < 0) { throw new Error('Max items must be a positive integer') } return parsed @@ -520,6 +525,18 @@ function pullRequestToDocument( * Fetches the repository record, used to resolve the canonical full name, the web * UI base URL, and the default branch — and to confirm access during validation. */ +/** + * The error a request against the configured repository throws: scope-unavailable + * when Bitbucket says the repository does not exist for this caller (404) or + * refuses them (403), a plain error for anything else. + */ +function repositoryRequestError(message: string, status: number): Error { + const described = `${message}: ${status}` + return status === 403 || status === 404 + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + async function fetchRepository( workspaceSlug: string, repoSlug: string, @@ -553,8 +570,9 @@ async function resolveRepository( const response = await fetchRepository(workspaceSlug, repoSlug, accessToken) if (!response.ok) { - throw new Error( - `Cannot access Bitbucket repository ${workspaceSlug}/${repoSlug}: ${response.status}` + throw repositoryRequestError( + `Cannot access Bitbucket repository ${workspaceSlug}/${repoSlug}`, + response.status ) } @@ -808,6 +826,8 @@ function applyMaxItemsCap( export const bitbucketConnector: ConnectorConfig = { ...bitbucketConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -1097,7 +1117,7 @@ export const bitbucketConnector: ConnectorConfig = { status: response.status, error: errorText.slice(0, 500), }) - throw new Error(`Failed to list Bitbucket pull requests: ${response.status}`) + throw repositoryRequestError('Failed to list Bitbucket pull requests', response.status) } const page = parsePagedResponse( diff --git a/apps/sim/connectors/bitbucket/meta.ts b/apps/sim/connectors/bitbucket/meta.ts index 6f599b8f314..d92c1498b91 100644 --- a/apps/sim/connectors/bitbucket/meta.ts +++ b/apps/sim/connectors/bitbucket/meta.ts @@ -34,6 +34,12 @@ export const bitbucketConnectorMeta: ConnectorMeta = { requiredScopes: ['pullrequest'], }, + /** + * The listing is one configured repository's files and pull requests: a + * member with read access to the repository lists all of them, one without + * lists nothing. + */ + permissionScopedListing: { capFieldIds: ['maxItems'] }, configFields: [ { id: 'workspaceSelector', diff --git a/apps/sim/connectors/google-chat/meta.ts b/apps/sim/connectors/google-chat/meta.ts index f20ad56cce0..6da79a86b3c 100644 --- a/apps/sim/connectors/google-chat/meta.ts +++ b/apps/sim/connectors/google-chat/meta.ts @@ -54,6 +54,12 @@ export const googleChatConnectorMeta: ConnectorMeta = { */ rehydrateOnFullSync: true, + /** + * `spaces.list` returns only the spaces the caller is a member of, so one + * member's crawl is exactly what they may read. `maxMessages` bounds each + * space document's window, not which spaces are listed, so it is not a cap. + */ + permissionScopedListing: { capFieldIds: ['maxSpaces'] }, configFields: [ { id: 'spaceTypes', diff --git a/apps/sim/connectors/google-meet/meta.ts b/apps/sim/connectors/google-meet/meta.ts index 88112e590d4..0b5f6afa5fe 100644 --- a/apps/sim/connectors/google-meet/meta.ts +++ b/apps/sim/connectors/google-meet/meta.ts @@ -14,6 +14,13 @@ export const googleMeetConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/meetings.space.readonly'], }, + /** + * `conferenceRecords.list` returns only the conferences the caller + * organized, so a member's crawl never reaches a meeting they cannot read. + * It also omits meetings they merely attended, the same shape as Zoom's + * own-recordings listing. + */ + permissionScopedListing: { capFieldIds: ['maxMeetings'] }, configFields: [ { id: 'maxMeetings', diff --git a/apps/sim/connectors/google-sheets/google-sheets.test.ts b/apps/sim/connectors/google-sheets/google-sheets.test.ts index 56c0d9b540c..8279753c025 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.test.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.test.ts @@ -89,6 +89,7 @@ interface FetchStubResponses { drive: { status: number; body: unknown } values?: unknown spreadsheet?: unknown + spreadsheetStatus?: number } /** @@ -110,7 +111,7 @@ function stubFetch(responses: FetchStubResponses) { } if (url.startsWith('https://sheets.googleapis.com/v4/spreadsheets/')) { return new Response(JSON.stringify(responses.spreadsheet ?? SPREADSHEET_METADATA), { - status: 200, + status: responses.spreadsheetStatus ?? 200, }) } throw new Error(`Unexpected fetch to ${url}`) @@ -178,6 +179,38 @@ describe('googleSheetsConnector trashed handling', () => { expect(result.documents).toHaveLength(2) }) + + it.each([403, 404])( + 'reports a spreadsheet the token cannot reach (%i) as an unavailable listing scope', + async (status) => { + stubFetch({ + drive: { status: 200, body: {} }, + spreadsheet: { error: 'denied' }, + spreadsheetStatus: status, + }) + + const error = await googleSheetsConnector + .listDocuments(ACCESS_TOKEN, SOURCE_CONFIG) + .catch((caught: unknown) => caught) + + expect(googleSheetsConnector.isListingScopeUnavailableError?.(error)).toBe(true) + } + ) + + it('keeps any other metadata failure retryable', async () => { + stubFetch({ + drive: { status: 200, body: {} }, + spreadsheet: { error: 'backend' }, + spreadsheetStatus: 500, + }) + + const error = await googleSheetsConnector + .listDocuments(ACCESS_TOKEN, SOURCE_CONFIG) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(Error) + expect(googleSheetsConnector.isListingScopeUnavailableError?.(error)).toBe(false) + }) }) describe('getDocument', () => { diff --git a/apps/sim/connectors/google-sheets/google-sheets.ts b/apps/sim/connectors/google-sheets/google-sheets.ts index 79cd8c6ad77..aba40c3d6f8 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.ts @@ -6,6 +6,8 @@ import { googleSheetsConnectorMeta } from '@/connectors/google-sheets/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, markSkipped, parseTagDate, readBodyWithLimit, @@ -172,7 +174,15 @@ async function fetchSpreadsheetMetadata( }) if (!response.ok) { - throw new Error(`Failed to fetch spreadsheet metadata: ${response.status}`) + const message = `Failed to fetch spreadsheet metadata: ${response.status}` + /** + * The Sheets API answers a spreadsheet that is not shared with the caller + * with 403, and one they cannot see at all with 404: either way the + * configured spreadsheet is out of this caller's reach. + */ + throw response.status === 403 || response.status === 404 + ? new ConnectorListingScopeUnavailableError(message, response.status) + : new Error(message) } return (await response.json()) as SpreadsheetMetadata @@ -402,6 +412,8 @@ async function sheetToDocument( export const googleSheetsConnector: ConnectorConfig = { ...googleSheetsConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/google-sheets/meta.ts b/apps/sim/connectors/google-sheets/meta.ts index 7a20991d719..2597c4f2e39 100644 --- a/apps/sim/connectors/google-sheets/meta.ts +++ b/apps/sim/connectors/google-sheets/meta.ts @@ -14,6 +14,11 @@ export const googleSheetsConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], }, + /** + * The listing is one configured spreadsheet's tabs: a member who can read + * the file lists every tab, one who cannot lists nothing. Nothing caps it. + */ + permissionScopedListing: { capFieldIds: [] }, configFields: [ { id: 'spreadsheetSelector', diff --git a/apps/sim/connectors/permission-scoped-listing.test.ts b/apps/sim/connectors/permission-scoped-listing.test.ts index 57546cfe067..362a324bd9a 100644 --- a/apps/sim/connectors/permission-scoped-listing.test.ts +++ b/apps/sim/connectors/permission-scoped-listing.test.ts @@ -23,7 +23,9 @@ const permissionScoped = Object.values(CONNECTOR_META_REGISTRY).filter( describe('permission-scoped connector listings', () => { it('covers the connectors that crawl per member', () => { expect(permissionScoped.map((meta) => meta.id).sort()).toEqual([ + 'airtable', 'asana', + 'bitbucket', 'box', 'clickup', 'confluence', @@ -31,9 +33,12 @@ describe('permission-scoped connector listings', () => { 'dropbox', 'gmail', 'google_calendar', + 'google_chat', 'google_docs', 'google_drive', 'google_forms', + 'google_meet', + 'google_sheets', 'google_slides', 'jira', 'jsm', diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index 8b2f28aba33..3d3f08b72ce 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -559,6 +559,92 @@ function createAttioManagedOAuthConnector(): ManagedOAuthConnectorConfig { } } +const BITBUCKET_API_BASE = 'https://api.bitbucket.org/2.0' +const BITBUCKET_EMAIL_SCOPE = 'email' + +/** + * Bitbucket's current-user endpoint carries no address, and its emails endpoint needs the + * `email` scope the consumer would not otherwise request; so this policy adds that scope and + * reads the two resources in turn. The subject is the immutable `account_id`; there is no + * tenant because one Bitbucket account belongs to any number of workspaces. + */ +function createBitbucketManagedOAuthConnector(): ManagedOAuthConnectorConfig { + return { + additionalScopes: [BITBUCKET_EMAIL_SCOPE], + requiresRefreshToken: true, + pkce: false, + nonceVerification: 'state_only', + includeLoginHint: false, + getAuthorizationAppId(clientId) { + return `bitbucket:${createHash('sha256').update(clientId).digest('hex')}` + }, + async verifyIdentity({ tokens }) { + if (!tokens.accessToken) { + throw new Error('Bitbucket returned an incomplete authorization') + } + const headers = { + Accept: 'application/json', + Authorization: `Bearer ${tokens.accessToken}`, + } + const userResponse = await fetch(`${BITBUCKET_API_BASE}/user`, { + headers, + signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), + }) + const userBody = await readResponseJsonWithLimit(userResponse, { + maxBytes: USER_INFO_MAX_BYTES, + label: 'Bitbucket identity response', + }) + if (!userResponse.ok) { + throw new Error(`Bitbucket identity request failed with HTTP ${userResponse.status}`) + } + const user = asProfileRecord(userBody, 'Bitbucket') + const accountId = requireIdentityField(user.account_id, 'Bitbucket account id') + const emailsResponse = await fetch(`${BITBUCKET_API_BASE}/user/emails`, { + headers, + signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), + }) + const emailsBody = await readResponseJsonWithLimit(emailsResponse, { + maxBytes: USER_INFO_MAX_BYTES, + label: 'Bitbucket emails response', + }) + if (!emailsResponse.ok) { + throw new Error(`Bitbucket emails request failed with HTTP ${emailsResponse.status}`) + } + const emails = asProfileRecord(emailsBody, 'Bitbucket').values + const primary = Array.isArray(emails) + ? emails.find( + (entry): entry is Record => + isRecordLike(entry) && entry.is_primary === true && entry.is_confirmed === true + ) + : undefined + const avatar = + isRecordLike(user.links) && isRecordLike(user.links.avatar) + ? user.links.avatar.href + : undefined + const base = withOptionalIdentityFields( + { + providerSubjectId: accountId, + email: requireIdentityField(primary?.email, 'Bitbucket confirmed primary email'), + /** Only a confirmed primary address is accepted above. */ + emailVerified: true, + }, + { displayName: user.display_name, avatarUrl: avatar } + ) + return { + ...base, + providerTenantId: null, + /** Bitbucket reports the consumer's granted scopes on the token response. */ + grantedScopes: [...new Set(tokens.scopes ?? [])], + } + }, + hasRequiredScopes(grantedScopes, requiredScopes) { + const granted = new Set(grantedScopes) + return requiredScopes.every((scope) => granted.has(scope)) + }, + isTerminalRefreshError, + } +} + /** * Managed enrollment policies for the providers whose identity endpoint reports an email the * provider itself vouches for. Keyed by connector provider id. @@ -719,6 +805,7 @@ const USER_INFO_MANAGED_OAUTH_CONNECTORS = new Map ManagedOAuthCon }), ], ['attio', createAttioManagedOAuthConnector], + ['bitbucket', createBitbucketManagedOAuthConnector], [ 'hubspot', () => @@ -1036,7 +1123,10 @@ function resolveManagedOAuthPolicy( providerId === 'google-calendar' || providerId === 'google-drive' || providerId === 'google-docs' || - providerId === 'google-forms' + providerId === 'google-forms' || + providerId === 'google-chat' || + providerId === 'google-meet' || + providerId === 'google-sheets' ) { return () => createGoogleManagedOAuthConnector(providerId) } diff --git a/apps/sim/lib/credential-groups/provider-registry.ts b/apps/sim/lib/credential-groups/provider-registry.ts index 9fc55743e94..304e30b5cd7 100644 --- a/apps/sim/lib/credential-groups/provider-registry.ts +++ b/apps/sim/lib/credential-groups/provider-registry.ts @@ -15,11 +15,15 @@ const CREDENTIAL_GROUP_PROVIDER_ADAPTERS: Record< 'google-drive': createStandardOAuthCredentialGroupProviderAdapter('google-drive'), 'google-docs': createStandardOAuthCredentialGroupProviderAdapter('google-docs'), 'google-forms': createStandardOAuthCredentialGroupProviderAdapter('google-forms'), + 'google-chat': createStandardOAuthCredentialGroupProviderAdapter('google-chat'), + 'google-meet': createStandardOAuthCredentialGroupProviderAdapter('google-meet'), + 'google-sheets': createStandardOAuthCredentialGroupProviderAdapter('google-sheets'), confluence: createStandardOAuthCredentialGroupProviderAdapter('confluence'), jira: createStandardOAuthCredentialGroupProviderAdapter('jira'), airtable: createStandardOAuthCredentialGroupProviderAdapter('airtable'), asana: createStandardOAuthCredentialGroupProviderAdapter('asana'), attio: createStandardOAuthCredentialGroupProviderAdapter('attio'), + bitbucket: createStandardOAuthCredentialGroupProviderAdapter('bitbucket'), box: createStandardOAuthCredentialGroupProviderAdapter('box'), calcom: createStandardOAuthCredentialGroupProviderAdapter('calcom'), clickup: createStandardOAuthCredentialGroupProviderAdapter('clickup'), diff --git a/apps/sim/lib/credential-groups/providers.ts b/apps/sim/lib/credential-groups/providers.ts index e56f4d8dbd8..f3fb9f99c65 100644 --- a/apps/sim/lib/credential-groups/providers.ts +++ b/apps/sim/lib/credential-groups/providers.ts @@ -7,11 +7,15 @@ export const CREDENTIAL_GROUP_STANDARD_OAUTH_PROVIDER_IDS = [ 'google-drive', 'google-docs', 'google-forms', + 'google-chat', + 'google-meet', + 'google-sheets', 'confluence', 'jira', 'airtable', 'asana', 'attio', + 'bitbucket', 'box', 'calcom', 'clickup', @@ -73,6 +77,21 @@ const CREDENTIAL_GROUP_PROVIDER_SUPPORT: Record< description: 'Let each person connect one Google Forms account', configuration: 'oauth', }, + 'google-chat': { + serviceId: 'google-chat', + description: 'Let each person connect one Google Chat account', + configuration: 'oauth', + }, + 'google-meet': { + serviceId: 'google-meet', + description: 'Let each person connect one Google Meet account', + configuration: 'oauth', + }, + 'google-sheets': { + serviceId: 'google-sheets', + description: 'Let each person connect one Google Sheets account', + configuration: 'oauth', + }, confluence: { serviceId: 'confluence', description: 'Let each person connect one Confluence account', @@ -98,6 +117,11 @@ const CREDENTIAL_GROUP_PROVIDER_SUPPORT: Record< description: 'Let each person connect one Attio account', configuration: 'oauth', }, + bitbucket: { + serviceId: 'bitbucket', + description: 'Let each person connect one Bitbucket account', + configuration: 'oauth', + }, box: { serviceId: 'box', description: 'Let each person connect one Box account', From 1fc7efc98028a3421ec2a142c7d95e19c1ad8902 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 01:54:00 -0700 Subject: [PATCH 37/76] feat(knowledge): sync Microsoft connectors per member Microsoft Teams, Outlook, OneDrive, SharePoint, and Microsoft Excel list only what the caller's own account can read, so each now declares its listing caps and can sync per member. Outlook reads a 0 conversation cap as unlimited instead of the default. A team, channel, folder, site, or workbook Graph answers 403 or 404 for reads as a complete listing of nothing for that member, as does a channel or folder the member's own listing does not show. A Microsoft managed OAuth policy lets a Credential Group collect each person's account: the id_token is verified against the identity platform's keys and bound to the access token through the OIDC userinfo subject, the person is recorded by oid and tid, and the email counts as proven only through the claims Entra vouches for. --- apps/sim/connectors/microsoft-excel/meta.ts | 2 + .../microsoft-excel/microsoft-excel.ts | 21 +- apps/sim/connectors/microsoft-teams/meta.ts | 5 + .../microsoft-teams/microsoft-teams.test.ts | 82 +++++++ .../microsoft-teams/microsoft-teams.ts | 13 +- apps/sim/connectors/onedrive/meta.ts | 1 + apps/sim/connectors/onedrive/onedrive.test.ts | 27 +++ apps/sim/connectors/onedrive/onedrive.ts | 19 +- apps/sim/connectors/outlook/meta.ts | 1 + apps/sim/connectors/outlook/outlook.test.ts | 88 +++++++ apps/sim/connectors/outlook/outlook.ts | 49 ++-- .../permission-scoped-listing.test.ts | 5 + apps/sim/connectors/sharepoint/meta.ts | 1 + .../connectors/sharepoint/sharepoint.test.ts | 41 ++++ apps/sim/connectors/sharepoint/sharepoint.ts | 46 +++- .../lib/auth/connectors/managed-oauth.test.ts | 225 +++++++++++++++++- apps/sim/lib/auth/connectors/managed-oauth.ts | 172 +++++++++++++ .../credential-groups/provider-registry.ts | 5 + apps/sim/lib/credential-groups/providers.ts | 30 +++ 19 files changed, 799 insertions(+), 34 deletions(-) create mode 100644 apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts diff --git a/apps/sim/connectors/microsoft-excel/meta.ts b/apps/sim/connectors/microsoft-excel/meta.ts index 05ac3517d45..142a0786071 100644 --- a/apps/sim/connectors/microsoft-excel/meta.ts +++ b/apps/sim/connectors/microsoft-excel/meta.ts @@ -14,6 +14,8 @@ export const microsoftExcelConnectorMeta: ConnectorMeta = { requiredScopes: ['Files.ReadWrite'], }, + /** Every worksheet of the one workbook is listed; nothing caps the listing. */ + permissionScopedListing: { capFieldIds: [] }, configFields: [ { id: 'driveId', diff --git a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts index aa28a9e2696..e9085e137a2 100644 --- a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts +++ b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts @@ -4,7 +4,13 @@ import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { microsoftExcelConnectorMeta } from '@/connectors/microsoft-excel/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { markSkipped, parseTagDate, readBodyWithLimit } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, + markSkipped, + parseTagDate, + readBodyWithLimit, +} from '@/connectors/utils' import type { ExcelCellValue } from '@/tools/microsoft_excel/types' import { escapeODataString, @@ -221,11 +227,18 @@ function worksheetUrl(basePath: string, sheetName: string): string { return `${basePath}/workbook/worksheets('${encodeURIComponent(escapeODataString(sheetName))}')` } -/** Throws a Graph-formatted error for a failed response. */ +/** + * Throws a Graph-formatted error for a failed response. A workbook Graph will + * not open for the caller (403 `accessDenied`, 404 `itemNotFound`) is a scope + * they cannot reach rather than a fault to retry. + */ async function graphError(response: Response, context: string): Promise { const body = await response.text().catch(() => '') const detail = parseGraphErrorMessage(response.status, response.statusText, body) - throw new Error(`${context}: ${detail}`) + const message = `${context}: ${detail}` + throw response.status === 403 || response.status === 404 + ? new ConnectorListingScopeUnavailableError(message, response.status) + : new Error(message) } /** Fetches the workbook drive item (name, webUrl, lastModifiedDateTime). */ @@ -490,6 +503,8 @@ function resolveBasePath(sourceConfig: Record): { export const microsoftExcelConnector: ConnectorConfig = { ...microsoftExcelConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/microsoft-teams/meta.ts b/apps/sim/connectors/microsoft-teams/meta.ts index 5cb79cea453..e0b943ddd55 100644 --- a/apps/sim/connectors/microsoft-teams/meta.ts +++ b/apps/sim/connectors/microsoft-teams/meta.ts @@ -21,6 +21,11 @@ export const microsoftTeamsConnectorMeta: ConnectorMeta = { requiredScopes: ['ChannelMessage.Read.All', 'Channel.ReadBasic.All', 'Team.ReadBasic.All'], }, + /** + * `maxMessages` bounds how much history each channel document carries, not + * which channels are listed, so a member's listing is complete under any value. + */ + permissionScopedListing: { capFieldIds: [] }, configFields: [ { id: 'teamSelector', diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts new file mode 100644 index 00000000000..5dba2c181b1 --- /dev/null +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts @@ -0,0 +1,82 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ MicrosoftTeamsIcon: () => null })) + +import { microsoftTeamsConnector } from '@/connectors/microsoft-teams/microsoft-teams' + +const GRAPH = 'https://graph.microsoft.com/v1.0' +const TEAM_ID = 'team-1' +const CHANNELS_URL = `${GRAPH}/teams/${TEAM_ID}/channels?$select=id,displayName,description` + +interface GraphRoute { + status?: number + body?: unknown +} + +/** Installs a URL-keyed fake Graph; unrouted URLs reply 404. */ +function mockGraph(routes: Record) { + mockFetchWithRetry.mockImplementation(async (url: string) => { + const route = routes[url] ?? { status: 404 } + const status = route.status ?? 200 + return { + ok: status >= 200 && status < 300, + status, + json: async () => route.body, + text: async () => JSON.stringify(route.body ?? {}), + } as unknown as Response + }) +} + +async function listingError(): Promise { + return microsoftTeamsConnector + .listDocuments('token', { teamId: TEAM_ID, channel: 'General' }) + .catch((error: unknown) => error) +} + +describe('microsoft teams listing scope', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each([403, 404])( + 'reads a %s on the configured team as a scope the caller cannot reach', + async (status) => { + mockGraph({ [CHANNELS_URL]: { status, body: {} } }) + + const error = await listingError() + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + } + ) + + it('reads a channel the caller cannot see as a scope they cannot reach', async () => { + mockGraph({ + [CHANNELS_URL]: { body: { value: [{ id: 'c1', displayName: 'Announcements' }] } }, + }) + + const error = await listingError() + + expect(error).toBeInstanceOf(Error) + expect(String(error)).toMatch(/Channel not found: General/) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('keeps any other listing failure retryable', async () => { + mockGraph({ [CHANNELS_URL]: { status: 500, body: {} } }) + + const error = await listingError() + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts index f63ec5c0a49..c458ff5460e 100644 --- a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts @@ -7,8 +7,10 @@ import { } from '@/connectors/microsoft-teams/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { + ConnectorListingScopeUnavailableError, computeContentHash, htmlToPlainText, + isListingScopeUnavailableError, parseMultiValue, parseTagDate, } from '@/connectors/utils' @@ -313,6 +315,15 @@ async function resolveChannel( export const microsoftTeamsConnector: ConnectorConfig = { ...microsoftTeamsConnectorMeta, + /** + * Graph answers 403 for a team or private channel the caller is not a member + * of and 404 for one it will not show them; a channel the caller's channel + * list does not resolve is reported the same way. + */ + isListingScopeUnavailableError: (error) => + isListingScopeUnavailableError(error) || + (error instanceof GraphApiError && (error.status === 403 || error.status === 404)), + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -341,7 +352,7 @@ export const microsoftTeamsConnector: ConnectorConfig = { for (const channelInput of channelInputs) { const channel = await resolveChannel(accessToken, teamId, channelInput) if (!channel) { - throw new Error(`Channel not found: ${channelInput}`) + throw new ConnectorListingScopeUnavailableError(`Channel not found: ${channelInput}`, 404) } const { threads, messageCount, lastActivityTs } = await fetchChannelMessages( diff --git a/apps/sim/connectors/onedrive/meta.ts b/apps/sim/connectors/onedrive/meta.ts index 4270ad6ddc1..b6dddae1c77 100644 --- a/apps/sim/connectors/onedrive/meta.ts +++ b/apps/sim/connectors/onedrive/meta.ts @@ -10,6 +10,7 @@ export const onedriveConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'onedrive', requiredScopes: ['Files.Read'] }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'folderPath', diff --git a/apps/sim/connectors/onedrive/onedrive.test.ts b/apps/sim/connectors/onedrive/onedrive.test.ts index b086b60f058..3439d7b31b1 100644 --- a/apps/sim/connectors/onedrive/onedrive.test.ts +++ b/apps/sim/connectors/onedrive/onedrive.test.ts @@ -412,3 +412,30 @@ describe('onedrive getDocument', () => { }) }) }) + +describe('onedrive listing scope', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each([403, 404])( + 'reads a %s on the configured folder as a scope the caller cannot reach', + async (status) => { + mockGraph({ [ROOT_URL]: { status, body: {} } }) + + const error = await onedriveConnector.listDocuments('token', {}).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(true) + } + ) + + it('keeps any other listing failure retryable', async () => { + mockGraph({ [ROOT_URL]: { status: 500, body: {} } }) + + const error = await onedriveConnector.listDocuments('token', {}).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index bf1c6196b3a..973a70972a9 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -12,12 +12,14 @@ import { assertMicrosoftGraphNextLink, CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, connectorFileExtension, decodeMicrosoftGraphTraversalCursor, encodeMicrosoftGraphTraversalCursor, extractConnectorText, hasIndexablePayload, isIndexableConnectorFile, + isListingScopeUnavailableError, isMicrosoftGraphDriveItem, isSkippedDocument, type MicrosoftGraphTraversalState, @@ -79,6 +81,19 @@ function parseMaxFiles(value: unknown): number { ) } +/** + * The error a failed Graph listing request throws. Graph reports a folder the + * caller cannot reach as 404 (`itemNotFound`) or 403 (`accessDenied`); either + * is a complete listing of nothing for that caller, while anything else is a + * fault the sync engines retry. + */ +function graphListingError(message: string, status: number): Error { + const described = `${message}: ${status}` + return status === 403 || status === 404 + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + interface OneDriveItem { id: string name: string @@ -218,6 +233,8 @@ function decodeCursor(cursor: string): OneDriveTraversalState { export const onedriveConnector: ConnectorConfig = { ...onedriveConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -263,7 +280,7 @@ export const onedriveConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to list OneDrive files: ${response.status}`) + throw graphListingError('Failed to list OneDrive files', response.status) } const data = parseMicrosoftGraphDriveItemList(await response.json(), 'OneDrive') diff --git a/apps/sim/connectors/outlook/meta.ts b/apps/sim/connectors/outlook/meta.ts index 379961692d5..96273c90d49 100644 --- a/apps/sim/connectors/outlook/meta.ts +++ b/apps/sim/connectors/outlook/meta.ts @@ -16,6 +16,7 @@ export const outlookConnectorMeta: ConnectorMeta = { requiredScopes: ['Mail.Read'], }, + permissionScopedListing: { capFieldIds: ['maxConversations'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/outlook/outlook.test.ts b/apps/sim/connectors/outlook/outlook.test.ts index e2258c5c433..e33c2fa1e26 100644 --- a/apps/sim/connectors/outlook/outlook.test.ts +++ b/apps/sim/connectors/outlook/outlook.test.ts @@ -470,3 +470,91 @@ describe('getDocument folder exclusion', () => { expect(folderCalls).toHaveLength(2) }) }) + +describe('listDocuments conversation cap', () => { + function inboxMessagesRoute(count: number): [RegExp, () => Response] { + return [ + /\/me\/mailFolders\/inbox\/messages\?/, + () => + jsonResponse({ + value: Array.from({ length: count }, (_, index) => + message({ id: `m${index}`, conversationId: `conv-${index}`, parentFolderId: INBOX_ID }) + ), + }), + ] + } + + it('caps the listing at maxConversations and flags it capped', async () => { + routeFetch([inboxMessagesRoute(3)]) + + const syncContext: Record = {} + const result = await outlookConnector.listDocuments( + 'token', + { folder: 'inbox', maxConversations: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBe(true) + }) + + it('lists every conversation when the cap is 0', async () => { + routeFetch([inboxMessagesRoute(3)]) + + const syncContext: Record = {} + const result = await outlookConnector.listDocuments( + 'token', + { folder: 'inbox', maxConversations: 0 }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(3) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('reads a folder Graph cannot find as a scope the caller cannot reach', async () => { + routeFetch([[/\/me\/mailFolders\/.*\/messages\?/, () => jsonResponse({}, { status: 404 })]]) + + const error = await outlookConnector + .listDocuments('token', { folder: 'missing-folder-id' }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(outlookConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('keeps any other listing failure retryable', async () => { + routeFetch([[/\/me\/mailFolders\/inbox\/messages\?/, () => jsonResponse({}, { status: 500 })]]) + + const error = await outlookConnector + .listDocuments('token', { folder: 'inbox' }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(outlookConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) + +describe('validateConfig conversation cap', () => { + it('accepts 0 as unlimited', async () => { + routeFetch([[/\/me\/mailFolders\/inbox\/messages\?/, () => jsonResponse({ value: [] })]]) + + await expect( + outlookConnector.validateConfig!('token', { folder: 'inbox', maxConversations: 0 }) + ).resolves.toEqual({ valid: true }) + }) + + it('rejects a fractional cap without calling Graph', async () => { + routeFetch([]) + + await expect( + outlookConnector.validateConfig!('token', { folder: 'inbox', maxConversations: '1.5' }) + ).resolves.toEqual({ + valid: false, + error: 'Max conversations must be a positive safe integer, or 0 for unlimited', + }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/outlook/outlook.ts b/apps/sim/connectors/outlook/outlook.ts index 25661fb0cda..344a311e364 100644 --- a/apps/sim/connectors/outlook/outlook.ts +++ b/apps/sim/connectors/outlook/outlook.ts @@ -3,7 +3,13 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_CONVERSATIONS, outlookConnectorMeta } from '@/connectors/outlook/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + isListingScopeUnavailableError, + listingRequestError, + parseOptionalUnlimitedSafeInteger, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('OutlookConnector') @@ -602,19 +608,30 @@ function formatConversation( } } +/** + * The conversation cap. A blank field keeps the default; 0 lifts the cap so a + * per-member listing is complete. + */ +function parseMaxConversations(value: unknown): number { + if (value === undefined || value === null || value === '') return DEFAULT_MAX_CONVERSATIONS + return parseOptionalUnlimitedSafeInteger( + value, + 'Max conversations must be a positive safe integer, or 0 for unlimited' + ) +} + export const outlookConnector: ConnectorConfig = { ...outlookConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, syncContext?: Record ): Promise => { - /** `validateConfig` rejects a non-positive value, so anything else here is drift. */ - const parsedMax = Number(sourceConfig.maxConversations) - const maxConversations = - Number.isFinite(parsedMax) && parsedMax > 0 ? parsedMax : DEFAULT_MAX_CONVERSATIONS + const maxConversations = parseMaxConversations(sourceConfig.maxConversations) // Initialize accumulator in syncContext if (syncContext && !syncContext._conversations) { @@ -648,7 +665,7 @@ export const outlookConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to fetch Outlook messages: ${response.status}`) + throw listingRequestError('Failed to fetch Outlook messages', response.status) } const data = await response.json() @@ -749,11 +766,12 @@ export const outlookConnector: ConnectorConfig = { }) /** - * Limit to `maxConversations`. Dropping the overflow makes the listing an - * incomplete view of the mailbox, so it is flagged as capped — otherwise - * reconciliation would hard-delete every conversation past the cap. + * Limit to `maxConversations` when one is set. Dropping the overflow makes + * the listing an incomplete view of the mailbox, so it is flagged as capped — + * otherwise reconciliation would hard-delete every conversation past the cap. */ - const limited = conversationEntries.slice(0, maxConversations) + const limited = + maxConversations > 0 ? conversationEntries.slice(0, maxConversations) : conversationEntries if (conversationEntries.length > limited.length && syncContext) { syncContext.listingCapped = true } @@ -911,13 +929,10 @@ export const outlookConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const maxConversations = sourceConfig.maxConversations as string | undefined - - if ( - maxConversations && - (Number.isNaN(Number(maxConversations)) || Number(maxConversations) <= 0) - ) { - return { valid: false, error: 'Max conversations must be a positive number' } + try { + parseMaxConversations(sourceConfig.maxConversations) + } catch (error) { + return { valid: false, error: toError(error).message } } try { diff --git a/apps/sim/connectors/permission-scoped-listing.test.ts b/apps/sim/connectors/permission-scoped-listing.test.ts index 57546cfe067..900b07faa96 100644 --- a/apps/sim/connectors/permission-scoped-listing.test.ts +++ b/apps/sim/connectors/permission-scoped-listing.test.ts @@ -38,8 +38,13 @@ describe('permission-scoped connector listings', () => { 'jira', 'jsm', 'linear', + 'microsoft_excel', + 'microsoft_teams', 'monday', + 'onedrive', + 'outlook', 'salesforce', + 'sharepoint', 'zoom', ]) }) diff --git a/apps/sim/connectors/sharepoint/meta.ts b/apps/sim/connectors/sharepoint/meta.ts index 38dbff4e259..ee9d0baee1d 100644 --- a/apps/sim/connectors/sharepoint/meta.ts +++ b/apps/sim/connectors/sharepoint/meta.ts @@ -10,6 +10,7 @@ export const sharepointConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'sharepoint', requiredScopes: ['Sites.Read.All'] }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'siteUrl', diff --git a/apps/sim/connectors/sharepoint/sharepoint.test.ts b/apps/sim/connectors/sharepoint/sharepoint.test.ts index 7871174bdf9..12b885ced96 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.test.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.test.ts @@ -801,3 +801,44 @@ describe('normalizeSegment', () => { expect(normalizeSegment('Reports')).toBe('reports') }) }) + +describe('listing scope', () => { + it.each([403, 404])( + 'reads a %s on the configured site as a scope the caller cannot reach', + async (status) => { + mockGraph({ [`${GRAPH}/sites/${SITE_URL}`]: { status, body: {} } }) + + const error = await sharepointConnector + .listDocuments('token', { siteUrl: SITE_URL }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + } + ) + + it('reads a folder Graph will not show the caller as a scope they cannot reach', async () => { + mockGraph({ + ...defaultDriveRoute, + ...sitesDrivesRoute, + ...rootChildren(DEFAULT_DRIVE_ID, [folder('a', 'Archive')]), + }) + + const error = await resolve('Reports').catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(String(error)).toMatch(/Folder not found: "Reports"/) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('keeps any other failure retryable', async () => { + mockGraph({ [`${GRAPH}/sites/${SITE_URL}`]: { status: 500, body: {} } }) + + const error = await sharepointConnector + .listDocuments('token', { siteUrl: SITE_URL }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index 043fe0d5cf9..2accd1caf4a 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -13,12 +13,14 @@ import { assertMicrosoftGraphNextLink, CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, connectorFileExtension, decodeMicrosoftGraphTraversalCursor, encodeMicrosoftGraphTraversalCursor, extractConnectorText, hasIndexablePayload, isIndexableConnectorFile, + isListingScopeUnavailableError, isMicrosoftGraphDriveItem, isSkippedDocument, type MicrosoftGraphTraversalState, @@ -74,6 +76,19 @@ function parseMaxFiles(value: unknown): number { ) } +/** + * The error a failed Graph request for the configured site, library, or folder + * throws. Graph reports a scope the caller cannot reach as 404 (`itemNotFound`) + * or 403 (`accessDenied`); either is a complete listing of nothing for that + * caller, while anything else is a fault the sync engines retry. + */ +function graphListingError(message: string, status: number, detail?: string): Error { + const described = detail ? `${message}: ${status} – ${detail}` : `${message}: ${status}` + return status === 403 || status === 404 + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + /** Microsoft Graph drive item shape (subset of fields we use). */ interface DriveItem { id: string @@ -216,8 +231,10 @@ async function resolveSiteId( if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error( - `Failed to resolve SharePoint site "${siteUrl}": ${response.status} – ${errorText}` + throw graphListingError( + `Failed to resolve SharePoint site "${siteUrl}"`, + response.status, + errorText ) } @@ -323,7 +340,7 @@ async function listFolderItems( if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error(`Failed to list folder items: ${response.status} – ${errorText}`) + throw graphListingError('Failed to list folder items', response.status, errorText) } const data = parseMicrosoftGraphDriveItemList(await response.json(), 'SharePoint') @@ -403,7 +420,7 @@ async function getItemByPath( if (response.status === 404) return null if (!response.ok) { - throw new Error(`Failed to resolve folder path: ${response.status}`) + throw graphListingError('Failed to resolve folder path', response.status) } return (await response.json()) as DriveItem @@ -427,7 +444,7 @@ async function listChildFolders( const response = await graphGet(url, accessToken, retryOptions) if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error(`Failed to list folder contents: ${response.status} – ${errorText}`) + throw graphListingError('Failed to list folder contents', response.status, errorText) } const rawData: unknown = await response.json() @@ -564,8 +581,9 @@ export async function resolveFolderTarget( retryOptions ) if (!defaultDriveResponse.ok) { - throw new Error( - `Failed to open the default document library for site "${siteUrl}": ${defaultDriveResponse.status}` + throw graphListingError( + `Failed to open the default document library for site "${siteUrl}"`, + defaultDriveResponse.status ) } const defaultDrive = (await defaultDriveResponse.json()) as Drive @@ -648,7 +666,8 @@ export async function resolveFolderTarget( ? { id: libraryMatch.id, name: libraryMatch.name || segments[0] } : { id: defaultDrive.id, name: defaultDriveName } - throw new Error( + /** A folder Graph will not show this caller is, for them, a scope of nothing. */ + throw new ConnectorListingScopeUnavailableError( await buildFolderNotFoundMessage( accessToken, reportDrive, @@ -658,7 +677,8 @@ export async function resolveFolderTarget( drives, reportDrive.id === defaultDrive.id, retryOptions - ) + ), + 404 ) } @@ -675,8 +695,10 @@ async function listSiteDrives( const response = await graphGet(url, accessToken, retryOptions) if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error( - `Failed to list SharePoint document libraries: ${response.status} – ${errorText}` + throw graphListingError( + 'Failed to list SharePoint document libraries', + response.status, + errorText ) } const data = parseDriveListResponse(await response.json()) @@ -784,6 +806,8 @@ function decodeCursor(cursor: string): PaginationState { export const sharepointConnector: ConnectorConfig = { ...sharepointConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/lib/auth/connectors/managed-oauth.test.ts b/apps/sim/lib/auth/connectors/managed-oauth.test.ts index 6987f1498cb..c8053e6cd29 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.test.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.test.ts @@ -1,7 +1,8 @@ /** * @vitest-environment node */ -import { afterEach, describe, expect, it, vi } from 'vitest' +import { exportJWK, generateKeyPair, SignJWT } from 'jose' +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest' import { createAtlassianManagedOAuthConnector, getManagedOAuthConnectorPolicy, @@ -395,3 +396,225 @@ describe('userinfo-backed managed OAuth connectors', () => { expect(identity.grantedScopes).toEqual(['data.records:read']) }) }) + +describe('Microsoft managed OAuth connector', () => { + const CLIENT_ID = 'client-1' + const TENANT_ID = 'tenant-1' + const MICROSOFT_PROVIDER_IDS = [ + 'microsoft-teams', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-excel', + ] + let privateKey: CryptoKey + let jwks: { keys: unknown[] } + + beforeAll(async () => { + const pair = await generateKeyPair('RS256', { extractable: true }) + privateKey = pair.privateKey + jwks = { + keys: [{ ...(await exportJWK(pair.publicKey)), kid: 'kid-1', use: 'sig', alg: 'RS256' }], + } + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + } + + async function signIdToken( + claims: Record, + audience: string = CLIENT_ID + ): Promise { + return new SignJWT({ + oid: 'oid-1', + tid: TENANT_ID, + sub: 'pairwise-1', + email: 'person@example.com', + name: 'Person', + nonce: 'nonce-1', + ...claims, + }) + .setProtectedHeader({ alg: 'RS256', kid: 'kid-1' }) + .setIssuer(`https://login.microsoftonline.com/${TENANT_ID}/v2.0`) + .setAudience(audience) + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey) + } + + function stubMicrosoft(userInfoSubject = 'pairwise-1'): ReturnType { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = String(input) + if (url.startsWith('https://login.microsoftonline.com/common/discovery/v2.0/keys')) { + return json(jwks) + } + if (url === 'https://graph.microsoft.com/oidc/userinfo') return json({ sub: userInfoSubject }) + throw new Error(`Unexpected fetch: ${url}`) + }) + vi.stubGlobal('fetch', fetchMock) + return fetchMock + } + + function policyFor(providerId: string) { + const policy = getManagedOAuthConnectorPolicy(providerId) + if (!policy) throw new Error(`No managed OAuth policy registered for ${providerId}`) + return policy + } + + it('governs every Microsoft provider through one app registration', () => { + const appIds = new Set( + MICROSOFT_PROVIDER_IDS.map((providerId) => { + const policy = policyFor(providerId) + expect(policy).toMatchObject({ + requiresRefreshToken: true, + pkce: true, + nonceVerification: 'id_token', + includeLoginHint: true, + prompt: 'select_account', + }) + return policy.getAuthorizationAppId(CLIENT_ID) + }) + ) + expect(appIds.size).toBe(1) + expect([...appIds][0]).toMatch(/^microsoft:[0-9a-f]{64}$/) + expect(getManagedOAuthConnectorPolicy('microsoft-word')).toBeUndefined() + }) + + it('verifies the id token, binds the access token to it, and reports what Entra proves', async () => { + const fetchMock = stubMicrosoft() + + const identity = await policyFor('onedrive').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + refreshToken: 'refresh-1', + idToken: await signIdToken({}), + scopes: ['Files.Read', 'User.Read'], + }, + clientId: CLIENT_ID, + }) + + expect(identity).toMatchObject({ + providerSubjectId: 'oid-1', + providerTenantId: TENANT_ID, + email: 'person@example.com', + emailVerified: false, + displayName: 'Person', + nonce: 'nonce-1', + }) + expect([...identity.grantedScopes].sort()).toEqual( + ['Files.Read', 'User.Read', 'email', 'offline_access', 'openid', 'profile'].sort() + ) + expect(fetchMock).toHaveBeenCalledWith( + 'https://graph.microsoft.com/oidc/userinfo', + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer access-1' }), + }) + ) + }) + + it.each([ + ['xms_edov', { xms_edov: true }], + ['email_verified', { email_verified: true }], + ])('counts the email verified when Entra asserts it through %s', async (_claim, claims) => { + stubMicrosoft() + + const identity = await policyFor('outlook').verifyIdentity({ + tokens: { tokenType: 'Bearer', accessToken: 'access-1', idToken: await signIdToken(claims) }, + clientId: CLIENT_ID, + }) + + expect(identity.emailVerified).toBe(true) + }) + + it('does not count offline access as granted without a refresh token', async () => { + stubMicrosoft() + + const identity = await policyFor('sharepoint').verifyIdentity({ + tokens: { tokenType: 'Bearer', accessToken: 'access-1', idToken: await signIdToken({}) }, + clientId: CLIENT_ID, + }) + + expect(identity.grantedScopes).not.toContain('offline_access') + }) + + it('rejects an access token that resolves to another subject', async () => { + stubMicrosoft('pairwise-2') + + await expect( + policyFor('microsoft-teams').verifyIdentity({ + tokens: { tokenType: 'Bearer', accessToken: 'access-1', idToken: await signIdToken({}) }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow('Microsoft returned an access token for another identity') + }) + + it('rejects an id token issued for another client', async () => { + stubMicrosoft() + + await expect( + policyFor('microsoft-excel').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + idToken: await signIdToken({}, 'client-2'), + }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow() + }) + + it.each([ + ['object id', { oid: undefined }], + ['tenant id', { tid: undefined }], + ['issuer of its own tenant', { tid: 'tenant-2' }], + ])('rejects an id token without the %s', async (_label, claims) => { + stubMicrosoft() + + await expect( + policyFor('onedrive').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + idToken: await signIdToken(claims), + }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow('Microsoft returned an invalid identity token') + }) + + it('rejects an id token that names no email to bind the invitation to', async () => { + stubMicrosoft() + + await expect( + policyFor('onedrive').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + idToken: await signIdToken({ email: undefined }), + }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow('Microsoft returned an identity token without an email') + }) + + it('compares scopes by name regardless of case or resource prefix', () => { + const policy = policyFor('onedrive') + + expect( + policy.hasRequiredScopes( + ['https://graph.microsoft.com/Files.Read', 'MAIL.READ', 'offline_access'], + ['files.read', 'Mail.Read'] + ) + ).toBe(true) + expect(policy.hasRequiredScopes(['Files.Read'], ['Files.ReadWrite'])).toBe(false) + }) +}) diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index 8b2f28aba33..79083fe5278 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -3,9 +3,11 @@ import { isRecordLike } from '@sim/utils/object' import type { OAuth2Tokens } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { OAuth2Client, type TokenPayload } from 'google-auth-library' +import { createRemoteJWKSet, type JWTPayload, jwtVerify } from 'jose' import { buildConnectorProviders } from '@/lib/auth/connectors/providers' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' +import { deriveMicrosoftEmailVerified, mapMicrosoftProfileToUser } from '@/lib/oauth/microsoft' import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' import { isTerminalRefreshError } from '@/lib/oauth/terminal-errors' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' @@ -21,6 +23,19 @@ const GMAIL_LABELS_SCOPE = 'https://www.googleapis.com/auth/gmail.labels' const ATLASSIAN_USER_INFO_URL = 'https://api.atlassian.com/me' const ATLASSIAN_USER_INFO_MAX_BYTES = 256 * 1024 const ATLASSIAN_USER_INFO_TIMEOUT_MS = 10_000 +const MICROSOFT_JWKS_URL = 'https://login.microsoftonline.com/common/discovery/v2.0/keys' +const MICROSOFT_OIDC_USER_INFO_URL = 'https://graph.microsoft.com/oidc/userinfo' +const MICROSOFT_OIDC_USER_INFO_MAX_BYTES = 256 * 1024 +const MICROSOFT_OIDC_USER_INFO_TIMEOUT_MS = 10_000 +const MICROSOFT_GRAPH_SCOPE_PREFIX = 'https://graph.microsoft.com/' +/** The Microsoft providers whose accounts a Credential Group can collect per person. */ +const MICROSOFT_MANAGED_OAUTH_PROVIDER_IDS = new Set([ + 'microsoft-teams', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-excel', +]) type AtlassianManagedOAuthProviderId = 'confluence' | 'jira' @@ -227,6 +242,160 @@ export function createAtlassianManagedOAuthConnector( } } +let microsoftJwks: ReturnType | undefined + +/** The signing keys of the multi-tenant Microsoft identity platform, cached across verifications. */ +function getMicrosoftJwks(): ReturnType { + microsoftJwks ??= createRemoteJWKSet(new URL(MICROSOFT_JWKS_URL)) + return microsoftJwks +} + +/** + * Graph delegated scopes compare by name regardless of case, and a token response may spell + * one as its resource-qualified form. + */ +function canonicalMicrosoftScope(scope: string): string { + const unqualified = scope.startsWith(MICROSOFT_GRAPH_SCOPE_PREFIX) + ? scope.slice(MICROSOFT_GRAPH_SCOPE_PREFIX.length) + : scope + return unqualified.toLowerCase() +} + +interface MicrosoftIdentityClaims { + oid: string + tid: string + sub: string + email: string + name?: string + nonce?: string + claims: Record +} + +/** + * Reads the claims a verified Microsoft id_token must carry to bind an enrollment. `oid` and + * `tid` identify the person and their tenant stably across every Microsoft app; `sub` is the + * app-pairwise subject the OIDC userinfo endpoint echoes back. The issuer is checked against the + * token's own tenant because the multi-tenant `/common` authority signs for every tenant. + */ +function requireMicrosoftIdentityClaims(payload: JWTPayload): MicrosoftIdentityClaims { + const claims: Record = { ...payload } + const { oid, tid, sub, iss, name, nonce } = claims + if ( + typeof oid !== 'string' || + !oid || + typeof tid !== 'string' || + !tid || + typeof sub !== 'string' || + !sub || + iss !== `https://login.microsoftonline.com/${tid}/v2.0` + ) { + throw new Error('Microsoft returned an invalid identity token') + } + const email = [claims.email, claims.preferred_username, claims.upn].find( + (value): value is string => typeof value === 'string' && value.trim().length > 0 + ) + if (!email) { + throw new Error('Microsoft returned an identity token without an email') + } + return { + oid, + tid, + sub, + email, + ...(typeof name === 'string' && name.trim() ? { name } : {}), + ...(typeof nonce === 'string' && nonce ? { nonce } : {}), + claims, + } +} + +/** + * The subject the access token resolves to at Microsoft's OIDC userinfo endpoint. It needs only + * the `openid` grant, so unlike Graph `/me` it does not fail for a tenant whose administrator has + * not consented to Graph. + */ +async function fetchMicrosoftAccessTokenSubject(accessToken: string): Promise { + const response = await fetch(MICROSOFT_OIDC_USER_INFO_URL, { + headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, + signal: AbortSignal.timeout(MICROSOFT_OIDC_USER_INFO_TIMEOUT_MS), + }) + const profile = await readResponseJsonWithLimit(response, { + maxBytes: MICROSOFT_OIDC_USER_INFO_MAX_BYTES, + label: 'Microsoft user identity response', + }) + if (!response.ok) { + throw new Error(`Microsoft user identity request failed with HTTP ${response.status}`) + } + if (!isRecordLike(profile) || typeof profile.sub !== 'string' || !profile.sub) { + throw new Error('Microsoft returned an invalid user identity') + } + return profile.sub +} + +/** + * Managed enrollment policy for the providers that share Sim's Microsoft app registration. + * + * Identity comes from the id_token, verified against the identity platform's published keys and + * bound to the access token through the OIDC userinfo subject, the way the Google policy binds + * through tokeninfo. Microsoft never asserts `email_verified` for a work account, so the email + * counts as proven only through the claims Entra does vouch for: the verified-email claims, or + * `xms_edov` asserting the domain belongs to the account's own tenant. + */ +export function createMicrosoftManagedOAuthConnector( + providerId: string +): ManagedOAuthConnectorConfig { + return { + additionalScopes: [], + requiresRefreshToken: true, + pkce: true, + nonceVerification: 'id_token', + includeLoginHint: true, + prompt: 'select_account', + getAuthorizationAppId(clientId) { + return `microsoft:${createHash('sha256').update(clientId).digest('hex')}` + }, + async verifyIdentity({ tokens, clientId }) { + if (!tokens.idToken || !tokens.accessToken) { + throw new Error(`Microsoft ${providerId} returned an incomplete authorization`) + } + const { payload } = await jwtVerify(tokens.idToken, getMicrosoftJwks(), { + audience: clientId, + }) + const identity = requireMicrosoftIdentityClaims(payload) + const accessTokenSubject = await fetchMicrosoftAccessTokenSubject(tokens.accessToken) + if (accessTokenSubject !== identity.sub) { + throw new Error('Microsoft returned an access token for another identity') + } + /** + * The token response's `scope` is not guaranteed to echo the OIDC scopes or + * `offline_access`, so each is counted only when the response itself proves the grant: an + * id_token for `openid`, its `name` and `email` claims for `profile` and `email`, and a + * refresh token for `offline_access`. + */ + const grantedScopes = new Set(tokens.scopes ?? []) + grantedScopes.add('openid') + if (identity.name) grantedScopes.add('profile') + if (typeof identity.claims.email === 'string') grantedScopes.add('email') + if (tokens.refreshToken) grantedScopes.add('offline_access') + return { + providerSubjectId: identity.oid, + providerTenantId: identity.tid, + email: identity.email, + emailVerified: + deriveMicrosoftEmailVerified(identity.claims, identity.email) || + mapMicrosoftProfileToUser(identity.claims).emailVerified === true, + ...(identity.name ? { displayName: identity.name } : {}), + ...(identity.nonce ? { nonce: identity.nonce } : {}), + grantedScopes: [...grantedScopes], + } + }, + hasRequiredScopes(grantedScopes, requiredScopes) { + const granted = new Set(grantedScopes.map(canonicalMicrosoftScope)) + return requiredScopes.every((scope) => granted.has(canonicalMicrosoftScope(scope))) + }, + isTerminalRefreshError, + } +} + const USER_INFO_TIMEOUT_MS = 10_000 const USER_INFO_MAX_BYTES = 256 * 1024 @@ -1043,6 +1212,9 @@ function resolveManagedOAuthPolicy( if (providerId === 'confluence' || providerId === 'jira') { return () => createAtlassianManagedOAuthConnector(providerId) } + if (MICROSOFT_MANAGED_OAUTH_PROVIDER_IDS.has(providerId)) { + return () => createMicrosoftManagedOAuthConnector(providerId) + } return USER_INFO_MANAGED_OAUTH_CONNECTORS.get(providerId) } diff --git a/apps/sim/lib/credential-groups/provider-registry.ts b/apps/sim/lib/credential-groups/provider-registry.ts index 9fc55743e94..2632228febf 100644 --- a/apps/sim/lib/credential-groups/provider-registry.ts +++ b/apps/sim/lib/credential-groups/provider-registry.ts @@ -15,6 +15,11 @@ const CREDENTIAL_GROUP_PROVIDER_ADAPTERS: Record< 'google-drive': createStandardOAuthCredentialGroupProviderAdapter('google-drive'), 'google-docs': createStandardOAuthCredentialGroupProviderAdapter('google-docs'), 'google-forms': createStandardOAuthCredentialGroupProviderAdapter('google-forms'), + 'microsoft-teams': createStandardOAuthCredentialGroupProviderAdapter('microsoft-teams'), + outlook: createStandardOAuthCredentialGroupProviderAdapter('outlook'), + onedrive: createStandardOAuthCredentialGroupProviderAdapter('onedrive'), + sharepoint: createStandardOAuthCredentialGroupProviderAdapter('sharepoint'), + 'microsoft-excel': createStandardOAuthCredentialGroupProviderAdapter('microsoft-excel'), confluence: createStandardOAuthCredentialGroupProviderAdapter('confluence'), jira: createStandardOAuthCredentialGroupProviderAdapter('jira'), airtable: createStandardOAuthCredentialGroupProviderAdapter('airtable'), diff --git a/apps/sim/lib/credential-groups/providers.ts b/apps/sim/lib/credential-groups/providers.ts index e56f4d8dbd8..1b799497f3c 100644 --- a/apps/sim/lib/credential-groups/providers.ts +++ b/apps/sim/lib/credential-groups/providers.ts @@ -7,6 +7,11 @@ export const CREDENTIAL_GROUP_STANDARD_OAUTH_PROVIDER_IDS = [ 'google-drive', 'google-docs', 'google-forms', + 'microsoft-teams', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-excel', 'confluence', 'jira', 'airtable', @@ -73,6 +78,31 @@ const CREDENTIAL_GROUP_PROVIDER_SUPPORT: Record< description: 'Let each person connect one Google Forms account', configuration: 'oauth', }, + 'microsoft-teams': { + serviceId: 'microsoft-teams', + description: 'Let each person connect one Microsoft Teams account', + configuration: 'oauth', + }, + outlook: { + serviceId: 'outlook', + description: 'Let each person connect one Outlook account', + configuration: 'oauth', + }, + onedrive: { + serviceId: 'onedrive', + description: 'Let each person connect one OneDrive account', + configuration: 'oauth', + }, + sharepoint: { + serviceId: 'sharepoint', + description: 'Let each person connect one SharePoint account', + configuration: 'oauth', + }, + 'microsoft-excel': { + serviceId: 'microsoft-excel', + description: 'Let each person connect one Microsoft Excel account', + configuration: 'oauth', + }, confluence: { serviceId: 'confluence', description: 'Let each person connect one Confluence account', From 7a2b6d57c90e3c2d1b9508b6b879e15ad2465efd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 01:54:35 -0700 Subject: [PATCH 38/76] fix(knowledge): close the connector access-switch and members-mode update races - Leaving members mode dispatched the first workspace sync against a fresh clock read while the row held the flip's nextSyncAt, so the queue refused it as stale; the dispatch now asserts the instant the flip wrote. - A workspace credential change kept the incremental watermark, so the new credential's corpus was never fully listed; it now drops lastSyncAt, makes the sync due, and refuses under a CAS while any sync owns the row. - Both switch directions released the lease before revoking the previous group's grant, and a revoke drops the connector from every option of the group; the lease now outlives the revoke. - The members-mode flip and the members-mode create lock the Credential Group row and re-check the option under it, the same lock the group's option removal and delete hold while they look for bound connectors. - Members-mode updates wrote interval changes and resumes to nextSyncAt, which the member scheduler never reads; they now land on nextMemberSyncAt under the matching CAS, a running member run refuses every edit, a queued one refuses config edits, and a pause releases the queued entry. --- apps/sim/lib/credential-groups/service.ts | 5 + .../orchestration/connector-access.test.ts | 249 +++++++++++++++--- .../orchestration/connector-access.ts | 144 ++++++---- .../orchestration/connectors.test.ts | 216 +++++++++++++++ .../lib/knowledge/orchestration/connectors.ts | 103 +++++++- 5 files changed, 625 insertions(+), 92 deletions(-) diff --git a/apps/sim/lib/credential-groups/service.ts b/apps/sim/lib/credential-groups/service.ts index 3cb3a02ad75..590171903f7 100644 --- a/apps/sim/lib/credential-groups/service.ts +++ b/apps/sim/lib/credential-groups/service.ts @@ -292,6 +292,11 @@ export async function createCredentialGroup( * connector syncs per member through one of them: the connector would be left * bound to nothing, and its members' documents dark, without anyone choosing * that. `optionIds` null means the whole group. + * + * Runs under the caller's `FOR UPDATE` on the group row. Every write that binds + * a connector row to an option (`lockCredentialGroupOption`) takes that same + * lock and re-checks the option under it, so a binding is either visible here + * or refused once this transaction commits; the check reads only the rows. */ async function refuseIfServingMemberConnectors( executor: DbOrTx, diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts index 1f0052fa6f5..17865e23eb1 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts @@ -1,7 +1,14 @@ /** * @vitest-environment node */ -import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { + dbChainMockFns, + hasMockCondition, + type MockCondition, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -103,6 +110,25 @@ function switchTo(target: Parameters ({ id, provider: 'google-drive', status: 'active' })) }, + ]) +} + +/** The values of the `set()` call that wrote `field`, so a test can read what a later call must repeat. */ +function setCallWith(field: string): Record { + const call = dbChainMockFns.set.mock.calls.find(([values]) => field in values) + if (!call) throw new Error(`No set() call wrote ${field}`) + return call[0] +} + +/** The `set()` calls carrying `field`, in order. */ +function setCallsWith(field: string): Record[] { + return dbChainMockFns.set.mock.calls.filter(([values]) => field in values).map(([v]) => v) +} + const SCOPED_META = { name: 'Google Drive', auth: { mode: 'oauth', provider: 'google-drive' }, @@ -238,9 +264,12 @@ describe('performUpdateKnowledgeConnectorAccess', () => { it('hides the documents, grants the option, flips to members mode, and queues the first member run', async () => { queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + queueGroupRow('option-1') dbChainMockFns.returning .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) + /** The rewrite finds nothing left, the flip lands under the lease, then the release. */ .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ id: 'c-1' }]) .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, nextMemberSyncAt: new Date() }]) const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) @@ -256,26 +285,91 @@ describe('performUpdateKnowledgeConnectorAccess', () => { 'admin-1' ) expect(dbChainMockFns.set).toHaveBeenCalledWith(expect.objectContaining({ acl: [] })) - expect(dbChainMockFns.set).toHaveBeenCalledWith( - expect.objectContaining({ - accessMode: 'members', - credentialId: null, - credentialGroupId: 'group-1', - credentialGroupOptionId: 'option-1', - accessRewritePending: false, - nextSyncAt: null, - nextMemberSyncAt: expect.any(Date), - status: 'active', - syncLockToken: null, - }) - ) - expect(mocks.dispatchMemberSync).toHaveBeenCalledWith( - 'c-1', - expect.objectContaining({ billingAttribution: BILLING, requireRunnable: true }) + /** The flip is written inside the group's row lock. */ + expect(dbChainMockFns.for).toHaveBeenCalledWith('update') + expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() + const flip = setCallWith('accessMode') + expect(flip).toMatchObject({ + accessMode: 'members', + credentialId: null, + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + accessRewritePending: false, + nextSyncAt: null, + nextMemberSyncAt: expect.any(Date), + }) + expect(flip).not.toHaveProperty('status') + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'active', syncLockToken: null, syncLockLeaseAt: null }) ) + /** The dispatch asserts the schedule the flip wrote, so the queue accepts it. */ + expect(mocks.dispatchMemberSync).toHaveBeenCalledWith('c-1', { + billingAttribution: BILLING, + expectedNextMemberSyncAt: flip.nextMemberSyncAt, + requestId: 'req-1', + requireRunnable: true, + }) expect(mocks.dispatchSync).not.toHaveBeenCalled() }) + it('refuses the flip, and undoes the grant, when the option is gone by the time the group is locked', async () => { + queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + queueGroupRow() + dbChainMockFns.returning + .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) + .mockResolvedValueOnce([]) + + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(setCallsWith('accessMode')).toEqual([]) + expect(mocks.revoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, + 'admin-1' + ) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'active', syncLockToken: null }) + ) + expect(mocks.dispatchMemberSync).not.toHaveBeenCalled() + }) + + it('keeps the lease until the previous group is revoked when moving between groups', async () => { + queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + queueGroupRow('option-2') + dbChainMockFns.returning + .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ id: 'c-1' }]) + .mockResolvedValueOnce([ + { ...MEMBERS_CONNECTOR, credentialGroupId: 'group-2', credentialGroupOptionId: 'option-2' }, + ]) + + const outcome = await switchTo({ + accessMode: 'members', + binding: { + credentialGroupId: 'group-2', + credentialGroupOptionId: 'option-2', + sourceConfig: {}, + }, + }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(mocks.revoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, + 'admin-1' + ) + /** + * A revoke drops the connector from every option of the group. Released + * first, a switch that had re-granted group-1 in between would lose it. + */ + const revokedAt = mocks.revoke.mock.invocationCallOrder[0] + const releasedAt = dbChainMockFns.set.mock.invocationCallOrder.at(-1) ?? 0 + expect(revokedAt).toBeLessThan(releasedAt) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'active', syncLockToken: null }) + ) + }) + it('drops the members, restores workspace access, revokes the grant, flips, and queues a content sync', async () => { queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) dbChainMockFns.returning @@ -294,34 +388,129 @@ describe('performUpdateKnowledgeConnectorAccess', () => { { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, 'admin-1' ) - expect(dbChainMockFns.set).toHaveBeenCalledWith( + const flip = setCallWith('accessMode') + expect(flip).toMatchObject({ + accessMode: 'workspace', + credentialId: 'cred-2', + credentialGroupId: null, + credentialGroupOptionId: null, + nextMemberSyncAt: null, + nextSyncAt: expect.any(Date), + }) + /** The revoke lands while the lease is still held; the release is the last write. */ + const revokedAt = mocks.revoke.mock.invocationCallOrder[0] + const releasedAt = dbChainMockFns.set.mock.invocationCallOrder.at(-1) ?? 0 + expect(revokedAt).toBeLessThan(releasedAt) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( expect.objectContaining({ - accessMode: 'workspace', - credentialId: 'cred-2', - credentialGroupId: null, - credentialGroupOptionId: null, - nextMemberSyncAt: null, - nextSyncAt: expect.any(Date), + accessRewritePending: false, + status: 'active', + syncLockToken: null, + syncLockLeaseAt: null, }) ) - expect(mocks.dispatchSync).toHaveBeenCalledWith( - 'c-1', - expect.objectContaining({ billingAttribution: BILLING, requireRunnable: true }) - ) + /** + * The row holds the instant the flip wrote as `nextSyncAt`; a dispatch + * asserting any later clock read is refused by the queue as stale. + */ + expect(mocks.dispatchSync).toHaveBeenCalledWith('c-1', { + billingAttribution: BILLING, + expectedNextSyncAt: flip.nextSyncAt, + requestId: 'req-1', + requireRunnable: true, + }) expect(mocks.dispatchMemberSync).not.toHaveBeenCalled() }) + it('changes a workspace credential without the lease, drops the watermark, and queues a full sync', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...WORKSPACE_CONNECTOR, lastSyncAt: new Date('2026-08-01T00:00:00Z') }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...WORKSPACE_CONNECTOR, credentialId: 'cred-2', lastSyncAt: null }, + ]) + + const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(dbChainMockFns.update).toHaveBeenCalledOnce() + const change = setCallWith('credentialId') + expect(change).toMatchObject({ + credentialId: 'cred-2', + lastSyncAt: null, + nextSyncAt: expect.any(Date), + }) + expect(change).not.toHaveProperty('status') + /** + * A running sync's terminal write would restore the watermark, so the + * write is refused while any sync holds the row. + */ + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls.at(-1)?.[0], + (node: MockCondition) => + node.type === 'isNull' && node.column === schemaMock.knowledgeConnector.syncLockToken + ) + ).toBe(true) + expect(mocks.dispatchSync).toHaveBeenCalledWith('c-1', { + billingAttribution: BILLING, + expectedNextSyncAt: change.nextSyncAt, + requestId: 'req-1', + requireRunnable: true, + }) + expect(mocks.grant).not.toHaveBeenCalled() + expect(mocks.revoke).not.toHaveBeenCalled() + }) + + it('refuses a credential change while a sync owns the connector', async () => { + queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + queueTableRows(schemaMock.knowledgeConnector, [ + { ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 'run-1' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + + const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) + + expect(outcome).toEqual({ + success: false, + error: 'Sync already in progress', + errorCode: 'conflict', + }) + expect(mocks.dispatchSync).not.toHaveBeenCalled() + }) + + it('changes the credential of a paused connector without queuing a sync', async () => { + queueTableRows(schemaMock.knowledgeConnector, [{ ...WORKSPACE_CONNECTOR, status: 'paused' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...WORKSPACE_CONNECTOR, status: 'paused', credentialId: 'cred-2' }, + ]) + + const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ credentialId: 'cred-2', lastSyncAt: null }) + ) + expect(mocks.dispatchSync).not.toHaveBeenCalled() + }) + it('leaves a paused connector paused and queues nothing', async () => { queueTableRows(schemaMock.knowledgeConnector, [{ ...WORKSPACE_CONNECTOR, status: 'paused' }]) + queueGroupRow('option-1') dbChainMockFns.returning .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'paused' }]) .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ id: 'c-1' }]) .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'paused' }]) - await switchTo({ accessMode: 'members', binding: BINDING }) + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + expect(outcome).toMatchObject({ success: true, changed: true }) expect(dbChainMockFns.set).toHaveBeenCalledWith( - expect.objectContaining({ accessMode: 'members', status: 'paused' }) + expect.objectContaining({ accessMode: 'members' }) + ) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'paused', syncLockToken: null }) ) expect(mocks.dispatchMemberSync).not.toHaveBeenCalled() }) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index 90662f66a65..fbe32cc2f9f 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -22,6 +22,7 @@ import { type ConnectorWithoutSecret, getKnowledgeConnector, type KnowledgeConnectorRow, + lockCredentialGroupOption, } from '@/lib/knowledge/orchestration/connectors' import { classifyKnowledgeFailure, @@ -157,21 +158,29 @@ function switchLeaseHeld(connectorId: string, switchId: string) { ) } -/** Releases a switch that could not complete, restoring the status it found. */ +/** + * Hands the lease back, restoring the status the switch found, along with any + * last values the switch writes as it ends. Returns the row as released, or + * null when the lease had already been taken away. + */ async function releaseSwitchLease( connectorId: string, switchId: string, - previousStatus: string -): Promise { - await db + previousStatus: string, + values: Partial = {} +): Promise { + const [row] = await db .update(knowledgeConnector) .set({ + ...values, status: previousStatus, syncLockToken: null, syncLockLeaseAt: null, updatedAt: new Date(), }) .where(switchLeaseHeld(connectorId, switchId)) + .returning() + return row ?? null } export interface PerformUpdateKnowledgeConnectorAccessParams extends KnowledgeOperationContext { @@ -194,15 +203,21 @@ export type PerformUpdateKnowledgeConnectorAccessResult = KnowledgeOrchestration * so neither engine runs against a half-rewritten corpus. * * Into members mode: grant the option's credentials first (a reversible policy - * write), rewrite every ACL to nobody, then flip. A rewrite that outgrows the - * request budget is finished by the first member run before it lists - * (`accessRewritePending`); documents are hidden early, never shown early. + * write), rewrite every ACL to nobody, flip under the Credential Group's row + * lock, then revoke the previous group's grant and release. A rewrite that + * outgrows the request budget is finished by the first member run before it + * lists (`accessRewritePending`); documents are hidden early, never shown + * early. * * Back to workspace mode: drop the members and flip in one transaction with * the rewrite marked pending, rewrite every ACL to the workspace while the - * lease is still held, then release and revoke the grant. A rewrite that + * lease is still held, then revoke the grant and release. A rewrite that * outgrows the budget, or is interrupted, is finished by the next content * sync (`accessRewritePending`); documents are hidden until then. + * + * Either way the lease outlives the revoke. A revoke drops the connector from + * every option of the group, so releasing first would let a switch that has + * just re-granted the same group lose its grant to this one's cleanup. */ export async function performUpdateKnowledgeConnectorAccess( params: PerformUpdateKnowledgeConnectorAccessParams @@ -257,18 +272,46 @@ export async function performUpdateKnowledgeConnectorAccess( } /** - * Staying in workspace mode with a different credential is a plain credential - * change: no document's visibility moves, so nothing needs the lease. + * Staying in workspace mode with a different credential moves no document's + * visibility, so the lease is not taken. It does change what the source + * shows: the new credential may see a different corpus, and only a full + * listing reconciles that, so the incremental watermark is dropped and a sync + * queued. The write refuses while a sync owns the row, whose terminal write + * would otherwise put the watermark straight back. */ if (target.accessMode === 'workspace' && existing.accessMode === 'workspace') { const now = new Date() const [updated] = await db .update(knowledgeConnector) - .set({ credentialId: target.credentialId, updatedAt: now }) - .where(and(eq(knowledgeConnector.id, connectorId), isNull(knowledgeConnector.deletedAt))) + .set({ + credentialId: target.credentialId, + lastSyncAt: null, + nextSyncAt: now, + updatedAt: now, + }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id), + inArray(knowledgeConnector.status, SWITCHABLE_CONNECTOR_STATUSES), + eq(knowledgeConnector.status, existing.status), + isNull(knowledgeConnector.syncLockToken), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) .returning() - if (!updated) return fail('Connector not found', 'not_found') + if (!updated) { + const current = await getKnowledgeConnector(kb.id, connectorId) + return current + ? fail('Sync already in progress', 'conflict') + : fail('Connector not found', 'not_found') + } + logger.info(`[${requestId}] Changed the credential of connector ${connectorId}`) const { encryptedApiKey: _secret, ...connector } = updated + if (existing.status !== 'paused') { + await dispatchContentSyncBestEffort(connectorId, params, requestId, now) + } return { success: true, connector, changed: true } } @@ -298,29 +341,39 @@ export async function performUpdateKnowledgeConnectorAccess( const rewritten = await rewriteConnectorAcls(connectorId, EMPTY_ACL, { deadlineAt: deadlineAt, }) - const now = new Date() - const [updated] = await db - .update(knowledgeConnector) - .set({ - accessMode: 'members', - credentialId: null, + /** + * The flip lands under the group's row lock, which the group's option + * edits and delete hold while they look for connectors bound to what + * they remove: an option gone by the time the lock is ours refuses the + * flip, and one removed after it finds this row. + */ + const flippedAt = new Date() + await db.transaction(async (tx) => { + await lockCredentialGroupOption(tx, { + workspaceId: kb.workspaceId, credentialGroupId: target.binding.credentialGroupId, credentialGroupOptionId: target.binding.credentialGroupOptionId, - sourceConfig: target.binding.sourceConfig, - accessRewritePending: !rewritten, - memberSyncStatus: 'idle', - memberSyncConsecutiveFailures: 0, - lastMemberSyncError: null, - nextMemberSyncAt: now, - nextSyncAt: null, - status: previousStatus, - syncLockToken: null, - syncLockLeaseAt: null, - updatedAt: now, }) - .where(switchLeaseHeld(connectorId, switchId)) - .returning() - if (!updated) throw new SwitchLeaseLostError() + const [row] = await tx + .update(knowledgeConnector) + .set({ + accessMode: 'members', + credentialId: null, + credentialGroupId: target.binding.credentialGroupId, + credentialGroupOptionId: target.binding.credentialGroupOptionId, + sourceConfig: target.binding.sourceConfig, + accessRewritePending: !rewritten, + memberSyncStatus: 'idle', + memberSyncConsecutiveFailures: 0, + lastMemberSyncError: null, + nextMemberSyncAt: flippedAt, + nextSyncAt: null, + updatedAt: flippedAt, + }) + .where(switchLeaseHeld(connectorId, switchId)) + .returning({ id: knowledgeConnector.id }) + if (!row) throw new SwitchLeaseLostError() + }) if ( existing.credentialGroupId && existing.credentialGroupId !== target.binding.credentialGroupId @@ -339,12 +392,14 @@ export async function performUpdateKnowledgeConnectorAccess( }) }) } + const updated = await releaseSwitchLease(connectorId, switchId, previousStatus) + if (!updated) throw new SwitchLeaseLostError() logger.info(`[${requestId}] Switched connector ${connectorId} to members mode`, { rewritten, }) const { encryptedApiKey: _secret, ...connector } = updated if (previousStatus !== 'paused') { - await dispatchMemberSyncBestEffort(connectorId, params, requestId, now) + await dispatchMemberSyncBestEffort(connectorId, params, requestId, flippedAt) } return { success: true, connector, changed: true } } catch (error) { @@ -426,19 +481,6 @@ export async function performUpdateKnowledgeConnectorAccess( const rewritten = await rewriteConnectorAcls(connectorId, WORKSPACE_ACL, { deadlineAt: deadlineAt, }) - const now = new Date() - const [updated] = await db - .update(knowledgeConnector) - .set({ - accessRewritePending: !rewritten, - status: previousStatus, - syncLockToken: null, - syncLockLeaseAt: null, - updatedAt: now, - }) - .where(switchLeaseHeld(connectorId, switchId)) - .returning() - if (!updated) throw new SwitchLeaseLostError() if (existing.credentialGroupId) { await revokeKnowledgeConnectorCredentialAccess( { workspaceId: kb.workspaceId, credentialGroupId: existing.credentialGroupId, connectorId }, @@ -450,12 +492,17 @@ export async function performUpdateKnowledgeConnectorAccess( }) }) } + const updated = await releaseSwitchLease(connectorId, switchId, previousStatus, { + accessRewritePending: !rewritten, + }) + if (!updated) throw new SwitchLeaseLostError() logger.info(`[${requestId}] Switched connector ${connectorId} to workspace mode`, { rewritten, }) const { encryptedApiKey: _secret, ...connector } = updated if (previousStatus !== 'paused') { - await dispatchContentSyncBestEffort(connectorId, params, requestId, now) + /** The dispatch asserts the schedule the flip wrote, not a later clock read. */ + await dispatchContentSyncBestEffort(connectorId, params, requestId, flippedAt) } return { success: true, connector, changed: true } } catch (error) { @@ -468,6 +515,7 @@ export async function performUpdateKnowledgeConnectorAccess( connectorId, error: releaseError, }) + return null }) return classifyKnowledgeFailure( error, diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index 1ee0603bc19..fb0bb804e28 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -67,6 +67,12 @@ vi.mock('@/connectors/registry.server', () => ({ auth: { mode: 'apiKey', optional: true }, validateConfig: vi.fn().mockResolvedValue({ valid: true }), }, + google_drive: { + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive' }, + permissionScopedListing: { capFieldIds: [] }, + validateConfig: vi.fn().mockResolvedValue({ valid: true }), + }, }, })) @@ -1028,6 +1034,216 @@ describe('members-mode connectors', () => { expect(mockDispatchMemberSync).not.toHaveBeenCalled() }) + /** A CAS clause of the last connector update, found by shape since the mock evaluates nothing. */ + function updateCasHas(predicate: (node: MockCondition) => boolean): boolean { + return hasMockCondition(dbChainMockFns.where.mock.calls.at(-1)?.[0], predicate) + } + + it('writes an interval change to the member schedule, which is what the member scheduler reads', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, nextSyncAt: null, nextMemberSyncAt: null }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { syncIntervalMinutes: 60 }, + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: true }) + const values = dbChainMockFns.set.mock.calls[0][0] + expect(values).toMatchObject({ syncIntervalMinutes: 60, nextMemberSyncAt: expect.any(Date) }) + expect(values).not.toHaveProperty('nextSyncAt') + expect( + updateCasHas( + (node) => + node.type === 'isNull' && node.column === schemaMock.knowledgeConnector.nextMemberSyncAt + ) + ).toBe(true) + expect(mockDispatchMemberSync).not.toHaveBeenCalled() + }) + + it('clears the member schedule when scheduled sync is turned off', async () => { + const scheduled = new Date(Date.now() + 60 * 60 * 1000) + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, nextSyncAt: null, nextMemberSyncAt: scheduled }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) + + await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { syncIntervalMinutes: 0 }, + resolveBillingAttribution, + }) + + expect(dbChainMockFns.set.mock.calls[0][0]).toMatchObject({ + syncIntervalMinutes: 0, + nextMemberSyncAt: null, + }) + expect( + updateCasHas( + (node) => + node.type === 'eq' && + node.left === schemaMock.knowledgeConnector.nextMemberSyncAt && + node.right === scheduled + ) + ).toBe(true) + }) + + it('resumes a paused members-mode connector by making its member run due', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, status: 'paused', nextSyncAt: null, nextMemberSyncAt: null }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { status: 'active' }, + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: true }) + const values = dbChainMockFns.set.mock.calls[0][0] + expect(values).toMatchObject({ status: 'active', nextMemberSyncAt: expect.any(Date) }) + expect(values).not.toHaveProperty('nextSyncAt') + }) + + it('refuses any edit while a member run is running', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, memberSyncStatus: 'running' }, + ]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { sourceConfig: { database: 'other' } }, + resolveBillingAttribution, + }) + + /** + * The member run reads `sourceConfig` once at its start and reconciles + * against it, exactly as the content engine does; `status` stays `active` + * throughout, so only the member lease shows the row is owned. + */ + expect(outcome).toMatchObject({ + success: false, + errorCode: 'conflict', + error: 'Sync already in progress', + }) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('refuses a config edit while a member run is queued, but lets a pause release the entry', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, memberSyncStatus: 'pending', memberSyncLockToken: 'd-1' }, + ]) + + const refused = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { syncIntervalMinutes: 30 }, + resolveBillingAttribution, + }) + expect(refused).toMatchObject({ success: false, errorCode: 'conflict' }) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, memberSyncStatus: 'pending', memberSyncLockToken: 'd-1' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'paused' }]) + + const paused = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { status: 'paused' }, + resolveBillingAttribution, + }) + + /** + * The queued task starts without re-checking `status`, so the entry has to + * go for the pause to hold; the CAS keeps that off a run that has started. + */ + expect(paused).toMatchObject({ success: true }) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'paused', + memberSyncStatus: 'idle', + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + }) + ) + expect( + updateCasHas( + (node) => + node.type === 'eq' && + node.left === schemaMock.knowledgeConnector.memberSyncStatus && + node.right === 'pending' + ) + ).toBe(true) + }) + + it('binds a new members-mode connector under the group lock', async () => { + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.credentialGroup, [{ options: [{ id: 'option-1' }] }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...MEMBERS_CONNECTOR, connectorType: 'google_drive' }, + ]) + + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: KB, + connectorType: 'google_drive', + sourceConfig: {}, + syncIntervalMinutes: 1440, + membersBinding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + resolveBillingAttribution, + resolveAccessToken: vi.fn(), + ...ACTOR, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(dbChainMockFns.for).toHaveBeenCalledWith('update') + expect(dbChainMockFns.insert).toHaveBeenCalledOnce() + expect(mockRevoke).not.toHaveBeenCalled() + }) + + it('refuses to create a members-mode connector on an option removed before the group was locked', async () => { + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.credentialGroup, [{ options: [{ id: 'option-2' }] }]) + + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: KB, + connectorType: 'google_drive', + sourceConfig: {}, + syncIntervalMinutes: 1440, + membersBinding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + resolveBillingAttribution, + resolveAccessToken: vi.fn(), + ...ACTOR, + }) + + /** + * The group's option edit refuses while a connector row is bound to what it + * removes, and this row is written under the same lock, so the two can + * never both commit: the grant is undone and no row is inserted. + */ + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + expect(mockRevoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: expect.any(String) }, + 'user-1' + ) + }) + it('refuses members mode for a connector whose listing is not permission scoped, before any grant', async () => { queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index c8e4bd60073..8c5596e220f 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -1,6 +1,7 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' import { + credentialGroup, document, embedding, knowledgeBase, @@ -16,6 +17,7 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attr import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' import { OrchestrationError, type OrchestrationErrorCode } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' +import type { DbOrTx } from '@/lib/db/types' import { findListingCapViolation, grantKnowledgeConnectorCredentialAccess, @@ -57,6 +59,41 @@ export interface ConnectorMembersBinding { credentialGroupOptionId: string } +/** + * Locks the Credential Group's row for the rest of the transaction and confirms + * the option is still part of it. The group's option edits and delete take the + * same row lock and refuse while a connector row is bound to what they remove, + * so a binding written under this lock is serialized against them: it either + * finds the option gone, or lands before the removal looks for it. The grant + * itself is a policy write with its own revision CAS, which is why the row + * write, not the grant, is what takes the lock. + */ +export async function lockCredentialGroupOption( + tx: DbOrTx, + binding: ConnectorMembersBinding & { workspaceId: string } +): Promise { + const [group] = await tx + .select({ options: credentialGroup.options }) + .from(credentialGroup) + .where( + and( + eq(credentialGroup.id, binding.credentialGroupId), + eq(credentialGroup.workspaceId, binding.workspaceId) + ) + ) + .limit(1) + .for('update') + if (!group) { + throw new OrchestrationError('validation', 'Credential Group was not found in this workspace') + } + if (!group.options.some((option) => option.id === binding.credentialGroupOptionId)) { + throw new OrchestrationError( + 'validation', + 'Credential option was not found in this Credential Group' + ) + } +} + /** A connector row exactly as stored, including its encrypted API key. */ export type KnowledgeConnectorRow = typeof knowledgeConnector.$inferSelect type ConnectorRow = KnowledgeConnectorRow @@ -333,6 +370,9 @@ export async function performCreateKnowledgeConnector( if (activeKb.length === 0) { throw new OrchestrationError('not_found', 'Knowledge base not found') } + if (membersBinding) { + await lockCredentialGroupOption(tx, { workspaceId, ...membersBinding }) + } for (const [semanticId, slot] of Object.entries(newTagSlots)) { const td = connectorConfig.tagDefinitions?.find((d) => d.id === semanticId) @@ -590,6 +630,24 @@ export async function performUpdateKnowledgeConnector( ) { return fail('Sync already in progress', 'conflict') } + /** + * A members-mode connector is run by the member engine, whose lease lives in + * `memberSyncStatus` while `status` stays `active`, so the two guards above + * never see it. The same two rules apply to that lease: a running member run + * owns the row, and a queued one has not read its config yet, so only a + * status change is safe. + */ + const syncsPerMember = existing.accessMode === 'members' + if (syncsPerMember && existing.memberSyncStatus === 'running') { + return fail('Sync already in progress', 'conflict') + } + if ( + syncsPerMember && + existing.memberSyncStatus === 'pending' && + (updates.sourceConfig !== undefined || updates.syncIntervalMinutes !== undefined) + ) { + return fail('Sync already in progress', 'conflict') + } if (updates.syncIntervalMinutes !== undefined) { if (!kb.workspaceId && updates.syncIntervalMinutes > 0 && updates.syncIntervalMinutes < 60) { @@ -634,7 +692,14 @@ export async function performUpdateKnowledgeConnector( updates.sourceConfig !== undefined && resultingStatus !== 'paused' && resultingStatus !== 'disabled' - const syncsPerMember = existing.accessMode === 'members' + /** + * The schedule this connector is picked up by: the member scheduler reads + * `nextMemberSyncAt` and the content scheduler `nextSyncAt`, each only for + * its own access mode, so every schedule write below lands on the one the + * connector's engine will read. + */ + const scheduleColumn = syncsPerMember ? 'nextMemberSyncAt' : 'nextSyncAt' + const existingSchedule = existing[scheduleColumn] let billingAttribution: BillingAttributionSnapshot | undefined let dispatchSourceSync: Awaited> | undefined let dispatchMemberSourceSync: Awaited> | undefined @@ -657,9 +722,9 @@ export async function performUpdateKnowledgeConnector( } if (updates.syncIntervalMinutes !== undefined) { values.syncIntervalMinutes = updates.syncIntervalMinutes - values.nextSyncAt = - existing.nextSyncAt && existing.nextSyncAt <= updateTimestamp - ? existing.nextSyncAt + values[scheduleColumn] = + existingSchedule && existingSchedule <= updateTimestamp + ? existingSchedule : updates.syncIntervalMinutes > 0 ? new Date(updateTimestamp.getTime() + updates.syncIntervalMinutes * 60 * 1000) : null @@ -669,25 +734,32 @@ export async function performUpdateKnowledgeConnector( /** * Releases a queue entry this status change is walking away from, so no * token survives on a row that is no longer `pending` and the reaper is not - * left with a lease it can never match. + * left with a lease it can never match. A queued member run is released the + * same way: its task starts without re-checking `status`, so the entry has + * to be gone for a pause to hold, and the CAS below keeps this off a run + * that has since started. */ if (existing.status === 'pending') { values.syncLockToken = null values.syncLockLeaseAt = null } + if (syncsPerMember && existing.memberSyncStatus === 'pending') { + values.memberSyncStatus = 'idle' + values.memberSyncLockToken = null + values.memberSyncLockLeaseAt = null + } if (updates.status === 'active') { values.consecutiveFailures = 0 values.lastSyncError = null // Resuming a paused connector syncs immediately unless this same request // set a schedule, which then owns the next run. - if (values.nextSyncAt === undefined) { - values.nextSyncAt = new Date() + if (values[scheduleColumn] === undefined) { + values[scheduleColumn] = new Date() } } } if (shouldDispatchSourceSync) { - if (syncsPerMember) values.nextMemberSyncAt = updateTimestamp - else values.nextSyncAt = updateTimestamp + values[scheduleColumn] = updateTimestamp } let updated: ConnectorRow @@ -699,11 +771,14 @@ export async function performUpdateKnowledgeConnector( isNull(knowledgeConnector.deletedAt), ] updateConditions.push(eq(knowledgeConnector.status, existing.status)) - if (values.nextSyncAt !== undefined) { + if (syncsPerMember) { + updateConditions.push(eq(knowledgeConnector.memberSyncStatus, existing.memberSyncStatus)) + } + if (values[scheduleColumn] !== undefined) { updateConditions.push( - existing.nextSyncAt - ? eq(knowledgeConnector.nextSyncAt, existing.nextSyncAt) - : isNull(knowledgeConnector.nextSyncAt) + existingSchedule + ? eq(knowledgeConnector[scheduleColumn], existingSchedule) + : isNull(knowledgeConnector[scheduleColumn]) ) } @@ -715,7 +790,7 @@ export async function performUpdateKnowledgeConnector( if (!row) { const current = await getKnowledgeConnector(kb.id, connectorId) - if (current?.status === 'syncing') { + if (current?.status === 'syncing' || current?.memberSyncStatus === 'running') { return fail('Sync already in progress', 'conflict') } if (current) { From 9624c0141e05fd41f71306fc0988b6a1d911f686 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 02:03:59 -0700 Subject: [PATCH 39/76] fix(knowledge): close member-sync races found in review - sweep stale member observations under the connector share lock and the member row lock, re-checking mode and staleness so a run's fresh observations and a mode switch's ACLs are never overwritten - gate the member queue CAS and lease on connector status and on the schedule the dispatch was made for; keep dispatching after one failure - prove the run lease inside every connector document write transaction - drop the per-member page cap on listing passes; a capped listing could never reach the documents behind page 200 - issue member invitations with reject-on-revoked so a concurrent revocation is never reactivated - re-apply the caller's access at document delete and upsert-replace - keep sourceModifiedAt/connectorType on provenance-bearing searches - read Google Calendar's updatedTime as the source modified time --- apps/sim/lib/credential-groups/enrollments.ts | 22 +- .../knowledge/application/documents.test.ts | 65 +++++- .../lib/knowledge/application/documents.ts | 46 +++- .../lib/knowledge/application/search.test.ts | 37 ++++ apps/sim/lib/knowledge/application/search.ts | 24 ++- .../connectors/member-observations.test.ts | 90 ++++++++ .../connectors/member-observations.ts | 119 ++++++++--- .../connectors/member-provisioning.ts | 60 ++++-- .../knowledge/connectors/member-queue.test.ts | 120 ++++++++++- .../lib/knowledge/connectors/member-queue.ts | 60 ++++-- .../connectors/member-sync-engine.ts | 23 +- .../connectors/source-modified-at.test.ts | 1 + .../connectors/source-modified-at.ts | 1 + .../knowledge/connectors/sync-engine.test.ts | 201 ++++++++++++------ .../lib/knowledge/connectors/sync-limits.ts | 15 +- .../sim/lib/knowledge/connectors/sync-lock.ts | 10 + .../knowledge/connectors/sync-persistence.ts | 41 +++- .../knowledge/connectors/sync-primitives.ts | 27 ++- apps/sim/lib/knowledge/documents/service.ts | 66 ++++-- 19 files changed, 846 insertions(+), 182 deletions(-) create mode 100644 apps/sim/lib/knowledge/connectors/member-observations.test.ts diff --git a/apps/sim/lib/credential-groups/enrollments.ts b/apps/sim/lib/credential-groups/enrollments.ts index a22ac3fd42a..7ac30a2c80a 100644 --- a/apps/sim/lib/credential-groups/enrollments.ts +++ b/apps/sim/lib/credential-groups/enrollments.ts @@ -64,9 +64,12 @@ interface InvitationContext { groupName: string } +/** What issuing an invitation does to an enrollment an admin revoked. */ +export type RevokedEnrollmentPolicy = 'reactivate' | 'reject' + interface SendInvitationOptions { expectedEnrollmentId?: string - revokedEnrollment: 'reactivate' | 'reject' + revokedEnrollment: RevokedEnrollmentPolicy } interface IssuedInvitation { @@ -772,11 +775,18 @@ export async function inviteCredentialGroupEnrollment( userId: string | undefined, /** See {@link sendInvitation}: absent for a workflow-issued invitation. */ inviterName: string | undefined, - email: string + email: string, + /** + * What a revoked enrollment does to the invitation, decided inside the + * issuing transaction. An admin's invite reactivates it; an automatic + * invitation rejects it, so a revocation that lands after the caller read + * the enrollment is never undone by a stale read. + */ + revokedEnrollment: RevokedEnrollmentPolicy = 'reactivate' ): Promise { const context = await getInvitationContext(workspaceId, groupId) return sendInvitation(context, userId, inviterName, normalizeEmail(email), { - revokedEnrollment: 'reactivate', + revokedEnrollment, }) } @@ -785,11 +795,13 @@ export async function createCredentialGroupInvitationLink( groupId: string, /** See {@link issueInvitation}: the issuer is attribution, never the authority. */ userId: string | undefined, - email: string + email: string, + /** See {@link inviteCredentialGroupEnrollment}. */ + revokedEnrollment: RevokedEnrollmentPolicy = 'reactivate' ): Promise { const context = await getInvitationContext(workspaceId, groupId) const issued = await issueInvitation(context, userId, normalizeEmail(email), { - revokedEnrollment: 'reactivate', + revokedEnrollment, }) return { enrollment: toCredentialGroupEnrollment(issued.enrollment), diff --git a/apps/sim/lib/knowledge/application/documents.test.ts b/apps/sim/lib/knowledge/application/documents.test.ts index cbad2a97702..e3fad4463aa 100644 --- a/apps/sim/lib/knowledge/application/documents.test.ts +++ b/apps/sim/lib/knowledge/application/documents.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -256,6 +256,69 @@ describe('knowledge document application use cases', () => { }) }) + /** + * The document being replaced is looked up and deleted under the caller's + * access, so a restricted document is neither confirmed nor replaced, and one + * that leaves the caller's reach mid-request keeps the replacement as an + * ordinary upload. + */ + it('replaces only a document the caller may read, under the same access', async () => { + queueTableRows(schemaMock.document, [{ id: 'existing-1' }]) + + const result = await upsertKnowledgeDocument.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + resolveBillingAttribution: async () => ({ + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }), + resolveSecretProvenances: () => undefined, + }, + }) + + expect(result).toMatchObject({ isUpdate: true, previousDocumentId: 'existing-1' }) + expect(mocks.deleteDocument).toHaveBeenCalledWith( + 'knowledge-1', + 'existing-1', + expect.any(String), + WORKSPACE_ACCESS_SCOPE + ) + expect(mocks.deleteDocumentById).not.toHaveBeenCalled() + }) + + it('keeps the replacement when the previous document left the caller’s reach', async () => { + queueTableRows(schemaMock.document, [{ id: 'existing-1' }]) + mocks.deleteDocument.mockRejectedValueOnce( + new OrchestrationError('not_found', 'Document not found') + ) + + const result = await upsertKnowledgeDocument.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + resolveBillingAttribution: async () => ({ + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }), + resolveSecretProvenances: () => undefined, + }, + }) + + expect(result).toMatchObject({ isUpdate: false, previousDocumentId: null }) + expect(mocks.deleteDocumentById).not.toHaveBeenCalled() + }) + it('authorizes the canonical knowledge base before listing documents', async () => { await listKnowledgeDocuments.execute({ principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index 47a35371412..c1a6961b4e3 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -11,6 +11,7 @@ import { import { authorizeWorkspaceOperation } from '@/lib/core/application' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { @@ -660,6 +661,11 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ userId, workspaceId: context.workspaceId, }) + /** + * Only a document the caller may read counts as the one being replaced: + * a restricted document is neither confirmed to exist nor replaced. + */ + const access = await context.access.get() let existingDocumentId: string | null = null if (input.documentId) { const [existing] = await db @@ -669,7 +675,8 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ and( eq(documentTable.id, input.documentId), eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), - isNull(documentTable.deletedAt) + isNull(documentTable.deletedAt), + knowledgeAccessCondition(access) ) ) .limit(1) @@ -682,7 +689,8 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ and( eq(documentTable.filename, input.filename), eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), - isNull(documentTable.deletedAt) + isNull(documentTable.deletedAt), + knowledgeAccessCondition(access) ) ) .limit(1) @@ -708,18 +716,36 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ if (!createdDocument) throw new Error('Knowledge document upsert created no document record') if (existingDocumentId) { try { - await deleteDocument(existingDocumentId, requestId) + await deleteKnowledgeDocumentInKnowledgeBase( + context.knowledgeBaseId, + existingDocumentId, + requestId, + access + ) } catch (error) { - try { - await deleteDocument(createdDocument.documentId, requestId) - } catch (rollbackError) { - logger.error('Failed to remove replacement after document upsert failure', { + /** + * The previous document went away — or out of the caller's reach — + * between the lookup and the delete. The replacement is an ordinary + * upload the caller may make, so it stays. + */ + if (error instanceof OrchestrationError && error.code === 'not_found') { + logger.warn('Document being replaced was no longer visible; keeping the replacement', { knowledgeBaseId: context.knowledgeBaseId, - documentId: createdDocument.documentId, - rollbackError, + previousDocumentId: existingDocumentId, }) + existingDocumentId = null + } else { + try { + await deleteDocument(createdDocument.documentId, requestId) + } catch (rollbackError) { + logger.error('Failed to remove replacement after document upsert failure', { + knowledgeBaseId: context.knowledgeBaseId, + documentId: createdDocument.documentId, + rollbackError, + }) + } + throw new Error('Failed to replace existing document', { cause: error }) } - throw new Error('Failed to replace existing document', { cause: error }) } } void dispatchDocumentProcessing({ diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 6070b096137..e5566f681bc 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -318,6 +318,43 @@ describe('knowledge search application use case', () => { }) }) + /** + * The provenance snapshot vouches for the name, URL, and tags; the source + * card's modified time and connector type only come from the access-filtered + * metadata read, which a provenance-bearing search must therefore still make. + */ + it('keeps the source card metadata when a provenance registry is present', async () => { + const registry = { markIncomplete: vi.fn() } + const sourceModifiedAt = new Date('2026-08-20T12:00:00Z') + mocks.getDocumentMetadata.mockResolvedValueOnce({ + 'document-1': { + filename: 'guide.pdf', + sourceUrl: 'https://example.com/guide', + sourceModifiedAt, + connectorType: 'google_drive', + }, + }) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + workspaceId: 'workspace-1', + knowledgeBaseIds: ['knowledge-1'], + query: 'answer', + topK: 5, + resultSecretRegistry: registry as never, + }, + }) + + expect(mocks.getDocumentMetadata).toHaveBeenCalledWith(['document-1'], expect.anything()) + expect(result.results[0]).toMatchObject({ + documentName: 'guide.pdf', + sourceUrl: 'https://example.com/guide', + sourceModifiedAt, + connectorType: 'google_drive', + }) + }) + describe('reranker outcome reporting', () => { const rerankedSearch = (rerankerEnabled?: boolean, query: string | undefined = 'answer') => searchKnowledge.execute({ diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index e8d025cb919..ae87287303b 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -486,17 +486,21 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ }) ) const tagMaps = new Map(tagDefinitionEntries) - const basicDocumentMetadata = provenanceSnapshot - ? {} - : await getDocumentMetadataByIds( - rows.map((row) => row.documentId), - access - ) + /** + * Always read: the provenance snapshot vouches for the name, URL, and tags + * a model may see, but the source card's modified time and connector type + * are only carried here, under the same access predicate as the search. + */ + const basicDocumentMetadata = await getDocumentMetadataByIds( + rows.map((row) => row.documentId), + access + ) const results = rows.map((row): KnowledgeSearchItem => { const metadata: Record = {} const tagMap = tagMaps.get(row.knowledgeBaseId) const provenanceDocument = provenanceSnapshot?.documentMetadata[row.documentId] - const document = provenanceDocument ?? basicDocumentMetadata[row.documentId] + const basicDocument = basicDocumentMetadata[row.documentId] + const document = provenanceDocument ?? basicDocument for (const slot of ALL_TAG_SLOTS) { const value = provenanceDocument && slot.startsWith('tag') @@ -513,10 +517,8 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ documentId: row.documentId, documentName: document?.filename ?? null, sourceUrl: document?.sourceUrl ?? null, - sourceModifiedAt: - document && 'sourceModifiedAt' in document ? (document.sourceModifiedAt ?? null) : null, - connectorType: - document && 'connectorType' in document ? (document.connectorType ?? null) : null, + sourceModifiedAt: basicDocument?.sourceModifiedAt ?? null, + connectorType: basicDocument?.connectorType ?? null, content: row.content, chunkIndex: row.chunkIndex, metadata, diff --git a/apps/sim/lib/knowledge/connectors/member-observations.test.ts b/apps/sim/lib/knowledge/connectors/member-observations.test.ts new file mode 100644 index 00000000000..85163b0e03d --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-observations.test.ts @@ -0,0 +1,90 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/knowledge/documents/service', () => ({ + hardDeleteDocuments: vi.fn(), +})) + +import { + staleMemberWindowMs, + sweepStaleMemberObservations, +} from '@/lib/knowledge/connectors/member-observations' +import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' + +const NOW = new Date('2026-09-01T12:00:00Z') +const STALE_MEMBER = { id: 'm-1', connectorId: 'c-1', syncIntervalMinutes: 60 } + +describe('staleMemberWindowMs', () => { + it('is the larger of a day and two intervals', () => { + const day = MEMBER_OBSERVATION_STALE_AFTER_HOURS * 60 * 60 * 1000 + expect(staleMemberWindowMs(60)).toBe(day) + expect(staleMemberWindowMs(0)).toBe(day) + expect(staleMemberWindowMs(24 * 60)).toBe(2 * 24 * 60 * 60 * 1000) + }) +}) + +describe('sweepStaleMemberObservations', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('sweeps a member that is still stale once its row is locked', async () => { + queueTableRows(schemaMock.knowledgeConnectorMember, [STALE_MEMBER]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) + queueTableRows(schemaMock.knowledgeConnectorMember, [{ id: 'm-1' }]) + dbChainMockFns.returning + .mockResolvedValueOnce([{ documentId: 'd-1' }, { documentId: 'd-2' }]) + .mockResolvedValueOnce([{ id: 'd-1' }, { id: 'd-2' }]) + .mockResolvedValueOnce([{ id: 'd-2' }]) + + await expect(sweepStaleMemberObservations(NOW)).resolves.toEqual({ + members: 1, + observationsRemoved: 2, + documentsRematerialized: 2, + docsTombstoned: 1, + }) + + expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() + expect(dbChainMockFns.for).toHaveBeenNthCalledWith(1, 'share') + expect(dbChainMockFns.for).toHaveBeenNthCalledWith(2, 'update') + expect(dbChainMockFns.delete).toHaveBeenCalledWith(schemaMock.knowledgeDocumentObservation) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith({ deletedAt: NOW }) + }) + + /** + * A run that claimed the member between the selection and the lock moved + * `lastStartedAt` forward, so the re-check under `FOR UPDATE` finds nothing + * and the observations that run is about to write are left alone. + */ + it('leaves a member that a run claimed after it was selected', async () => { + queueTableRows(schemaMock.knowledgeConnectorMember, [STALE_MEMBER]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) + queueTableRows(schemaMock.knowledgeConnectorMember, []) + + await expect(sweepStaleMemberObservations(NOW)).resolves.toEqual({ + members: 0, + observationsRemoved: 0, + documentsRematerialized: 0, + docsTombstoned: 0, + }) + + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + /** A connector that left members mode after the selection no longer matches the shared lock's re-check. */ + it('leaves a connector that left members mode after it was selected', async () => { + queueTableRows(schemaMock.knowledgeConnectorMember, [STALE_MEMBER]) + queueTableRows(schemaMock.knowledgeConnector, []) + + await expect(sweepStaleMemberObservations(NOW)).resolves.toMatchObject({ members: 0 }) + + expect(dbChainMockFns.for).toHaveBeenCalledWith('share') + expect(dbChainMockFns.for).not.toHaveBeenCalledWith('update') + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index f4a2c549696..f75def9484d 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -26,7 +26,11 @@ import { MEMBER_PURGE_MAX_PER_RUN, MEMBER_TOMBSTONE_PURGE_DAYS, } from '@/lib/knowledge/connectors/sync-limits' -import type { SyncRunLease } from '@/lib/knowledge/connectors/sync-lock' +import { + connectorIsLive, + MEMBER_LOCKABLE_CONNECTOR_STATUSES, + type SyncRunLease, +} from '@/lib/knowledge/connectors/sync-lock' import { type ConnectorSyncDeletionGuard, hardDeleteDocuments, @@ -364,6 +368,30 @@ export interface StaleMemberSweepResult { docsTombstoned: number } +/** How long a member's crawls may be silent before the sweep treats them as gone: `max(24 h, 2 × interval)`. */ +export function staleMemberWindowMs(syncIntervalMinutes: number): number { + return Math.max( + MEMBER_OBSERVATION_STALE_AFTER_HOURS * 60 * 60 * 1000, + 2 * syncIntervalMinutes * 60 * 1000 + ) +} + +/** The staleness a member is re-checked against once its row is locked; the same clock as the selection. */ +function memberStillStale(memberId: string, cutoff: Date) { + return and( + eq(knowledgeConnectorMember.id, memberId), + eq(knowledgeConnectorMember.status, 'active'), + or( + isNull(knowledgeConnectorMember.lastStartedAt), + lt(knowledgeConnectorMember.lastStartedAt, cutoff) + ), + or( + isNull(knowledgeConnectorMember.lastCompleteListingAt), + lt(knowledgeConnectorMember.lastCompleteListingAt, cutoff) + ) + ) +} + /** * Removes the observations of members whose crawls have stopped, so the * documents only they observed go dark instead of staying readable forever. @@ -379,6 +407,14 @@ export interface StaleMemberSweepResult { * `MEMBER_SUSPENDED_PURGE_DAYS`. The member row survives; the next run that * lists for them rebuilds their observations. Purging is left to a run holding * the lease. + * + * Each member is swept in one transaction that first shares the connector row + * — which a member run holds `FOR UPDATE` while it writes and a mode switch + * updates when it flips — and then locks the member row, which `claimNextMember` + * skips while locked. Both are re-checked under those locks, so a run that + * claimed the member after the selection, or a switch that left members mode, + * makes the sweep skip rather than delete observations a run just wrote or + * rewrite ACLs the switch just set. */ export async function sweepStaleMemberObservations(now: Date): Promise { const staleWindow = sql`GREATEST( @@ -390,6 +426,7 @@ export async function sweepStaleMemberObservations(now: Date): Promise row.documentId) - result.observationsRemoved += documentIds.length - result.documentsRematerialized += await materializeDocumentAcls(member.connectorId, documentIds) - if (documentIds.length > 0) { - const tombstoned = await db - .update(document) - .set({ deletedAt: now }) + const memberCutoff = new Date(now.getTime() - staleMemberWindowMs(member.syncIntervalMinutes)) + const swept = await db.transaction(async (tx) => { + const [connector] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) .where( and( - inArray(document.id, documentIds), - eq(document.connectorId, member.connectorId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - hasNoObservation() + eq(knowledgeConnector.id, member.connectorId), + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.status, MEMBER_LOCKABLE_CONNECTOR_STATUSES), + ne(knowledgeConnector.memberSyncStatus, 'disabled'), + connectorIsLive() ) ) - .returning({ id: document.id }) - result.docsTombstoned += tombstoned.length - } + .for('share') + if (!connector) return null + const [stale] = await tx + .select({ id: knowledgeConnectorMember.id }) + .from(knowledgeConnectorMember) + .where(memberStillStale(member.id, memberCutoff)) + .for('update') + if (!stale) return null + + const removed = await tx + .delete(knowledgeDocumentObservation) + .where(eq(knowledgeDocumentObservation.memberId, member.id)) + .returning({ documentId: knowledgeDocumentObservation.documentId }) + const documentIds = removed.map((row) => row.documentId) + const rematerialized = await materializeDocumentAcls(member.connectorId, documentIds, tx) + const tombstoned = + documentIds.length === 0 + ? [] + : await tx + .update(document) + .set({ deletedAt: now }) + .where( + and( + inArray(document.id, documentIds), + eq(document.connectorId, member.connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + hasNoObservation() + ) + ) + .returning({ id: document.id }) + return { + observationsRemoved: documentIds.length, + documentsRematerialized: rematerialized, + docsTombstoned: tombstoned.length, + } + }) + if (!swept) continue + result.members += 1 + result.observationsRemoved += swept.observationsRemoved + result.documentsRematerialized += swept.documentsRematerialized + result.docsTombstoned += swept.docsTombstoned } return result } diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.ts index 2ce0d531d79..6d5e93a9d20 100644 --- a/apps/sim/lib/knowledge/connectors/member-provisioning.ts +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.ts @@ -12,6 +12,7 @@ import { normalizeEmail } from '@sim/utils/string' import { and, eq, inArray, isNull } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { + CredentialGroupEnrollmentError, createCredentialGroupInvitationLink, inviteCredentialGroupEnrollment, } from '@/lib/credential-groups/enrollments' @@ -174,9 +175,11 @@ export interface InviteWorkspaceMembersResult { /** * Invites every workspace member who has no enrollment in the group yet, so * joining the workspace is all a person has to do before connecting their - * account. An enrollment an admin revoked is left alone. Runs inside a member - * run: `beforeBatch` beats the run's lease between batches, and failures are - * logged per person rather than aborting the run. + * account. An enrollment an admin revoked is left alone — the invitation is + * issued with `reject`, so a revocation that lands after the enrollments were + * read is refused inside the issuing transaction rather than reactivated. + * Runs inside a member run: `beforeBatch` beats the run's lease between + * batches, and failures are logged per person rather than aborting the run. */ export async function inviteWorkspaceMembersToCredentialGroup(input: { workspaceId: string @@ -205,7 +208,8 @@ export async function inviteWorkspaceMembersToCredentialGroup(input: { input.credentialGroupId, undefined, undefined, - email + email, + 'reject' ) result.invited += 1 } catch (error) { @@ -326,7 +330,9 @@ export async function resolveViewerConnectorMemberships(input: { * on demand so a workspace member never has to find the invitation email. * Issued without an inviter — the person is inviting themselves — and refused * for an enrollment an admin revoked or an account whose email is unverified, - * which could connect but would never be granted a token. + * which could connect but would never be granted a token. The revocation is + * decided inside the issuing transaction (`reject`), so an admin who revokes + * between the read here and the issue is never overridden by a link. */ export async function createViewerConnectorEnrollmentLink(input: { userId: string @@ -346,27 +352,43 @@ export async function createViewerConnectorEnrollmentLink(input: { ) } const email = normalizeEmail(viewer.email) + const revoked = new OrchestrationError( + 'forbidden', + 'A workspace admin removed your access to this connector' + ) + if (await isEnrollmentRevoked(input.credentialGroupId, email)) throw revoked + try { + const { invitationLink } = await createCredentialGroupInvitationLink( + input.workspaceId, + input.credentialGroupId, + undefined, + email, + 'reject' + ) + return invitationLink + } catch (error) { + /** The issue refused a revocation that landed after the read above; report it as such. */ + if ( + error instanceof CredentialGroupEnrollmentError && + error.status === 409 && + (await isEnrollmentRevoked(input.credentialGroupId, email)) + ) { + throw revoked + } + throw error + } +} + +async function isEnrollmentRevoked(credentialGroupId: string, email: string): Promise { const [enrollment] = await db .select({ status: credentialGroupEnrollment.status }) .from(credentialGroupEnrollment) .where( and( - eq(credentialGroupEnrollment.credentialGroupId, input.credentialGroupId), + eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), eq(credentialGroupEnrollment.email, email) ) ) .limit(1) - if (enrollment?.status === 'revoked') { - throw new OrchestrationError( - 'forbidden', - 'A workspace admin removed your access to this connector' - ) - } - const { invitationLink } = await createCredentialGroupInvitationLink( - input.workspaceId, - input.credentialGroupId, - undefined, - email - ) - return invitationLink + return enrollment?.status === 'revoked' } diff --git a/apps/sim/lib/knowledge/connectors/member-queue.test.ts b/apps/sim/lib/knowledge/connectors/member-queue.test.ts index 024731eeb7e..68567ff7ee3 100644 --- a/apps/sim/lib/knowledge/connectors/member-queue.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-queue.test.ts @@ -4,14 +4,24 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockExecuteMemberSync, mockIsTriggerAvailable, mockTrigger, mockResolveRegion } = - vi.hoisted(() => ({ - mockExecuteMemberSync: vi.fn(), - mockIsTriggerAvailable: vi.fn(), - mockTrigger: vi.fn(), - mockResolveRegion: vi.fn(), - })) +const { + mockExecuteMemberSync, + mockIsTriggerAvailable, + mockTrigger, + mockResolveRegion, + mockResolveSystemBilling, +} = vi.hoisted(() => ({ + mockExecuteMemberSync: vi.fn(), + mockIsTriggerAvailable: vi.fn(), + mockTrigger: vi.fn(), + mockResolveRegion: vi.fn(), + mockResolveSystemBilling: vi.fn(), +})) +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + assertBillingAttributionSnapshot: (value: unknown) => value, + resolveSystemBillingAttribution: mockResolveSystemBilling, +})) vi.mock('@/lib/knowledge/connectors/member-sync-engine', () => ({ executeMemberSync: mockExecuteMemberSync, })) @@ -24,9 +34,11 @@ vi.mock('@trigger.dev/sdk', () => ({ })) vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: mockResolveRegion })) +import { eq, inArray } from 'drizzle-orm' import { assertMemberSyncPayload, dispatchMemberSync, + dispatchMemberSyncsForCredentialOption, MEMBER_SYNC_TASK_ID, } from '@/lib/knowledge/connectors/member-queue' @@ -186,6 +198,7 @@ describe('member sync queue', () => { queueTableRows(schemaMock.knowledgeConnector, [ { accessMode: 'members', + status: 'active', memberSyncStatus: 'idle', syncLockToken: 'content-run', archivedAt: null, @@ -209,5 +222,98 @@ describe('member sync queue', () => { dispatchMemberSync('c-1', { billingAttribution: BILLING, requestId: 'r-1' }) ).rejects.toThrow('does not match connector workspace ws-2') }) + + /** + * The guards above the CAS read the row once; a pause or a schedule change + * that lands after that read is only visible to the CAS itself. + */ + it('decides the connector status and the schedule inside the queue CAS', async () => { + const expected = new Date('2026-09-01T06:00:00Z') + queueTableRows(schemaMock.knowledgeConnector, [ + { ...CONNECTOR_ROW, nextMemberSyncAt: expected }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) + + await expect( + dispatchMemberSync('c-1', { + billingAttribution: BILLING, + requestId: 'r-1', + requireRunnable: true, + expectedNextMemberSyncAt: expected, + }) + ).resolves.toEqual({ queued: true }) + + expect(vi.mocked(inArray)).toHaveBeenCalledWith(schemaMock.knowledgeConnector.status, [ + 'active', + 'error', + ]) + expect(vi.mocked(eq)).toHaveBeenCalledWith( + schemaMock.knowledgeConnector.nextMemberSyncAt, + expected + ) + }) + + it.each([ + [ + 'a connector paused after the read', + { status: 'paused', nextMemberSyncAt: new Date('2026-09-01T06:00:00Z') }, + 'Connector is paused and is not synced', + ], + [ + 'a schedule that moved after the read', + { status: 'active', nextMemberSyncAt: new Date('2026-09-01T07:00:00Z') }, + 'The member sync schedule changed after this run was scheduled', + ], + ])('explains a queue entry refused for %s', async (_name, current, reason) => { + const expected = new Date('2026-09-01T06:00:00Z') + queueTableRows(schemaMock.knowledgeConnector, [ + { ...CONNECTOR_ROW, nextMemberSyncAt: expected }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + queueTableRows(schemaMock.knowledgeConnector, [ + { + accessMode: 'members', + memberSyncStatus: 'idle', + syncLockToken: null, + archivedAt: null, + deletedAt: null, + ...current, + }, + ]) + + await expect( + dispatchMemberSync('c-1', { + billingAttribution: BILLING, + requestId: 'r-1', + requireRunnable: true, + expectedNextMemberSyncAt: expected, + }) + ).resolves.toEqual({ queued: false, reason }) + expect(mockTrigger).not.toHaveBeenCalled() + }) + }) + + describe('dispatchMemberSyncsForCredentialOption', () => { + it('keeps dispatching the remaining connectors when one hand-off throws', async () => { + mockResolveSystemBilling.mockResolvedValue(BILLING) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }, { id: 'c-2' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ ...CONNECTOR_ROW, workspaceId: 'ws-2' }]) + queueTableRows(schemaMock.knowledgeConnector, [CONNECTOR_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-2' }]) + + await expect( + dispatchMemberSyncsForCredentialOption({ + workspaceId: 'ws-1', + credentialGroupOptionId: 'opt-1', + }) + ).resolves.toBeUndefined() + + expect(mockTrigger).toHaveBeenCalledOnce() + expect(mockTrigger).toHaveBeenCalledWith( + MEMBER_SYNC_TASK_ID, + expect.objectContaining({ connectorId: 'c-2' }), + expect.any(Object) + ) + }) }) }) diff --git a/apps/sim/lib/knowledge/connectors/member-queue.ts b/apps/sim/lib/knowledge/connectors/member-queue.ts index 4159ad498b3..ea04f785163 100644 --- a/apps/sim/lib/knowledge/connectors/member-queue.ts +++ b/apps/sim/lib/knowledge/connectors/member-queue.ts @@ -17,7 +17,10 @@ import { SYNC_DISPATCH_FAILED_ERROR, type SyncDispatchResult, } from '@/lib/knowledge/connectors/queue' -import { connectorIsLive } from '@/lib/knowledge/connectors/sync-lock' +import { + connectorIsLive, + MEMBER_LOCKABLE_CONNECTOR_STATUSES, +} from '@/lib/knowledge/connectors/sync-lock' import { isTriggerAvailable } from '@/lib/knowledge/documents/service' const logger = createLogger('ConnectorMemberSyncQueue') @@ -77,9 +80,16 @@ export function assertMemberSyncPayload(value: unknown): MemberSyncPayload { /** * Takes the member-sync queue entry, mirroring `markSyncPending` over the * member lease columns. Refuses while the content engine holds its lock, so - * the two engines can never be queued against one connector at once. + * the two engines can never be queued against one connector at once, and + * refuses a connector that is no longer runnable or whose schedule moved + * since the dispatch read it: the guards above this CAS cannot see a pause + * or a schedule change that lands after they ran, so the CAS is where those + * are decided. */ -async function markMemberSyncPending(connectorId: string): Promise { +async function markMemberSyncPending( + connectorId: string, + expectedNextMemberSyncAt: Date | undefined +): Promise { const dispatchToken = generateId() const now = new Date() const taken = await db @@ -94,7 +104,11 @@ async function markMemberSyncPending(connectorId: string): Promise 0 ? dispatchToken : null } -async function describeUnacceptedMemberSync(connectorId: string): Promise { +async function describeUnacceptedMemberSync( + connectorId: string, + expectedNextMemberSyncAt: Date | undefined +): Promise { const [row] = await db .select({ accessMode: knowledgeConnector.accessMode, + status: knowledgeConnector.status, memberSyncStatus: knowledgeConnector.memberSyncStatus, + nextMemberSyncAt: knowledgeConnector.nextMemberSyncAt, syncLockToken: knowledgeConnector.syncLockToken, archivedAt: knowledgeConnector.archivedAt, deletedAt: knowledgeConnector.deletedAt, @@ -119,8 +138,17 @@ async function describeUnacceptedMemberSync(connectorId: string): Promise status === row.status)) { + return `Connector is ${row.status} and is not synced` + } if (row.syncLockToken) return 'A workspace sync is still running for this connector' if (row.memberSyncStatus === 'disabled') return 'Member sync is disabled for this connector' + if ( + expectedNextMemberSyncAt && + row.nextMemberSyncAt?.getTime() !== expectedNextMemberSyncAt.getTime() + ) { + return 'The member sync schedule changed after this run was scheduled' + } return 'A member sync is already queued or running for this connector' } @@ -266,9 +294,9 @@ export async function dispatchMemberSync( ) } - const dispatchToken = await markMemberSyncPending(connectorId) + const dispatchToken = await markMemberSyncPending(connectorId, options.expectedNextMemberSyncAt) if (!dispatchToken) { - const reason = await describeUnacceptedMemberSync(connectorId) + const reason = await describeUnacceptedMemberSync(connectorId, options.expectedNextMemberSyncAt) logger.info('Skipping member sync dispatch: connector is not accepting a queued run', { connectorId, reason, @@ -323,8 +351,9 @@ export async function dispatchMemberSync( /** * Queues a member run for every connector that crawls through the option a * member just connected, so their documents arrive within minutes rather - * than at the next scheduled run. Best effort: a refused dispatch is logged - * and the schedule catches up. + * than at the next scheduled run. Best effort: a refused or failed dispatch is + * logged, the remaining connectors are still queued, and the schedule catches + * up on whichever was not. */ export async function dispatchMemberSyncsForCredentialOption(input: { workspaceId: string @@ -348,11 +377,18 @@ export async function dispatchMemberSyncsForCredentialOption(input: { if (connectors.length === 0) return const billingAttribution = await resolveSystemBillingAttribution(input.workspaceId) for (const connector of connectors) { - const dispatch = await dispatchMemberSync(connector.id, { billingAttribution }) - if (!dispatch.queued) { - logger.info('Member sync after a member connected was not queued', { + try { + const dispatch = await dispatchMemberSync(connector.id, { billingAttribution }) + if (!dispatch.queued) { + logger.info('Member sync after a member connected was not queued', { + connectorId: connector.id, + reason: dispatch.reason, + }) + } + } catch (error) { + logger.warn('Member sync after a member connected could not be dispatched', { connectorId: connector.id, - reason: dispatch.reason, + error: toError(error).message, }) } } diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index b26cc7e853a..3d05da63d79 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -54,6 +54,7 @@ import { import { createMemberSyncLease, holdsMemberSyncLockToken, + MEMBER_LOCKABLE_CONNECTOR_STATUSES, SyncLockLostException, stillHoldsMemberSyncLock, } from '@/lib/knowledge/connectors/sync-lock' @@ -438,6 +439,7 @@ async function acquireMemberSyncLock( and( eq(knowledgeConnector.id, connectorId), eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.status, MEMBER_LOCKABLE_CONNECTOR_STATUSES), inArray(knowledgeConnector.memberSyncStatus, ['idle', 'pending', 'error']), ...(dispatchToken ? [eq(knowledgeConnector.memberSyncLockToken, dispatchToken)] : []), isNull(knowledgeConnector.syncLockToken), @@ -851,6 +853,13 @@ async function listForMember(input: { full || !member.memberSyncedThrough ? undefined : new Date(member.memberSyncedThrough.getTime() - INCREMENTAL_OVERLAP_MS) + /** + * A listing pass has no cursor to resume from, so a page cap would relist + * the same first pages every run and never reach the documents behind + * them. The run's deadline is its only bound: a member the budget cuts off + * is re-armed at once (`resumable`), and one no run can finish alone + * backs off (`exhaustedRunAlone`) instead of being silently truncated. + */ const listing = await runListingPass({ connectorId: run.connectorId, connectorConfig, @@ -860,7 +869,7 @@ async function listForMember(input: { beforePage: run.lease.beatIfDue, getAccessToken: () => input.tokens.get(member.id), deadlineAt: run.deadlineAt, - maxPages: MEMBER_SYNC_MAX_PAGES_PER_MEMBER, + maxPages: Number.POSITIVE_INFINITY, }) const complete = listing.exhausted && @@ -1285,6 +1294,7 @@ export async function executeMemberSync( if (!connector) { const [current] = await db .select({ + status: knowledgeConnector.status, memberSyncStatus: knowledgeConnector.memberSyncStatus, memberSyncLockToken: knowledgeConnector.memberSyncLockToken, syncLockToken: knowledgeConnector.syncLockToken, @@ -1292,8 +1302,15 @@ export async function executeMemberSync( .from(knowledgeConnector) .where(eq(knowledgeConnector.id, connectorId)) .limit(1) - if (current?.memberSyncStatus === 'disabled' || current?.syncLockToken) { - logger.info('Connector is not accepting member syncs, skipping', { connectorId }) + if ( + current?.memberSyncStatus === 'disabled' || + current?.syncLockToken || + (current && !MEMBER_LOCKABLE_CONNECTOR_STATUSES.some((status) => status === current.status)) + ) { + logger.info('Connector is not accepting member syncs, skipping', { + connectorId, + status: current.status, + }) return skipped(result, 'connector_not_syncable') } if (options.dispatchToken && current?.memberSyncLockToken !== options.dispatchToken) { diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts index a1ad969ce84..ed7c1a21729 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts @@ -9,6 +9,7 @@ const NOW = new Date('2026-09-01T12:00:00Z') describe('resolveSourceModifiedAt', () => { it.each([ ['modifiedTime', '2026-08-20T12:00:00Z'], + ['updatedTime', '2026-08-20T12:00:00Z'], ['lastModified', '2026-08-20T12:00:00.000Z'], ['lastModifiedDateTime', '2026-08-20T12:00:00Z'], ['updatedAt', '2026-08-20T12:00:00Z'], diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.ts index 868d809b217..f39ac86dc61 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.ts @@ -5,6 +5,7 @@ */ const SOURCE_MODIFIED_AT_KEYS = [ 'modifiedTime', + 'updatedTime', 'lastModified', 'lastModifiedDateTime', 'modifiedAt', diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index 1ab922bbc5c..61ac44d2c7f 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -713,31 +713,82 @@ describe('connector content replacement processing state', () => { }) }) +/** The run's lease as the persistence writes see it; the condition itself is opaque to the chain mock. */ +const lease = { stillHeld: () => ({ type: 'lease' }) as never } + describe('persistSkippedDocuments', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() }) + /** + * The heartbeat before a batch only proves the lease was held then; the + * write itself re-proves it inside its transaction, so a run reclaimed in + * between lands nothing over its replacement's. + */ + it('refuses to write once the run no longer holds its lease', async () => { + const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') + const { SyncLockLostException } = await import('@/lib/knowledge/connectors/sync-lock') + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, []) + + await expect( + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'empty-hash', + skippedReason: 'Document contains no extractable text', + }, + }, + ], + undefined, + 'workspace', + lease + ) + ).rejects.toBeInstanceOf(SyncLockLostException) + + expect(dbChainMockFns.for).toHaveBeenCalledWith('share') + expect(dbChainMockFns.values).not.toHaveBeenCalled() + }) + it('persists a new skipped document without dispatching processing', async () => { const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) await expect( - persistSkippedDocuments('kb-1', 'connector-1', 'no-tags', [ - { - type: 'skip', - extDoc: { - externalId: 'external-1', - title: 'Empty document', - content: '', - mimeType: 'text/plain', - contentHash: 'empty-hash', - skippedReason: 'Document contains no extractable text', - skippedExistingDisposition: 'replace', + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'empty-hash', + skippedReason: 'Document contains no extractable text', + skippedExistingDisposition: 'replace', + }, }, - }, - ]) + ], + undefined, + 'workspace', + lease + ) ).resolves.toBe(1) expect(dbChainMockFns.values).toHaveBeenCalledWith([ @@ -756,25 +807,34 @@ describe('persistSkippedDocuments', () => { const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') const oldFileUrl = '/api/files/serve/kb/old-document.txt?context=knowledge-base' queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) queueTableRows(schemaMock.document, [{ fileUrl: oldFileUrl }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'doc-1' }]) await expect( - persistSkippedDocuments('kb-1', 'connector-1', 'no-tags', [ - { - type: 'skip', - existingId: 'doc-1', - extDoc: { - externalId: 'external-1', - title: 'Empty document', - content: '', - mimeType: 'text/plain', - contentHash: 'new-empty-hash', - skippedReason: 'Document contains no extractable text', - skippedExistingDisposition: 'replace', + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + existingId: 'doc-1', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'new-empty-hash', + skippedReason: 'Document contains no extractable text', + skippedExistingDisposition: 'replace', + }, }, - }, - ]) + ], + undefined, + 'workspace', + lease + ) ).resolves.toBe(1) expect(dbChainMockFns.set).toHaveBeenCalledWith( @@ -803,24 +863,33 @@ describe('persistSkippedDocuments', () => { it('does not delete old storage when the authoritative replacement fails', async () => { const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) queueTableRows(schemaMock.document, []) await expect( - persistSkippedDocuments('kb-1', 'connector-1', 'no-tags', [ - { - type: 'skip', - existingId: 'missing-doc', - extDoc: { - externalId: 'external-1', - title: 'Empty document', - content: '', - mimeType: 'text/plain', - contentHash: 'new-empty-hash', - skippedReason: 'Document contains no extractable text', - skippedExistingDisposition: 'replace', + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + existingId: 'missing-doc', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'new-empty-hash', + skippedReason: 'Document contains no extractable text', + skippedExistingDisposition: 'replace', + }, }, - }, - ]) + ], + undefined, + 'workspace', + lease + ) ).rejects.toThrow('Document missing-doc is no longer active') expect(mockDeleteFile).not.toHaveBeenCalled() @@ -840,16 +909,22 @@ describe('persistSkippedRetryHashes', () => { '@/lib/knowledge/connectors/sync-persistence' ) queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'doc-1' }]) await expect( - persistSkippedRetryHashes('kb-1', 'connector-1', [ - { - existingId: 'doc-1', - externalId: 'page-1', - contentHash: 'notion:retry:v1:page-1', - }, - ]) + persistSkippedRetryHashes( + 'kb-1', + 'connector-1', + [ + { + existingId: 'doc-1', + externalId: 'page-1', + contentHash: 'notion:retry:v1:page-1', + }, + ], + lease + ) ).resolves.toEqual([]) expect(dbChainMockFns.set).toHaveBeenCalledOnce() @@ -874,21 +949,27 @@ describe('persistSkippedRetryHashes', () => { '@/lib/knowledge/connectors/sync-persistence' ) queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'live-doc' }]).mockResolvedValueOnce([]) await expect( - persistSkippedRetryHashes('kb-1', 'connector-1', [ - { - existingId: 'live-doc', - externalId: 'live-page', - contentHash: 'notion:retry:v1:live-page', - }, - { - existingId: 'detached-doc', - externalId: 'detached-page', - contentHash: 'notion:retry:v1:detached-page', - }, - ]) + persistSkippedRetryHashes( + 'kb-1', + 'connector-1', + [ + { + existingId: 'live-doc', + externalId: 'live-page', + contentHash: 'notion:retry:v1:live-page', + }, + { + existingId: 'detached-doc', + externalId: 'detached-page', + contentHash: 'notion:retry:v1:detached-page', + }, + ], + lease + ) ).resolves.toEqual(['detached-page']) expect(dbChainMockFns.set).toHaveBeenCalledWith({ diff --git a/apps/sim/lib/knowledge/connectors/sync-limits.ts b/apps/sim/lib/knowledge/connectors/sync-limits.ts index 23f23a96c17..707e85e64a7 100644 --- a/apps/sim/lib/knowledge/connectors/sync-limits.ts +++ b/apps/sim/lib/knowledge/connectors/sync-limits.ts @@ -113,10 +113,17 @@ export const MEMBER_SYNC_SOFT_BUDGET_SECONDS = 2700 export const MEMBER_SYNC_STALE_LOCK_TTL_MS = MEMBER_SYNC_MAX_DURATION_SECONDS * 2 * 1000 /** - * Pages one member's listing may consume in one run. A member whose corpus - * exceeds it is recorded as incomplete — additions are kept, removals are - * withheld — and retried without back-off, so a single huge member can never - * monopolise a run or silently lose access. + * Pages one member's change-feed pass may consume in one run. The feed's + * cursor is stored past every page read, so a pass the cap stops is recorded + * as incomplete — additions are kept, removals are withheld — and the next + * run continues from where it left off; a single huge feed can never + * monopolise a run or lose a change. + * + * Deliberately not applied to a listing pass, which has no cursor to resume + * from: capping it would relist the same first pages every run and never + * grant access to the documents behind them. A listing is bounded by the run + * deadline instead, and a member no run can finish alone backs off through + * `exhaustedRunAlone`. */ export const MEMBER_SYNC_MAX_PAGES_PER_MEMBER = 200 diff --git a/apps/sim/lib/knowledge/connectors/sync-lock.ts b/apps/sim/lib/knowledge/connectors/sync-lock.ts index b920124c0ee..7f71a93c376 100644 --- a/apps/sim/lib/knowledge/connectors/sync-lock.ts +++ b/apps/sim/lib/knowledge/connectors/sync-lock.ts @@ -205,6 +205,16 @@ export function createContentSyncLease(connectorId: string, syncLogId: string): } } +/** + * The connector statuses a members-mode run may be queued from or take its + * lease in. The same reasoning as {@link LOCKABLE_CONNECTOR_STATUSES}: the + * queue outlives the decision to sync, so a connector paused after its member + * run was dispatched still had a task in flight, and a lease CAS that ignored + * `status` let that task crawl a paused connector. `pending` is absent because + * the member lease never coexists with the content queue's entry. + */ +export const MEMBER_LOCKABLE_CONNECTOR_STATUSES = ['active', 'error'] as const + /** * Ownership only, for the members-mode lease: this run still holds the * member-sync lock, whether or not the connector is still live. The member diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 11813f3a1d6..b7532ac49aa 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -9,6 +9,7 @@ import type { DbOrTx } from '@/lib/db/types' import { textArrayLiteral } from '@/lib/knowledge/access/predicate' import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' +import { SyncLockLostException, type SyncRunLease } from '@/lib/knowledge/connectors/sync-lock' import type { DocumentData } from '@/lib/knowledge/documents/service' import { StorageService } from '@/lib/uploads' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' @@ -148,6 +149,30 @@ async function isKnowledgeBaseActiveInTx( return rows.length > 0 } +/** The lease a document write proves before it lands, as the run that makes it holds it. */ +export type SyncWriteLease = Pick + +/** + * Proves, inside the write's own transaction, that the run still owns the + * connector. A heartbeat taken before the batch only says the lease was held + * then; the hydration and storage work between it and the row write can + * outlast the lease. The share lock keeps the scheduler's reclaim from landing + * until this write commits, and a row that no longer matches aborts the write + * instead of landing stale content over the replacement run's. + */ +async function assertSyncLeaseHeldInTx( + tx: KnowledgeBaseLockingTx, + connectorId: string, + lease: SyncWriteLease +): Promise { + const [held] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(lease.stillHeld()) + .for('share') + if (!held) throw new SyncLockLostException(connectorId) +} + /** * Resolves tag values from connector metadata using the connector's mapTags function. * Translates semantic keys returned by mapTags to actual DB slots using the @@ -256,7 +281,8 @@ export async function persistSkippedDocuments( extDoc: ExternalDocument }>, sourceConfig: Record | undefined, - access: SyncDocumentAccess + access: SyncDocumentAccess, + lease: SyncWriteLease ): Promise { if (skipOps.length === 0) { return 0 @@ -283,6 +309,7 @@ export async function persistSkippedDocuments( if (!isActive) { throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) if (inserts.length > 0) { await tx.insert(document).values(inserts) @@ -371,7 +398,8 @@ export async function persistSkippedDocuments( export async function persistSkippedRetryHashes( knowledgeBaseId: string, connectorId: string, - updates: Array<{ existingId: string; externalId: string; contentHash: string }> + updates: Array<{ existingId: string; externalId: string; contentHash: string }>, + lease: SyncWriteLease ): Promise { if (updates.length === 0) return [] @@ -382,6 +410,7 @@ export async function persistSkippedRetryHashes( if (!isActive) { throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) for (const update of updates) { const persisted = await tx @@ -409,7 +438,8 @@ export async function addDocument( extDoc: ExternalDocument, kbOwner: KnowledgeBaseOwner, sourceConfig: Record | undefined, - access: SyncDocumentAccess + access: SyncDocumentAccess, + lease: SyncWriteLease ): Promise { const documentId = generateId() const artifact = connectorStoredArtifact(extDoc) @@ -437,6 +467,7 @@ export async function addDocument( if (!isActive) { throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) await tx.insert(document).values({ id: documentId, @@ -507,7 +538,8 @@ export async function updateDocument( extDoc: ExternalDocument, kbOwner: KnowledgeBaseOwner, sourceConfig: Record | undefined, - access: SyncDocumentAccess + access: SyncDocumentAccess, + lease: SyncWriteLease ): Promise { const existingRows = await db .select({ fileUrl: document.fileUrl }) @@ -543,6 +575,7 @@ export async function updateDocument( if (!isActive) { throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) await tx .update(document) diff --git a/apps/sim/lib/knowledge/connectors/sync-primitives.ts b/apps/sim/lib/knowledge/connectors/sync-primitives.ts index dabd90c9031..83ca0b3d566 100644 --- a/apps/sim/lib/knowledge/connectors/sync-primitives.ts +++ b/apps/sim/lib/knowledge/connectors/sync-primitives.ts @@ -1495,7 +1495,7 @@ export interface ProcessDocOpsInput { forceRehydrate: boolean state: SyncRunState hydration: DocOpHydration - lease: Pick + lease: Pick /** Who may read the documents this pass writes. */ documentAccess: SyncDocumentAccess } @@ -1504,7 +1504,8 @@ export interface ProcessDocOpsInput { * Hydrates, stores, and dispatches the pending operations in batches bounded * by both count and in-flight content bytes. Every failure is counted on the * run state rather than thrown, except a provider rate limit, which ends the - * run so the connector backs off. + * run so the connector backs off, and a lost lease, which ends it so no + * further write lands beside the replacement run's. */ export async function processDocOps(input: ProcessDocOpsInput): Promise { const { @@ -1645,7 +1646,8 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { /** * Hydration above may have outlasted the lease. Nothing from this batch is - * written until the run proves it still owns the connector, so a run that + * written until the run proves it still owns the connector, and every + * write below proves it again inside its own transaction, so a run that * was replaced meanwhile cannot land stale content or queue processing * over the replacement's. */ @@ -1656,7 +1658,8 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { const missedExternalIds = await persistSkippedRetryHashes( connector.knowledgeBaseId, connectorId, - skippedRetryHashUpdates + skippedRetryHashUpdates, + input.lease ) if (missedExternalIds.length > 0) { logger.warn('Skipped retry hashes were not persisted for detached documents', { @@ -1682,10 +1685,12 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { connector.connectorType, skipOps, sourceConfig, - documentAccess + documentAccess, + input.lease ) result.docsSkipped += recorded } catch (error) { + if (error instanceof SyncLockLostException) throw error /** * The source items were intentionally skipped, but failing to persist their visible * failed rows is an actual sync failure. @@ -1714,7 +1719,8 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { op.extDoc, kbOwner, sourceConfig, - documentAccess + documentAccess, + input.lease ) } return updateDocument( @@ -1725,11 +1731,18 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { op.extDoc, kbOwner, sourceConfig, - documentAccess + documentAccess, + input.lease ) }) ) + const leaseLost = settled.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === 'rejected' && outcome.reason instanceof SyncLockLostException + ) + if (leaseLost) throw leaseLost.reason + const batchDocs: DocumentData[] = [] for (let j = 0; j < settled.length; j++) { const outcome = settled[j] diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 9cedc591da5..75566c4ae4b 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -3669,10 +3669,17 @@ async function excludeConnectorDocuments( return updated.length } +/** + * Deletes documents by their lifecycle: connector-owned ones are excluded, + * uploads are hard deleted. `access`, when given, is applied to the selection + * and to every write, so a document the caller stopped being able to read + * after they looked it up is left alone rather than deleted on a stale view. + */ async function deleteDocumentsByLifecyclePolicy( documentIds: string[], requestId: string, - expectedKnowledgeBaseId?: string + expectedKnowledgeBaseId?: string, + access?: KnowledgeAccessScope ): Promise { const ids = [...new Set(documentIds)] if (ids.length === 0) { @@ -3692,7 +3699,8 @@ async function deleteDocumentsByLifecyclePolicy( eq(document.knowledgeBaseId, expectedKnowledgeBaseId), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + access ? knowledgeAccessCondition(access) : undefined ) : inArray(document.id, ids) ) @@ -3702,9 +3710,21 @@ async function deleteDocumentsByLifecyclePolicy( const [excludedCount, hardDeletedCount] = await Promise.all([ expectedKnowledgeBaseId - ? excludeConnectorKnowledgeDocuments(expectedKnowledgeBaseId, connectorBackedIds, requestId) + ? excludeConnectorKnowledgeDocuments( + expectedKnowledgeBaseId, + connectorBackedIds, + requestId, + access + ) : excludeConnectorDocuments(connectorBackedIds, requestId), - hardDeleteDocuments(hardDeleteIds, requestId, undefined, expectedKnowledgeBaseId), + hardDeleteDocuments( + hardDeleteIds, + requestId, + undefined, + expectedKnowledgeBaseId, + undefined, + access + ), ]) return excludedCount + hardDeletedCount @@ -3756,7 +3776,9 @@ export async function hardDeleteDocuments( */ expectedConnectorId?: string, expectedKnowledgeBaseId?: string, - connectorSyncGuard?: ConnectorSyncDeletionGuard + connectorSyncGuard?: ConnectorSyncDeletionGuard, + /** When provided, only documents the caller may currently read are deleted, re-verified at the delete itself. */ + access?: KnowledgeAccessScope ): Promise { const ids = [...new Set(documentIds)] if (ids.length === 0) { @@ -3770,7 +3792,8 @@ export async function hardDeleteDocuments( requestId, expectedConnectorId, expectedKnowledgeBaseId, - connectorSyncGuard + connectorSyncGuard, + access ) } return deletedCount @@ -3785,13 +3808,15 @@ async function hardDeleteDocumentBatch( requestId: string, expectedConnectorId?: string, expectedKnowledgeBaseId?: string, - connectorSyncGuard?: ConnectorSyncDeletionGuard + connectorSyncGuard?: ConnectorSyncDeletionGuard, + access?: KnowledgeAccessScope ): Promise { const ids = [...new Set(documentIds)] const scopedConnectorId = connectorSyncGuard?.connectorId ?? expectedConnectorId const scopedKnowledgeBaseId = connectorSyncGuard?.knowledgeBaseId ?? expectedKnowledgeBaseId const requireEligibleDocument = Boolean(expectedKnowledgeBaseId || connectorSyncGuard) const requireVisibleDocument = Boolean(expectedKnowledgeBaseId && !connectorSyncGuard) + const accessCondition = access ? knowledgeAccessCondition(access) : undefined const documentsToDelete = await db .select({ id: document.id, @@ -3812,7 +3837,8 @@ async function hardDeleteDocumentBatch( scopedKnowledgeBaseId ? eq(document.knowledgeBaseId, scopedKnowledgeBaseId) : undefined, requireEligibleDocument ? eq(document.userExcluded, false) : undefined, requireEligibleDocument ? isNull(document.archivedAt) : undefined, - requireVisibleDocument ? isNull(document.deletedAt) : undefined + requireVisibleDocument ? isNull(document.deletedAt) : undefined, + accessCondition ) ) @@ -3924,7 +3950,7 @@ async function hardDeleteDocumentBatch( * ID set rather than the stale `existingIds`. */ const stillTargetedIds = - scopedConnectorId || scopedKnowledgeBaseId + scopedConnectorId || scopedKnowledgeBaseId || accessCondition ? ( await tx .select({ id: document.id }) @@ -3938,7 +3964,8 @@ async function hardDeleteDocumentBatch( : undefined, requireEligibleDocument ? eq(document.userExcluded, false) : undefined, requireEligibleDocument ? isNull(document.archivedAt) : undefined, - requireVisibleDocument ? isNull(document.deletedAt) : undefined + requireVisibleDocument ? isNull(document.deletedAt) : undefined, + accessCondition ) ) .orderBy(asc(document.id)) @@ -4011,7 +4038,11 @@ export async function deleteDocument( } } -/** Deletes one currently visible document within its canonical knowledge base. */ +/** + * Deletes one currently visible document within its canonical knowledge base. + * The caller's access is re-applied at the delete itself, so a token member + * sync revokes between the lookup and the write cannot still delete. + */ export async function deleteKnowledgeDocumentInKnowledgeBase( knowledgeBaseId: string, documentId: string, @@ -4020,14 +4051,20 @@ export async function deleteKnowledgeDocumentInKnowledgeBase( ): Promise { const current = await getKnowledgeDocument(knowledgeBaseId, documentId, access) if (!current) throw new OrchestrationError('not_found', 'Document not found') - const affected = await deleteDocumentsByLifecyclePolicy([documentId], requestId, knowledgeBaseId) + const affected = await deleteDocumentsByLifecyclePolicy( + [documentId], + requestId, + knowledgeBaseId, + access + ) if (affected !== 1) throw new OrchestrationError('not_found', 'Document not found') } async function excludeConnectorKnowledgeDocuments( knowledgeBaseId: string, documentIds: string[], - requestId: string + requestId: string, + access?: KnowledgeAccessScope ): Promise { if (documentIds.length === 0) return 0 const updated = await db @@ -4040,7 +4077,8 @@ async function excludeConnectorKnowledgeDocuments( isNotNull(document.connectorId), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + access ? knowledgeAccessCondition(access) : undefined ) ) .returning({ id: document.id }) From 884681bb1627535e96d6df3acade1f5f1f914e55 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 02:07:05 -0700 Subject: [PATCH 40/76] fix(knowledge): resolve Sim Search review findings - Sim Search connect: the first connect of a source requires a workspace admin and refuses everyone else with the way forward; per-member availability is checked before anything is created; creation runs under a per-workspace advisory lock with a re-check; the route forwards the source's setup fields - Connector access switch: workspace mode requires credentialId in the contract and the credential must belong to the connector's provider - Base64 hydration reads knowledge-base files as the run's principal - Search surfaces gate on the workspace's per-member access flag; search results skip legacy personal bases, surface list errors, and report indexing across every per-member connector - Enrollment hook: explicit copy for failed/disabled member sync, awaited connections are dropped once connected, each path clears the other's error; Search mode keeps its query in an existing chat and keeps its context chips diffable; citation labels are Markdown-escaped; an access switch also refetches per-document caches --- .../api/knowledge/sim-search/connect/route.ts | 6 +- .../app/api/workflows/[id]/execute/route.ts | 3 + .../knowledge-search-results.tsx | 42 ++- .../components/chat-content/chat-content.tsx | 11 +- .../mothership-chat/mothership-chat.tsx | 8 + .../search-sources/search-sources.tsx | 34 ++- .../home/components/user-input/user-input.tsx | 7 +- .../app/workspace/[workspaceId]/home/home.tsx | 2 + .../[workspaceId]/search/search.test.tsx | 21 +- .../workspace/[workspaceId]/search/search.tsx | 44 +++- apps/sim/executor/execution/block-executor.ts | 1 + .../executor/handlers/agent/agent-handler.ts | 1 + .../resolvers/reference-async.server.ts | 1 + .../executor/variables/resolvers/reference.ts | 3 + apps/sim/hooks/queries/kb/connectors.ts | 4 +- apps/sim/hooks/use-member-enrollment.ts | 35 ++- .../contracts/knowledge/connectors.test.ts | 6 + .../lib/api/contracts/knowledge/connectors.ts | 7 + .../knowledge/application/connector-access.ts | 26 +- .../lib/knowledge/application/connectors.ts | 20 ++ .../lib/knowledge/application/operations.ts | 7 +- .../knowledge/application/sim-search.test.ts | 246 ++++++++++++++++++ .../lib/knowledge/application/sim-search.ts | 147 ++++++++--- .../utils/user-file-base64.server.test.ts | 90 +++++-- .../uploads/utils/user-file-base64.server.ts | 10 + .../lib/workflows/executor/execute-service.ts | 2 + apps/sim/lib/workflows/streaming/streaming.ts | 7 + 27 files changed, 675 insertions(+), 116 deletions(-) create mode 100644 apps/sim/lib/knowledge/application/sim-search.test.ts diff --git a/apps/sim/app/api/knowledge/sim-search/connect/route.ts b/apps/sim/app/api/knowledge/sim-search/connect/route.ts index cc842ee90a0..c13ebd49c16 100644 --- a/apps/sim/app/api/knowledge/sim-search/connect/route.ts +++ b/apps/sim/app/api/knowledge/sim-search/connect/route.ts @@ -14,7 +14,11 @@ export const POST = defineInternalJsonRoute({ operation: knowledgeOperations.simSearchConnect, rateLimit: internalRateLimits.none({ reason: 'One click per source; mints a single-use link' }), errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ body }) => ({ workspaceId: body.workspaceId, connectorType: body.connectorType }), + mapInput: ({ body }) => ({ + workspaceId: body.workspaceId, + connectorType: body.connectorType, + sourceConfig: body.sourceConfig, + }), useCase: connectSimSearchConnector, present: (result) => ({ success: true as const, data: result }), }) diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index 53e07eb7e1d..2d99b7a7a7f 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -1497,6 +1497,7 @@ async function handleExecutePost( fileKeys: outputFileKeys, allowLargeValueWorkflowScope, userId: actorUserId, + principal: executionPrincipal, maxBytes: base64MaxBytes, preserveLargeValueMetadata: true, })) as NormalizedBlockOutput) @@ -1701,6 +1702,7 @@ async function handleExecutePost( workspaceId, workflowId, userId: actorUserId, + principal: executionPrincipal, allowLargeValueWorkflowScope, requestSignal: req.signal, requestHeaders: req.headers, @@ -2303,6 +2305,7 @@ async function handleExecutePost( fileKeys: outputFileKeys, allowLargeValueWorkflowScope, userId: actorUserId, + principal: executionPrincipal, maxBytes: base64MaxBytes, preserveLargeValueMetadata: true, }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 5f5b608dbcc..2687438dbe9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -5,10 +5,7 @@ import { Button, Chip } from '@sim/emcn' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' -import { - isIndexing, - simSearchConnectionsByType, -} from '@/app/workspace/[workspaceId]/home/components/search-sources' +import { isIndexing } from '@/app/workspace/[workspaceId]/home/components/search-sources' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { useWorkspaceMemberConnectors } from '@/hooks/queries/kb/connectors' import { useKnowledgeBasesQuery, useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' @@ -112,10 +109,22 @@ export function KnowledgeSearchResults({ onSummarize, onAnswer, }: KnowledgeSearchResultsProps) { - const { data: knowledgeBases = [], isPending: basesPending } = useKnowledgeBasesQuery(workspaceId) + const { + data: knowledgeBases = [], + isPending: basesPending, + error: basesError, + } = useKnowledgeBasesQuery(workspaceId) + /** + * The list also carries the viewer's legacy personal bases, which have no + * workspace; a search names one workspace and refuses a base outside it. + */ const knowledgeBaseIds = useMemo( - () => knowledgeBases.slice(0, MAX_SEARCHED_KNOWLEDGE_BASES).map((kb) => kb.id), - [knowledgeBases] + () => + knowledgeBases + .filter((kb) => kb.workspaceId === workspaceId) + .slice(0, MAX_SEARCHED_KNOWLEDGE_BASES) + .map((kb) => kb.id), + [knowledgeBases, workspaceId] ) const { data: results, @@ -124,11 +133,15 @@ export function KnowledgeSearchResults({ error, } = useWorkspaceKnowledgeSearch(workspaceId, knowledgeBaseIds, query) const { data: memberConnectors = [] } = useWorkspaceMemberConnectors(workspaceId) + /** Every per-member connector still indexing for the viewer, in any base the search spans. */ const indexing = useMemo( - () => - [...simSearchConnectionsByType(memberConnectors).values()] - .filter(isIndexing) - .map((connection) => connectorName(connection.connectorType)), + () => [ + ...new Set( + memberConnectors + .filter(isIndexing) + .map((connection) => connectorName(connection.connectorType)) + ), + ], [memberConnectors] ) const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) @@ -153,6 +166,10 @@ export function KnowledgeSearchResults({ }) }, [documents, showFilters, sourceFilter, updatedFilter]) + const failure = basesError ?? error + if (failure) { + return

{failure.message}

+ } if (!basesPending && knowledgeBaseIds.length === 0) { return (

@@ -160,9 +177,6 @@ export function KnowledgeSearchResults({

) } - if (error) { - return

{error.message}

- } if (isPending || (isFetching && !results)) { return

Searching…

} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx index 03ccd3c54e0..d75aedac99f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx @@ -154,6 +154,15 @@ function SourceReference({ index, children }: SourceReferenceProps) { return } +/** + * A source's name as a Markdown link label. A site name or knowledge-base + * name is free text: an unescaped `]` would end the label early and a `*` or + * `_` would style it, so the delimiters are backslash-escaped. + */ +function escapeLinkLabel(label: string): string { + return label.replace(/[\\[\]*_`<>]/g, '\\$&') +} + function appendInlineReferenceMarkdown( currentMarkdown: string, referenceMarkdown: string, @@ -659,7 +668,7 @@ function ChatContentInner({ if (pendingMarkdown && !/\s$/.test(pendingMarkdown)) pendingMarkdown += ' ' pendingMarkdown = appendInlineReferenceMarkdown( pendingMarkdown, - `[${sourceLabel(s.data)}](<${SOURCE_LINK_PREFIX}${sourceIndex++}>)`, + `[${escapeLinkLabel(sourceLabel(s.data))}](<${SOURCE_LINK_PREFIX}${sourceIndex++}>)`, nextSegment ) } else if (s.type === 'thinking') { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx index 9449cb13d7d..5f4ece490d4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx @@ -70,6 +70,10 @@ interface MothershipChatProps { fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[] ) => void + /** Off in Search mode, where the query stays put so the person can refine it. */ + clearOnSubmit?: boolean + /** Fires when the composer's text goes from something to nothing. */ + onCleared?: () => void onStopGeneration: () => void messageQueue: QueuedMessage[] editingQueuedId: string | null @@ -320,6 +324,8 @@ export function MothershipChat({ isReconnecting = false, isLoading = false, onSubmit, + clearOnSubmit, + onCleared, onStopGeneration, messageQueue, editingQueuedId, @@ -838,6 +844,8 @@ export function MothershipChat({ key={draftScopeKey} ref={userInputRef} onSubmit={onSubmit} + clearOnSubmit={clearOnSubmit} + onCleared={onCleared} isSending={isStreamActive} onStopGeneration={onStopGeneration} isInitialView={false} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx index 658f2e101c4..7446053dd75 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx @@ -12,6 +12,7 @@ import { SIM_SEARCH_KNOWLEDGE_BASE_NAME, } from '@/lib/sim-search/connectors' import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' +import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import { BrandIcon } from '@/blocks/brand-icon' import { memberConnectorKeys, @@ -22,6 +23,7 @@ import { CONNECTABLE_MEMBERSHIPS, useMemberEnrollment } from '@/hooks/use-member import { usePermissionConfig } from '@/hooks/use-permission-config' const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] +const MEMBER_ACCESS_UNAVAILABLE = 'Per-member access is not available in this workspace' /** The sources a person can connect themselves, alphabetical. */ const PERSONAL_SEARCH_CONNECTORS = SEARCH_CONNECTORS.filter((connector) => @@ -81,7 +83,8 @@ function sourceState( interface SourceChipProps { connector: SearchConnector connection: WorkspaceMemberConnector | undefined - unavailable: boolean + /** Why the source cannot be connected here, shown as the chip's title; null when it can. */ + unavailableReason: string | null waiting: boolean disabled: boolean onConnect: () => void @@ -90,22 +93,21 @@ interface SourceChipProps { function SourceChip({ connector, connection, - unavailable, + unavailableReason, waiting, disabled, onConnect, }: SourceChipProps) { const state = sourceState(connection, waiting) const connected = connection?.viewerMembership === 'connected' + const unavailable = unavailableReason !== null const actionable = !unavailable && !waiting && (!connection || CONNECTABLE_MEMBERSHIPS.has(connection.viewerMembership)) - const title = unavailable - ? `${connector.meta.name} is unavailable in this deployment` - : connected - ? `${connector.meta.name}: ${state}` - : `Connect ${connector.meta.name}` + const title = + unavailableReason ?? + (connected ? `${connector.meta.name}: ${state}` : `Connect ${connector.meta.name}`) return ( simSearchConnectionsByType(memberConnectors), [memberConnectors] @@ -187,12 +196,17 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) {
{ordered.map((connector) => { const connection = connectionByType.get(connector.type) + const unavailableReason = !isSearchConnectorAvailable(connector, integrationAvailability) + ? `${connector.meta.name} is unavailable in this deployment` + : memberAccessAvailable + ? null + : MEMBER_ACCESS_UNAVAILABLE return ( startConnect(connector)} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index ba71adcec1a..b7f833cd588 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -579,10 +579,15 @@ const UserInputImpl = forwardRef(function UserI if (draftScopeKeyRef.current) { useMothershipDraftsStore.getState().clearDraft(draftScopeKeyRef.current) } + /** + * The chips are gone with the text, and clearing is not a removal to + * report. A composer that keeps its text (Search mode) keeps its chips + * too, so the diff base stays in step with what is still selected. + */ + prevSelectedContextsRef.current = [] } resetTranscript() currentFiles.clearAttachedFiles() - prevSelectedContextsRef.current = [] }, [onSubmit, resetTranscript]) /** diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index e9c5b35777e..13527b2e5dd 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -736,6 +736,8 @@ export function Home({ chatId, userName, userId }: HomeProps) { isReconnecting={isReconnecting} isLoading={showChatSkeleton} onSubmit={handleSubmit} + clearOnSubmit={composerMode !== 'search'} + onCleared={clearSearch} onStopGeneration={handleStopGeneration} messageQueue={messageQueue} editingQueuedId={editingQueuedId} diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx index 24a689f2210..347bd2e6a22 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx @@ -5,14 +5,18 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, describe, expect, it, vi } from 'vitest' -const { mockConnect, mockConnectSource } = vi.hoisted(() => ({ +const { mockConnect, mockConnectSource, mockFeatures } = vi.hoisted(() => ({ mockConnect: vi.fn(), mockConnectSource: vi.fn(), + mockFeatures: vi.fn(), })) vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), })) +vi.mock('@/app/workspace/[workspaceId]/providers/workspace-host-provider', () => ({ + useWorkspaceHostContext: () => ({ features: mockFeatures() }), +})) vi.mock('nuqs', () => ({ useQueryState: () => ['', vi.fn()], })) @@ -127,7 +131,8 @@ import { Search } from '@/app/workspace/[workspaceId]/search/search' let root: Root | null = null let container: HTMLDivElement | null = null -function mount() { +function mount(features: { knowledgeMemberAccess?: boolean } = { knowledgeMemberAccess: true }) { + mockFeatures.mockReturnValue({ credentialGroups: true, ...features }) ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true container = document.createElement('div') document.body.appendChild(container) @@ -178,4 +183,16 @@ describe('Search', () => { expect(mockConnect).toHaveBeenCalledWith('kb-sales', 'conn-sales-drive') expect(mockConnectSource).not.toHaveBeenCalled() }) + + it('offers no connection while per-member access is unavailable in the workspace', () => { + mount({ knowledgeMemberAccess: false }) + + expect(sectionLabels()).toEqual(['Sim Search Connectors']) + const text = container?.textContent ?? '' + expect(text).toContain( + 'Per-member access is not available in this workspace. Contact your administrator.' + ) + expect(text).not.toContain('Connected · 12 documents') + expect(buttons().find((button) => button.textContent === 'Connect')).toBeUndefined() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx index 1303e258856..34009c338f6 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -18,6 +18,7 @@ import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/ import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' +import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import { MemberConnectorsSection, memberConnectorName, @@ -46,6 +47,8 @@ const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] const CONNECTORS_LABEL = 'Sim Search Connectors' const NEEDS_KNOWLEDGE_BASE_SETUP = 'Set up by a workspace admin from a knowledge base.' const UNAVAILABLE = 'Unavailable in this deployment. Contact your administrator.' +const MEMBER_ACCESS_UNAVAILABLE = + 'Per-member access is not available in this workspace. Contact your administrator.' /** What a source row says once the viewer's own indexing has settled. */ function connectedDescription(connector: WorkspaceMemberConnector): string { @@ -57,7 +60,8 @@ interface SourceRowProps { connector: SearchConnector /** The Sim Search per-member connector for this source, once anyone has connected it. */ connection: WorkspaceMemberConnector | undefined - unavailable: boolean + /** Why the source cannot be connected here, shown in place of its state; null when it can. */ + unavailableReason: string | null waiting: boolean isPending: boolean onConnect: () => void @@ -72,11 +76,12 @@ interface SourceRowProps { function SourceRow({ connector, connection, - unavailable, + unavailableReason, waiting, isPending, onConnect, }: SourceRowProps) { + const unavailable = unavailableReason !== null const personal = canConnectPersonally(connector.meta) const membership = connection?.viewerMembership const state = connection @@ -89,7 +94,7 @@ function SourceRow({ : waiting ? `Finish connecting your ${connector.meta.name} account in the other tab.` : connector.meta.description - const description = unavailable ? UNAVAILABLE : personal ? state : NEEDS_KNOWLEDGE_BASE_SETUP + const description = unavailableReason ?? (personal ? state : NEEDS_KNOWLEDGE_BASE_SETUP) const connectable = !unavailable && personal && (!membership || CONNECTABLE_MEMBERSHIPS.has(membership)) return ( @@ -123,6 +128,13 @@ export function Search() { const params = useParams() const workspaceId = (params?.workspaceId as string) || '' const { integrationAvailability } = usePermissionConfig() + const { features } = useWorkspaceHostContext() + /** + * Judged by the workspace, as the server judges it: with per-member access + * off, every connect is refused, so the rows say so instead of offering + * one and the memberships are not fetched. + */ + const memberAccessAvailable = features?.knowledgeMemberAccess === true const [searchTerm, setSearchTermParam] = useQueryState(connectorSearchParam.key, { ...connectorSearchParam.parser, @@ -136,8 +148,10 @@ export function Search() { const setSearchTerm = useDebouncedSearchSetter(setSearchTermParam) const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS, isPending: connectionsPending } = - useWorkspaceMemberConnectors(workspaceId) - useScrollRestoration(scrollContainerRef, { ready: !connectionsPending }) + useWorkspaceMemberConnectors(memberAccessAvailable ? workspaceId : undefined) + useScrollRestoration(scrollContainerRef, { + ready: !memberAccessAvailable || !connectionsPending, + }) /** The Sim Search connection per source; other knowledge bases' connectors keep their own section. */ const { connectionByType, sharedConnectors } = useMemo(() => { @@ -212,12 +226,20 @@ export function Search() { {visibleConnectors.map((connector) => { const connection = connectionByType.get(connector.type) + const unavailableReason = !isSearchConnectorAvailable( + connector, + integrationAvailability + ) + ? UNAVAILABLE + : memberAccessAvailable + ? null + : MEMBER_ACCESS_UNAVAILABLE return ( @@ -233,10 +255,12 @@ export function Search() { )} - + {memberAccessAvailable && ( + + )} {error &&

{error}

} {setupConnector && ( diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index be84046151a..744f6b63123 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -331,6 +331,7 @@ export class BlockExecutor { fileKeys: blockCtx.fileKeys, allowLargeValueWorkflowScope: blockCtx.allowLargeValueWorkflowScope, userId: blockCtx.userId, + principal: blockCtx.principal, maxBytes: blockCtx.base64MaxBytes, preserveLargeValueMetadata: true, })) as NormalizedBlockOutput diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 00a7dfc196f..1788d602878 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -1728,6 +1728,7 @@ export class AgentBlockHandler implements BlockHandler { fileKeys: ctx.fileKeys, allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, userId: ctx.userId, + principal: ctx.principal, logger, maxBytes: inlineMaxBytes, onServableFileContributors: async (file, contributors) => { diff --git a/apps/sim/executor/variables/resolvers/reference-async.server.ts b/apps/sim/executor/variables/resolvers/reference-async.server.ts index 8f40d04b058..e4080d1c768 100644 --- a/apps/sim/executor/variables/resolvers/reference-async.server.ts +++ b/apps/sim/executor/variables/resolvers/reference-async.server.ts @@ -80,6 +80,7 @@ async function hydrateExplicitBase64( fileKeys: context.executionContext.fileKeys, allowLargeValueWorkflowScope: context.executionContext.allowLargeValueWorkflowScope, userId: context.executionContext.userId, + principal: context.executionContext.principal, maxBytes: context.executionContext.base64MaxBytes, }) if (!hydrated.base64) { diff --git a/apps/sim/executor/variables/resolvers/reference.ts b/apps/sim/executor/variables/resolvers/reference.ts index a5005fcf4bc..e2db9898eb9 100644 --- a/apps/sim/executor/variables/resolvers/reference.ts +++ b/apps/sim/executor/variables/resolvers/reference.ts @@ -1,3 +1,4 @@ +import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' import { materializeLargeValueRefSync, materializeLargeValueRefSyncOrThrow, @@ -19,6 +20,8 @@ export interface PathNavigationExecutionContext { fileKeys?: string[] allowLargeValueWorkflowScope?: boolean userId?: string + /** The principal behind the run; knowledge-base files hydrated along the path are read as them. */ + principal?: WorkflowExecutionPrincipal metadata?: { requestId?: string } base64MaxBytes?: number } diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 995c82316c1..028a148f332 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -392,7 +392,8 @@ export function useStartConnectorMemberEnrollment() { * Moves a connector between workspace and members mode. The switch rewrites * document access, so everything under the base is refetched: the connector * list and detail for the new mode and member state, and the document lists - * whose rows may have become hidden or visible. + * and per-document caches whose rows and chunks may have become hidden or + * visible. */ export function useUpdateConnectorAccess() { const queryClient = useQueryClient() @@ -402,6 +403,7 @@ export function useUpdateConnectorAccess() { onSettled: (_data, _error, { knowledgeBaseId }) => { queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentLists(knowledgeBaseId) }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentDetails(knowledgeBaseId) }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index b8d36ea3eb4..69375631755 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -57,9 +57,17 @@ export function describeMembership({ }: DescribeMembershipInput): string | null { switch (membership) { case 'connected': - return memberSyncStatus === 'pending' || memberSyncStatus === 'running' - ? `Syncing the ${name} documents shared with you. They appear when the sync completes.` - : null + switch (memberSyncStatus) { + case 'pending': + case 'running': + return `Syncing the ${name} documents shared with you. They appear when the sync completes.` + case 'error': + return `The last ${name} sync failed; the documents you already have stay visible while it retries.` + case 'disabled': + return `Syncing ${name} per member is turned off. Ask a workspace admin to turn it back on.` + default: + return null + } case 'needs_reauth': return `Reconnect your ${name} account to keep seeing the documents shared with you.` case 'unverified_email': @@ -106,7 +114,12 @@ export function useMemberEnrollment({ connectedRef.current = connectedConnectorIds }, [connectedConnectorIds]) - const awaiting = [...awaitingSince.keys()].some((id) => !connectedConnectorIds.has(id)) + /** + * Polls while any connection is awaited, and once more after the last one + * connects: that tick drops the connected ids, so a token that later needs + * reauthorization is not mistaken for a connection still being awaited. + */ + const awaiting = awaitingSince.size > 0 useEffect(() => { if (!awaiting) return const timer = setInterval(() => { @@ -151,8 +164,13 @@ export function useMemberEnrollment({ }) } + /** + * Each path clears the other's failure first: the surface shows one error, + * and a stale one from the other path would outlive a success on this one. + */ const connect = (knowledgeBaseId: string, connectorId: string) => - openEnrollment(({ onSuccess, onError }) => + openEnrollment(({ onSuccess, onError }) => { + sourceConnection.reset() enrollment.mutate( { knowledgeBaseId, connectorId }, { @@ -163,7 +181,7 @@ export function useMemberEnrollment({ }, } ) - ) + }) /** * Connects a Sim Search source: its per-member connector exists afterwards, @@ -175,7 +193,8 @@ export function useMemberEnrollment({ connectorType: string, sourceConfig?: Record ) => - openEnrollment(({ onSuccess, onError }) => + openEnrollment(({ onSuccess, onError }) => { + enrollment.reset() sourceConnection.mutate( { workspaceId, connectorType, sourceConfig }, { @@ -186,7 +205,7 @@ export function useMemberEnrollment({ }, } ) - ) + }) const isAwaiting = (connectorId: string) => awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId) diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts index 5eeef18534f..214523edcec 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts @@ -96,4 +96,10 @@ describe('connector access binding contracts', () => { }).success ).toBe(false) }) + + it('refuses a switch to workspace mode that names no credential', () => { + const parsed = updateConnectorAccessBodySchema.safeParse({ accessMode: 'workspace' }) + expect(parsed.success).toBe(false) + expect(parsed.error?.issues.map((issue) => issue.path)).toEqual([['credentialId']]) + }) }) diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index e35d9c10b5a..1e7ffc1213d 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -101,6 +101,13 @@ export const updateConnectorAccessBodySchema = z message: 'A members-mode connector crawls with member credentials, not a credentialId', }) } + if (value.accessMode === 'workspace' && !value.credentialId) { + ctx.addIssue({ + code: 'custom', + path: ['credentialId'], + message: 'Switching to workspace mode needs the credentialId the connector syncs as', + }) + } }) export type UpdateConnectorAccessBody = z.input diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index 5106eb13f14..d0ded5301cf 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -23,7 +23,9 @@ import { } from '@/lib/knowledge/orchestration/connector-access' import { getKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' import type { KnowledgeOperationSource } from '@/lib/knowledge/orchestration/shared' +import { getServiceConfigByProviderId, getServiceConfigByServiceId } from '@/lib/oauth' import { getConnectorMeta } from '@/connectors/registry' +import type { ConnectorMeta } from '@/connectors/types' export interface StartKnowledgeConnectorMemberEnrollmentInput { knowledgeBaseId: string @@ -131,7 +133,7 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ accessMode: 'workspace' as const, credentialId: await requireUsableCredential({ credentialId: input.credentialId, - connectorAuthMode: connectorMeta.auth.mode, + connectorMeta, workspaceId, actingUserId, requestId, @@ -178,28 +180,40 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ }) /** - * Workspace mode needs a credential the caller may use, and one that yields a - * token, since the connector syncs as it from then on. Only an OAuth connector - * can change modes: an API-key connector has no account to sync per member. + * Workspace mode needs a credential the caller may use, of the connector's own + * provider, and one that yields a token, since the connector syncs as it from + * then on with no call to the source to catch a mismatch. Only an OAuth + * connector can change modes: an API-key connector has no account to sync per + * member. */ async function requireUsableCredential(input: { credentialId: string | undefined - connectorAuthMode: 'oauth' | 'apiKey' + connectorMeta: Pick workspaceId: string actingUserId: string requestId: string }): Promise { - if (input.connectorAuthMode !== 'oauth') { + const { auth } = input.connectorMeta + if (auth.mode !== 'oauth') { throw new OrchestrationError('validation', 'Only OAuth connectors can change access mode') } if (!input.credentialId) { throw new OrchestrationError('validation', 'credentialId is required for workspace mode') } + const service = + getServiceConfigByServiceId(auth.provider) ?? getServiceConfigByProviderId(auth.provider) + if (!service) { + throw new OrchestrationError( + 'validation', + `${input.connectorMeta.name} has no OAuth service to validate the credential against` + ) + } const token = await resolveConnectorCredentialAccessToken({ credentialId: input.credentialId, workspaceId: input.workspaceId, actingUserId: input.actingUserId, requestId: input.requestId, + service, }) if (!token) { throw new OrchestrationError( diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 7891dd941b3..729ce72b0dc 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -64,6 +64,7 @@ import type { KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' import { isMemberSyncStatus } from '@/lib/knowledge/types' +import { credentialProviderMatchesService, type ServiceProviderIdentity } from '@/lib/oauth' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { CAPABILITY_RULES, refuseCapability } from '@/lib/permission-groups/capabilities' import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' @@ -200,6 +201,7 @@ async function resolveAuthorizedConnectorCredentialIdentity(input: { credentialId: string workspaceId: string actingUserId: string + service?: ServiceProviderIdentity }) { const access = await getCredentialActorContext(input.credentialId, input.actingUserId) if ( @@ -212,14 +214,32 @@ async function resolveAuthorizedConnectorCredentialIdentity(input: { 'Credential is not available to you in this workspace. Ask a credential administrator to grant access or select another credential.' ) } + if ( + input.service && + (!access.credential.providerId || + !credentialProviderMatchesService(access.credential.providerId, input.service)) + ) { + throw new OrchestrationError( + 'validation', + 'Credential belongs to another service. Select a credential for the connector’s own provider.' + ) + } return resolveCredentialTokenIdentity(input.credentialId, input.workspaceId) } +/** + * The access token a connector syncs with, once the caller may use the + * credential in this workspace. Pass `service` to also refuse a credential + * of another provider: creation validates the source config with the token, + * which catches that on its own, but a mode switch stores the credential + * without a call to the source. + */ export async function resolveConnectorCredentialAccessToken(input: { credentialId: string workspaceId: string actingUserId: string requestId: string + service?: ServiceProviderIdentity }): Promise { const identity = await resolveAuthorizedConnectorCredentialIdentity(input) if (!identity) return null diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index adea8d1349c..0228b0d2fd8 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -418,9 +418,10 @@ export const knowledgeOperations = { principalKinds: ['session'], }), /** - * Connecting a Sim Search source: any reader may connect their own account; - * the first connect of a source also creates its knowledge base and - * connector, which those operations reserve for an admin. + * Connecting a Sim Search source: any reader may connect their own account. + * The first connect of a source also creates its knowledge base and + * connector, which the use case reserves for an admin and refuses to anyone + * else with the way forward (ask an admin to connect the source first). */ simSearchConnect: defineWorkspaceOperation({ id: 'knowledge.simSearch.connect', diff --git a/apps/sim/lib/knowledge/application/sim-search.test.ts b/apps/sim/lib/knowledge/application/sim-search.test.ts new file mode 100644 index 00000000000..b85a0622dd9 --- /dev/null +++ b/apps/sim/lib/knowledge/application/sim-search.test.ts @@ -0,0 +1,246 @@ +/** + * @vitest-environment node + */ + +import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveWorkspace: vi.fn(), + resolvePermission: vi.fn(), + isMemberAccessAvailable: vi.fn(), + createKnowledgeBase: vi.fn(), + createConnector: vi.fn(), + enroll: vi.fn(), + getUserPermissionConfig: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: {}, + AuditResourceType: {}, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveKnowledgeWorkspaceContext: mocks.resolveWorkspace, +})) + +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: mocks.isMemberAccessAvailable, +})) + +vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ + createKnowledgeBase: { execute: mocks.createKnowledgeBase }, +})) + +vi.mock('@/lib/knowledge/application/connectors', () => ({ + createKnowledgeConnector: { execute: mocks.createConnector }, +})) + +vi.mock('@/lib/knowledge/application/connector-access', () => ({ + startKnowledgeConnectorMemberEnrollment: { execute: mocks.enroll }, +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mocks.getUserPermissionConfig, +})) + +vi.mock('@/lib/sim-search/connectors', () => ({ + SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', + canConnectPersonally: (meta: { permissionScopedListing?: unknown }) => + Boolean(meta.permissionScopedListing), + missingSetupFields: ( + meta: { configFields: Array<{ id: string; title: string; required?: boolean }> }, + sourceConfig: Record + ) => + meta.configFields.filter( + (field) => field.required && typeof sourceConfig[field.id] !== 'string' + ), +})) + +vi.mock('@/connectors/registry', () => ({ + CONNECTOR_META_REGISTRY: { + google_drive: { + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive' }, + permissionScopedListing: { capFieldIds: [] }, + configFields: [], + }, + confluence: { + name: 'Confluence', + auth: { mode: 'oauth', provider: 'confluence' }, + permissionScopedListing: { capFieldIds: [] }, + configFields: [{ id: 'spaceKey', title: 'a space key', required: true }], + }, + }, +})) + +import { connectSimSearchConnector } from '@/lib/knowledge/application/sim-search' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' + +const workspaceContext = { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'owner-1', +} + +const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } +const existingConnector = { knowledgeBaseId: 'kb-search', connectorId: 'connector-drive' } + +/** The first lookup runs outside the setup lock and the second inside it. */ +function queueConnectorLookups(...results: Array) { + for (const result of results) { + queueTableRows(knowledgeConnector, result ? [result] : []) + } +} + +describe('connectSimSearchConnector', () => { + afterAll(resetDbChainMock) + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.resolveWorkspace.mockResolvedValue(workspaceContext) + mocks.getUserPermissionConfig.mockResolvedValue(DEFAULT_PERMISSION_GROUP_CONFIG) + mocks.isMemberAccessAvailable.mockResolvedValue(true) + mocks.createKnowledgeBase.mockResolvedValue({ knowledgeBase: { id: 'kb-new' } }) + mocks.createConnector.mockResolvedValue({ connector: { id: 'connector-new' } }) + mocks.enroll.mockResolvedValue({ url: 'https://sim.test/enroll/token' }) + }) + + it('enrolls a reader in a source someone already connected', async () => { + mocks.resolvePermission.mockResolvedValue('read') + queueConnectorLookups(existingConnector) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + + expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(mocks.enroll).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: { ...existingConnector, assertedWorkspaceId: 'workspace-1' }, + }) + ) + }) + + it('tells a reader to ask an admin when the source has no connector yet', async () => { + mocks.resolvePermission.mockResolvedValue('read') + queueConnectorLookups(null) + + await expect( + connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + ).rejects.toMatchObject({ + code: 'forbidden', + message: expect.stringContaining('Ask a workspace admin to connect Google Drive first'), + }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(mocks.enroll).not.toHaveBeenCalled() + }) + + it('refuses before creating anything when per-member access is unavailable', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + mocks.isMemberAccessAvailable.mockResolvedValue(false) + queueConnectorLookups(null) + + await expect( + connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + }) + + it('lets an admin create the base and the connector with the setup fields, then enrolls them', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueConnectorLookups(null, null) + queueTableRows(knowledgeBase, []) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { + workspaceId: 'workspace-1', + connectorType: 'confluence', + sourceConfig: { spaceKey: 'ENG' }, + }, + }) + + expect(mocks.createKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: expect.objectContaining({ workspaceId: 'workspace-1', name: 'Sim Search' }), + }) + ) + expect(mocks.createConnector).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: expect.objectContaining({ + knowledgeBaseId: 'kb-new', + assertedWorkspaceId: 'workspace-1', + connectorType: 'confluence', + sourceConfig: { spaceKey: 'ENG' }, + accessMode: 'members', + }), + }) + ) + expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1) + expect(result).toEqual({ + knowledgeBaseId: 'kb-new', + connectorId: 'connector-new', + url: 'https://sim.test/enroll/token', + }) + }) + + it('refuses a first connect that leaves a setup field empty', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueConnectorLookups(null) + + await expect( + connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'confluence' }, + }) + ).rejects.toMatchObject({ + code: 'validation', + message: 'Confluence needs a space key to connect', + }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + }) + + it('reuses the connector another first connect created while it waited for the lock', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueConnectorLookups(null, existingConnector) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) + }) +}) diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts index e63de78a909..056ece81a8b 100644 --- a/apps/sim/lib/knowledge/application/sim-search.ts +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -1,11 +1,21 @@ +import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' -import { and, asc, eq, isNull } from 'drizzle-orm' +import { + permissionSatisfies, + resolveEffectiveWorkspacePermission, +} from '@sim/platform-authz/workspace' +import { and, asc, eq, isNull, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { DbOrTx } from '@/lib/db/types' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' import { createKnowledgeConnector } from '@/lib/knowledge/application/connectors' -import { resolveKnowledgeWorkspaceContext } from '@/lib/knowledge/application/contexts' +import { + type KnowledgeWorkspaceContext, + resolveKnowledgeWorkspaceContext, +} from '@/lib/knowledge/application/contexts' import { createKnowledgeBase } from '@/lib/knowledge/application/knowledge-bases' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { @@ -19,6 +29,8 @@ const SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION = 'What each person can open in the sources they connected, searched as them.' /** Between runs the change feeds keep deletions and unshares fresh; the hourly run fills the rest. */ const SIM_SEARCH_SYNC_INTERVAL_MINUTES = 60 +/** How long a first connect waits for another first connect of the same workspace to finish. */ +const SIM_SEARCH_SETUP_LOCK_TIMEOUT_MS = 10_000 export interface ConnectSimSearchConnectorInput { workspaceId: string @@ -35,8 +47,12 @@ export interface ConnectSimSearchConnectorResult { url: string } -async function findSimSearchConnector(workspaceId: string, connectorType: string) { - const [row] = await db +async function findSimSearchConnector( + executor: DbOrTx, + workspaceId: string, + connectorType: string +) { + const [row] = await executor .select({ knowledgeBaseId: knowledgeBase.id, connectorId: knowledgeConnector.id }) .from(knowledgeConnector) .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) @@ -56,8 +72,8 @@ async function findSimSearchConnector(workspaceId: string, connectorType: string return row ?? null } -async function findSimSearchKnowledgeBase(workspaceId: string) { - const [row] = await db +async function findSimSearchKnowledgeBase(executor: DbOrTx, workspaceId: string) { + const [row] = await executor .select({ id: knowledgeBase.id }) .from(knowledgeBase) .where( @@ -72,13 +88,43 @@ async function findSimSearchKnowledgeBase(workspaceId: string) { return row ?? null } +/** + * The first connect of a source turns it on for the whole workspace, which is + * an admin decision the same way a members-mode connector is. Refused with + * the way forward rather than the nested operations' generic role error, so a + * reader learns whom to ask and for what. + */ +async function requireSimSearchSetupAdmin( + userId: string, + context: KnowledgeWorkspaceContext, + sourceName: string +): Promise { + const permission = await resolveEffectiveWorkspacePermission( + userId, + context.workspaceId, + context.workspaceOrganizationId + ) + if (!permissionSatisfies(permission, 'admin')) { + throw new OrchestrationError( + 'forbidden', + `${sourceName} is not connected in this workspace yet. Ask a workspace admin to connect ${sourceName} first; after that everyone connects their own account.` + ) + } +} + /** * One click on a Sim Search source: the workspace's Sim Search knowledge base - * and a per-member connector for that source exist after this (the first - * connect creates them, which the connector operation reserves for an admin, - * and supplies the source's setup fields when it has any), and the caller - * gets the link that connects their own account. The OAuth completion queues - * their member run, so indexing starts on its own. + * and a per-member connector for that source exist after this, and the caller + * gets the link that connects their own account. The first connect of a + * source creates both, which takes a workspace admin and the source's setup + * fields when it has any; every connect after that only enrolls. The OAuth + * completion queues the member run, so indexing starts on its own. + * + * The creating branch runs under a per-workspace advisory lock and re-checks + * for the connector once it holds it: nothing in the schema keeps two + * concurrent first connects from each creating a Sim Search base and a + * connector of the same source, and the second would index the same + * accounts twice. */ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.simSearchConnect, @@ -93,8 +139,21 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ ) } const workspaceId = context.workspaceId - let target = await findSimSearchConnector(workspaceId, input.connectorType) + let target = await findSimSearchConnector(db, workspaceId, input.connectorType) if (!target) { + /** + * Judged before anything is created: the connector creation below checks + * the same availability, but only after the knowledge base exists. + */ + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId }))) { + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) + } + const userId = resolvePrincipalSubjectUserId(principal) + if (!userId) throw new OrchestrationError('forbidden', 'Sign in to connect your account') + await requireSimSearchSetupAdmin(userId, context, meta.name) const sourceConfig = input.sourceConfig ?? {} const missing = missingSetupFields(meta, sourceConfig) if (missing.length > 0) { @@ -103,34 +162,44 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ `${meta.name} needs ${missing.map((field) => field.title).join(' and ')} to connect` ) } - const knowledgeBaseId = - (await findSimSearchKnowledgeBase(workspaceId))?.id ?? - ( - await createKnowledgeBase.execute({ - principal, - input: { - workspaceId, - name: SIM_SEARCH_KNOWLEDGE_BASE_NAME, - description: SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION, - source: 'ui', - }, - request, - }) - ).knowledgeBase.id - const created = await createKnowledgeConnector.execute({ - principal, - input: { - knowledgeBaseId, - assertedWorkspaceId: workspaceId, - connectorType: input.connectorType, - sourceConfig, - syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, - accessMode: 'members', - source: 'ui', - }, - request, + target = await db.transaction(async (tx) => { + await tx.execute( + sql`select set_config('lock_timeout', ${`${SIM_SEARCH_SETUP_LOCK_TIMEOUT_MS}ms`}, true)` + ) + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${`sim-search:connect:${workspaceId}`}, 0))` + ) + const existing = await findSimSearchConnector(tx, workspaceId, input.connectorType) + if (existing) return existing + const knowledgeBaseId = + (await findSimSearchKnowledgeBase(tx, workspaceId))?.id ?? + ( + await createKnowledgeBase.execute({ + principal, + input: { + workspaceId, + name: SIM_SEARCH_KNOWLEDGE_BASE_NAME, + description: SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION, + source: 'ui', + }, + request, + }) + ).knowledgeBase.id + const created = await createKnowledgeConnector.execute({ + principal, + input: { + knowledgeBaseId, + assertedWorkspaceId: workspaceId, + connectorType: input.connectorType, + sourceConfig, + syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, + accessMode: 'members', + source: 'ui', + }, + request, + }) + return { knowledgeBaseId, connectorId: created.connector.id } }) - target = { knowledgeBaseId, connectorId: created.connector.id } } const { url } = await startKnowledgeConnectorMemberEnrollment.execute({ principal, diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts index df053676271..c9fe94f0725 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts @@ -9,26 +9,32 @@ import { } from '@/lib/uploads/utils/user-file-base64.server' import type { UserFile } from '@/executor/types' -const { mockDownloadFile, mockDownloadServableFileFromStorage, mockRedis, mockVerifyFileAccess } = - vi.hoisted(() => { - const mockRedis = { - get: vi.fn(), - set: vi.fn(), - hget: vi.fn(), - hset: vi.fn(), - hgetall: vi.fn(), - expire: vi.fn(), - scan: vi.fn(), - del: vi.fn(), - eval: vi.fn(), - } - return { - mockDownloadFile: vi.fn(), - mockDownloadServableFileFromStorage: vi.fn(), - mockRedis, - mockVerifyFileAccess: vi.fn(), - } - }) +const { + mockDownloadFile, + mockDownloadServableFileFromStorage, + mockRedis, + mockVerifyFileAccess, + mockResolveKnowledgeAccessScope, +} = vi.hoisted(() => { + const mockRedis = { + get: vi.fn(), + set: vi.fn(), + hget: vi.fn(), + hset: vi.fn(), + hgetall: vi.fn(), + expire: vi.fn(), + scan: vi.fn(), + del: vi.fn(), + eval: vi.fn(), + } + return { + mockDownloadFile: vi.fn(), + mockDownloadServableFileFromStorage: vi.fn(), + mockRedis, + mockVerifyFileAccess: vi.fn(), + mockResolveKnowledgeAccessScope: vi.fn(), + } +}) const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient @@ -53,6 +59,10 @@ vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: mockVerifyFileAccess, })) +vi.mock('@/lib/knowledge/access/scope', () => ({ + resolveKnowledgeAccessScope: mockResolveKnowledgeAccessScope, +})) + describe('hydrateUserFilesWithBase64', () => { beforeEach(() => { vi.clearAllMocks() @@ -240,6 +250,46 @@ describe('hydrateUserFilesWithBase64', () => { expect(hydrated.file).not.toHaveProperty('base64') }) + it('reads a knowledge-base file as the principal behind the run', async () => { + mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) + const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } + const scope = { kind: 'user' as const, tokens: ['user:user-1'] } + mockResolveKnowledgeAccessScope.mockResolvedValue(scope) + const file: UserFile = { + id: 'file-1', + name: 'shared.txt', + key: 'kb/workspace/shared.txt', + url: '/api/files/serve/kb/workspace/shared.txt?context=knowledge-base', + size: 5, + type: 'text/plain', + context: 'knowledge-base', + } + + const hydrated = await hydrateUserFilesWithBase64( + { file }, + { + workspaceId: 'workspace', + workflowId: 'workflow', + userId: 'user-1', + principal, + maxBytes: 10, + } + ) + + expect(hydrated.file.base64).toBe(Buffer.from('hello').toString('base64')) + expect(mockResolveKnowledgeAccessScope).toHaveBeenCalledWith(principal, { + workspaceId: 'workspace', + }) + expect(mockVerifyFileAccess).toHaveBeenCalledWith( + file.key, + 'user-1', + undefined, + 'knowledge-base', + false, + { knowledgeAccess: scope } + ) + }) + it('hydrates prior-execution files when workflow-scoped reads are enabled', async () => { mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) const file: UserFile = { diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.ts index cf2103bbb0e..87921a31836 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.ts @@ -1,3 +1,4 @@ +import type { Principal } from '@sim/auth/principal' import type { Logger } from '@sim/logger' import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' @@ -163,6 +164,13 @@ export interface Base64HydrationOptions { fileKeys?: string[] allowLargeValueWorkflowScope?: boolean userId?: string + /** + * The principal behind the run. A knowledge-base file is read as them, so a + * document shared with only this person still hydrates; `userId` alone may + * be the workflow owner standing in for an actorless run and must not widen + * what the run can read. + */ + principal?: Principal logger?: Logger maxBytes?: number allowUnknownSize?: boolean @@ -454,6 +462,7 @@ async function resolveBase64( fileKeys: options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, encoding: 'base64', maxBytes, }) @@ -487,6 +496,7 @@ async function hydrateUserFile( fileKeys: options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, logger, }) } catch (error) { diff --git a/apps/sim/lib/workflows/executor/execute-service.ts b/apps/sim/lib/workflows/executor/execute-service.ts index 3ebb700717f..7edae7f8b2c 100644 --- a/apps/sim/lib/workflows/executor/execute-service.ts +++ b/apps/sim/lib/workflows/executor/execute-service.ts @@ -523,6 +523,7 @@ export async function executeWorkflowService( workspaceId, workflowId, userId: actorUserId, + principal, allowLargeValueWorkflowScope: false, requestSignal: abortSignal, requestHeaders: headers, @@ -695,6 +696,7 @@ export async function executeWorkflowService( fileKeys: result.metadata?.fileKeys ?? [], allowLargeValueWorkflowScope: false, userId: actorUserId, + principal, maxBytes: base64MaxBytes, preserveLargeValueMetadata: true, })) as NormalizedBlockOutput) diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index dda38e49dab..49dca891fd0 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -1,3 +1,4 @@ +import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, omit } from '@sim/utils/object' @@ -95,6 +96,8 @@ export interface StreamingResponseOptions { workspaceId?: string workflowId?: string userId?: string + /** The principal behind the run; knowledge-base files in the output are read as them. */ + principal?: WorkflowExecutionPrincipal /** Incoming fetch/request abort — combined with the stream timeout. */ requestSignal?: AbortSignal /** Used with the independent event policies to negotiate agent-events SSE. */ @@ -155,6 +158,7 @@ type OutputExtractionContext = Pick< | 'fileKeys' | 'allowLargeValueWorkflowScope' | 'userId' + | 'principal' > & { base64MaxBytes?: number } async function extractOutputValue( @@ -174,6 +178,7 @@ async function extractOutputValue( fileKeys: context.fileKeys, allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, userId: context.userId, + principal: context.principal, metadata: { requestId: context.requestId }, base64MaxBytes: context.base64MaxBytes, }, @@ -225,6 +230,7 @@ function buildMaterializationContext( fileKeys: context.fileKeys, allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, userId: context.userId, + principal: context.principal, } } @@ -748,6 +754,7 @@ export async function createStreamingResponse( fileKeys: options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, base64MaxBytes: Math.min( base64MaxBytes ?? MAX_INLINE_MATERIALIZATION_BYTES, getBase64DecodedByteBudget(remainingBytes) From 37ec2695acdd3b86c0bc5a10eca664c8d46ef23a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 02:10:00 -0700 Subject: [PATCH 41/76] fix(connectors): withdraw member access on unreachable scopes and page unlimited listings Gmail treated the members-mode cap of 0 as "capped after the first page", so a member's sync never listed past 100 threads and could never withdraw access. Box turned a subfolder a member cannot reach into a capped listing, which suppressed removals forever; the members-mode crawl now marks its sync context, and Box takes the skipped folder as absent from that member's complete listing while a shared credential still caps. Google Calendar with several calendars withdrew everything when one calendar answered 404; per member it now skips only that calendar. JSM classifies 403 (a service desk the caller may not view) alongside 404, and the Atlassian connectors also treat a token that reaches only other sites as not on the site. Monday reports every configured board coming back absent as the scope being unavailable. Salesforce classifies 400 INVALID_TYPE and 403 INSUFFICIENT_ACCESS as the object being unreadable, and filters on LastModifiedDate with supportsIncrementalSync so member passes stop enumerating the whole org. The duplicated Graph listing error helper moves into the shared connector utils. --- apps/sim/connectors/box/box.test.ts | 78 +++++++++++ apps/sim/connectors/box/box.ts | 10 +- .../connectors/confluence/confluence.test.ts | 27 ++++ apps/sim/connectors/confluence/confluence.ts | 13 +- apps/sim/connectors/gmail/gmail.test.ts | 88 +++++++++++++ apps/sim/connectors/gmail/gmail.ts | 2 +- .../google-calendar/google-calendar.test.ts | 76 +++++++++++ .../google-calendar/google-calendar.ts | 29 +++- apps/sim/connectors/jira/jira.ts | 9 +- apps/sim/connectors/jsm/jsm.test.ts | 75 +++++++++++ apps/sim/connectors/jsm/jsm.ts | 35 ++++- apps/sim/connectors/monday/monday.test.ts | 74 +++++++++++ apps/sim/connectors/monday/monday.ts | 36 ++++- apps/sim/connectors/onedrive/onedrive.ts | 17 +-- apps/sim/connectors/salesforce/meta.ts | 2 + .../connectors/salesforce/salesforce.test.ts | 124 ++++++++++++++++++ apps/sim/connectors/salesforce/salesforce.ts | 78 +++++++++-- apps/sim/connectors/sharepoint/sharepoint.ts | 26 +--- apps/sim/connectors/utils.ts | 30 +++++ .../connectors/member-sync-engine.ts | 7 +- 20 files changed, 778 insertions(+), 58 deletions(-) create mode 100644 apps/sim/connectors/box/box.test.ts create mode 100644 apps/sim/connectors/gmail/gmail.test.ts create mode 100644 apps/sim/connectors/jsm/jsm.test.ts create mode 100644 apps/sim/connectors/salesforce/salesforce.test.ts diff --git a/apps/sim/connectors/box/box.test.ts b/apps/sim/connectors/box/box.test.ts new file mode 100644 index 00000000000..1ab7d1b4d6f --- /dev/null +++ b/apps/sim/connectors/box/box.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ BoxCompanyIcon: () => null })) + +import { boxConnector } from '@/connectors/box/box' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' + +interface FolderReply { + status?: number + entries?: unknown[] +} + +/** Routes `GET /folders/:id/items` by folder id; unknown folders answer 404. */ +function mockFolders(folders: Record) { + mockFetchWithRetry.mockImplementation(async (url: string) => { + const folderId = /\/folders\/([^/]+)\/items/.exec(url)?.[1] ?? '' + const reply = folders[folderId] ?? { status: 404 } + const status = reply.status ?? 200 + const body = { entries: reply.entries ?? [], next_marker: null } + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response + }) +} + +const FILE = { type: 'file', id: 'f1', name: 'notes.txt', extension: 'txt', size: 10 } +const SUBFOLDER = { type: 'folder', id: 'sub', name: 'Private' } + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('box listDocuments with a subfolder the caller cannot reach', () => { + it('caps the listing under a shared credential so nothing is reconciled as deleted', async () => { + mockFolders({ '0': { entries: [FILE, SUBFOLDER] }, sub: { status: 403 } }) + const syncContext: Record = {} + + const result = await boxConnector.listDocuments('token', {}, undefined, syncContext) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['f1']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it("lists completely under a member's own token so their access to it is withdrawn", async () => { + mockFolders({ '0': { entries: [FILE, SUBFOLDER] }, sub: { status: 403 } }) + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const result = await boxConnector.listDocuments('token', {}, undefined, syncContext) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['f1']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('reports the configured root being unreachable as the scope being unavailable', async () => { + mockFolders({ '42': { status: 403 } }) + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const error = await boxConnector + .listDocuments('token', { folderId: '42' }, undefined, syncContext) + .catch((caught: unknown) => caught) + + expect(boxConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) +}) diff --git a/apps/sim/connectors/box/box.ts b/apps/sim/connectors/box/box.ts index e21b1e5224f..3f36323905f 100644 --- a/apps/sim/connectors/box/box.ts +++ b/apps/sim/connectors/box/box.ts @@ -10,6 +10,7 @@ import { ConnectorListingScopeUnavailableError, htmlToPlainText, isListingScopeUnavailableError, + isPerMemberListing, isSkippedDocument, markSkipped, parseTagDate, @@ -385,7 +386,7 @@ async function fetchExtractedText( /** * Lists one page of a folder. A folder the credential can no longer read is * reported rather than thrown, so one inaccessible subtree does not abort the - * whole listing — the caller flags the listing as capped instead. + * whole listing — the caller decides whether that caps the listing. */ async function listFolderPage( accessToken: string, @@ -471,10 +472,13 @@ export const boxConnector: ConnectorConfig = { files.push(item) } } - } else if (syncContext) { + } else if (syncContext && !isPerMemberListing(syncContext)) { /** * A folder was skipped, so documents that still exist in Box are absent from - * this listing. Without this flag the engine would reconcile them as deleted. + * this listing. Under a shared credential the engine would otherwise + * reconcile them as deleted; under a member's own token the folder is + * simply not shared with that member, so their listing stays complete and + * their access to its files is withdrawn. */ syncContext.listingCapped = true } diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index aad5aa82a77..70f06538d2b 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -2,8 +2,13 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { + AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, +} from '@/lib/atlassian/discovery' import { buildLastModifiedClause, + confluenceConnector, escapeCql, extractCursor, isCurrentContent, @@ -55,6 +60,28 @@ describe('buildLastModifiedClause', () => { }) }) +describe('confluence listing scope classification', () => { + it.concurrent('treats a token that reaches no Atlassian site as not on the site', () => { + expect( + confluenceConnector.isListingScopeUnavailableError?.( + new AtlassianSiteNotAccessibleError('none') + ) + ).toBe(true) + }) + + it.concurrent('treats a token that reaches only other Atlassian sites the same way', () => { + expect( + confluenceConnector.isListingScopeUnavailableError?.( + new AtlassianSiteNotMatchedError('elsewhere') + ) + ).toBe(true) + }) + + it.concurrent('leaves other failures for the sync engines to retry', () => { + expect(confluenceConnector.isListingScopeUnavailableError?.(new Error('boom'))).toBe(false) + }) +}) + describe('isCurrentContent', () => { it.concurrent('keeps current content', () => { expect(isCurrentContent({ id: '1', status: 'current' })).toBe(true) diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 8f2aaba579d..e1b5a4ce798 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -1,7 +1,10 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import * as cheerio from 'cheerio' -import { AtlassianSiteNotAccessibleError } from '@/lib/atlassian/discovery' +import { + AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, +} from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -508,12 +511,14 @@ export const confluenceConnector: ConnectorConfig = { }, /** - * A member who is not on the Atlassian site, or cannot see the configured - * space, lists nothing: a complete listing of nothing, not an error. + * A member who is not on the Atlassian site — their token reaches no site, + * or only sites other than the configured one — or who cannot see the + * configured space, lists nothing: a complete listing of nothing, not an error. */ isListingScopeUnavailableError: (error) => error instanceof ConfluenceSpaceNotFoundError || - error instanceof AtlassianSiteNotAccessibleError, + error instanceof AtlassianSiteNotAccessibleError || + error instanceof AtlassianSiteNotMatchedError, } /** diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts new file mode 100644 index 00000000000..74023e74f51 --- /dev/null +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -0,0 +1,88 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) + +import { gmailConnector } from '@/connectors/gmail/gmail' + +function threads(count: number, prefix: string) { + return Array.from({ length: count }, (_, i) => ({ id: `${prefix}-${i}`, historyId: '1' })) +} + +/** Queues thread-list pages in order; each call records the requested URL. */ +function mockPages(pages: { threads: unknown[]; nextPageToken?: string }[]) { + const urls: string[] = [] + let call = 0 + mockFetchWithRetry.mockImplementation(async (url: string) => { + urls.push(url) + const page = pages[call++] ?? { threads: [] } + return { + ok: true, + status: 200, + json: async () => page, + text: async () => JSON.stringify(page), + } as unknown as Response + }) + return urls +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('gmail listDocuments with maxThreads 0 (unlimited, a per-member sync)', () => { + it('pages past a full page and never marks the listing capped', async () => { + const urls = mockPages([ + { threads: threads(100, 'a'), nextPageToken: 'page-2' }, + { threads: threads(50, 'b') }, + ]) + const syncContext: Record = {} + + const first = await gmailConnector.listDocuments( + 'token', + { maxThreads: 0 }, + undefined, + syncContext + ) + expect(first.documents).toHaveLength(100) + expect(first.hasMore).toBe(true) + expect(first.nextCursor).toBe('page-2') + expect(syncContext.listingCapped).toBeUndefined() + + const second = await gmailConnector.listDocuments( + 'token', + { maxThreads: 0 }, + first.nextCursor, + syncContext + ) + expect(second.documents).toHaveLength(50) + expect(second.hasMore).toBe(false) + expect(syncContext.totalThreadsFetched).toBe(150) + expect(syncContext.listingCapped).toBeUndefined() + expect(urls[1]).toContain('pageToken=page-2') + expect(urls[1]).toContain('maxResults=100') + }) + + it('still stops and flags a cap that truncates a longer listing', async () => { + mockPages([{ threads: threads(100, 'a'), nextPageToken: 'page-2' }]) + const syncContext: Record = {} + + const result = await gmailConnector.listDocuments( + 'token', + { maxThreads: 100 }, + undefined, + syncContext + ) + expect(result.hasMore).toBe(false) + expect(result.nextCursor).toBeUndefined() + expect(syncContext.listingCapped).toBe(true) + }) +}) diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index 90b4c429a43..c8ea8fd70b7 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -512,7 +512,7 @@ export const gmailConnector: ConnectorConfig = { const newTotal = totalFetched + documents.length if (syncContext) syncContext.totalThreadsFetched = newTotal - const hitLimit = newTotal >= maxThreads + const hitLimit = maxThreads > 0 && newTotal >= maxThreads /** * Only a cap that actually truncates a longer listing blocks deletion diff --git a/apps/sim/connectors/google-calendar/google-calendar.test.ts b/apps/sim/connectors/google-calendar/google-calendar.test.ts index 00964259747..dee0f6b747a 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.test.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { googleCalendarConnector } from '@/connectors/google-calendar/google-calendar' import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' const ORGANIZER_EMAIL = 'organizer@example.com' const ATTENDEE_EMAIL = 'attendee@example.com' @@ -58,6 +59,81 @@ async function listOne(sourceConfig: Record) { return result.documents[0] } +describe('google-calendar listDocuments with a calendar the caller cannot reach', () => { + function mockCalendars(unreachable: string) { + fetchMock.mockImplementation(async (input) => { + const url = String(input) + if (url.includes(`/calendars/${unreachable}/events?`)) { + return jsonResponse({ error: { code: 404 } }, 404) + } + if (url.includes('/events?')) return jsonResponse({ items: [EVENT] }) + throw new Error(`Unexpected fetch: ${url}`) + }) + } + + it("skips only the unreachable calendar under a member's own token", async () => { + mockCalendars('alpha') + const sourceConfig = { calendarId: 'alpha,beta' } + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const first = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + undefined, + syncContext + ) + expect(first.documents).toHaveLength(0) + expect(first.hasMore).toBe(true) + expect(JSON.parse(first.nextCursor ?? '{}')).toEqual({ calendarIndex: 1 }) + + const second = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + first.nextCursor, + syncContext + ) + expect(second.documents).toHaveLength(1) + expect(second.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('ends the listing when the unreachable calendar is the last one', async () => { + mockCalendars('beta') + const sourceConfig = { calendarId: 'alpha,beta' } + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const first = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + undefined, + syncContext + ) + const second = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + first.nextCursor, + syncContext + ) + expect(second.documents).toHaveLength(0) + expect(second.hasMore).toBe(false) + }) + + it('reports a sole unreachable calendar as the whole scope being unavailable', async () => { + mockCalendars('alpha') + const error = await googleCalendarConnector + .listDocuments('token', { calendarId: 'alpha' }, undefined, { ...PER_MEMBER_LISTING_CONTEXT }) + .catch((caught: unknown) => caught) + expect(googleCalendarConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('still fails the sync under a shared credential rather than dropping the calendar', async () => { + mockCalendars('alpha') + await expect( + googleCalendarConnector.listDocuments('token', { calendarId: 'alpha,beta' }, undefined, {}) + ).rejects.toThrow('Failed to list Google Calendar events: 404') + }) +}) + describe('google-calendar attendee PII opt-out', () => { it('exposes an includeAttendees config field defaulting to on', () => { const field = googleCalendarConnectorMeta.configFields.find((f) => f.id === 'includeAttendees') diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index d79eff2c868..1c61fbccb30 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -5,6 +5,7 @@ import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/go import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { isListingScopeUnavailableError, + isPerMemberListing, listingRequestError, parseMultiValue, parseTagDate, @@ -404,7 +405,33 @@ export const googleCalendarConnector: ConnectorConfig = { calendarId, error: errorText, }) - throw listingRequestError('Failed to list Google Calendar events', response.status) + const error = listingRequestError('Failed to list Google Calendar events', response.status) + /** + * One of several calendars a member cannot reach is absent from their + * listing, not the end of it: move on to the next calendar so the rest of + * their access survives. A sole unreachable calendar is the whole scope, + * which the members-mode crawl reads as a complete listing of nothing, and + * a shared credential still fails the sync rather than silently dropping + * the calendar's events. + */ + if ( + isListingScopeUnavailableError(error) && + calendarIds.length > 1 && + isPerMemberListing(syncContext) + ) { + logger.warn('Skipping a Google Calendar the member cannot reach', { + calendarId, + status: response.status, + }) + return calendarIndex + 1 < calendarIds.length + ? { + documents: [], + nextCursor: JSON.stringify({ calendarIndex: calendarIndex + 1 }), + hasMore: true, + } + : { documents: [], hasMore: false } + } + throw error } const data = await response.json() diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index 419986d9be2..44e16893f5f 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, normalizeAtlassianSiteUrl, } from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' @@ -140,8 +141,14 @@ function issueToFullDocument(issue: Record, siteUrl: string): E export const jiraConnector: ConnectorConfig = { ...jiraConnectorMeta, + /** + * A member whose token reaches no Atlassian site, or only sites other than + * the configured one, lists nothing: a complete listing of nothing, not an error. + */ isListingScopeUnavailableError: (error) => - isListingScopeUnavailableError(error) || error instanceof AtlassianSiteNotAccessibleError, + isListingScopeUnavailableError(error) || + error instanceof AtlassianSiteNotAccessibleError || + error instanceof AtlassianSiteNotMatchedError, listDocuments: async ( accessToken: string, diff --git a/apps/sim/connectors/jsm/jsm.test.ts b/apps/sim/connectors/jsm/jsm.test.ts new file mode 100644 index 00000000000..6aab2f5e135 --- /dev/null +++ b/apps/sim/connectors/jsm/jsm.test.ts @@ -0,0 +1,75 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ JiraServiceManagementIcon: () => null })) +vi.mock('@/tools/jira/utils', () => ({ + getJiraCloudId: vi.fn(), + extractAdfText: () => '', +})) + +import { AtlassianSiteNotMatchedError } from '@/lib/atlassian/discovery' +import { jsmConnector } from '@/connectors/jsm/jsm' + +const SOURCE_CONFIG = { domain: 'example.atlassian.net', serviceDeskId: '10' } + +function mockStatus(status: number) { + mockFetchWithRetry.mockResolvedValue({ + ok: status >= 200 && status < 300, + status, + json: async () => ({}), + text: async () => '', + } as unknown as Response) +} + +async function listingError(): Promise { + return jsmConnector + .listDocuments('token', SOURCE_CONFIG, undefined, { cloudId: 'cloud-1' }) + .catch((caught: unknown) => caught) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('jsm listing scope classification', () => { + it('treats a 403 on the request listing as a service desk the caller may not view', async () => { + mockStatus(403) + expect(jsmConnector.isListingScopeUnavailableError?.(await listingError())).toBe(true) + }) + + it('treats a 404 on the request listing as a service desk that does not exist for the caller', async () => { + mockStatus(404) + expect(jsmConnector.isListingScopeUnavailableError?.(await listingError())).toBe(true) + }) + + it('leaves other failures for the sync engines to retry', async () => { + mockStatus(500) + const error = await listingError() + expect(error).toBeInstanceOf(Error) + expect(jsmConnector.isListingScopeUnavailableError?.(error)).toBe(false) + }) + + it('treats a 403 while resolving a project key to a service desk id the same way', async () => { + mockStatus(403) + const error = await jsmConnector + .listDocuments('token', { ...SOURCE_CONFIG, serviceDeskId: 'ITH' }, undefined, { + cloudId: 'cloud-1', + }) + .catch((caught: unknown) => caught) + expect(jsmConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('treats a token that reaches only other Atlassian sites as not on the site', () => { + expect( + jsmConnector.isListingScopeUnavailableError?.(new AtlassianSiteNotMatchedError('elsewhere')) + ).toBe(true) + }) +}) diff --git a/apps/sim/connectors/jsm/jsm.ts b/apps/sim/connectors/jsm/jsm.ts index 7f7a493a061..a1a05066dca 100644 --- a/apps/sim/connectors/jsm/jsm.ts +++ b/apps/sim/connectors/jsm/jsm.ts @@ -1,6 +1,9 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { AtlassianSiteNotAccessibleError } from '@/lib/atlassian/discovery' +import { + AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, +} from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jsmConnectorMeta } from '@/connectors/jsm/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -311,6 +314,15 @@ async function resolveCloudId( return cloudId } +/** + * JSM answers a service desk the caller may not view with 403 and one that + * does not exist for them with 404; either is a complete listing of nothing + * for that caller. + */ +function isJsmScopeUnavailableStatus(status: number): boolean { + return status === 403 || status === 404 +} + /** * Resolves a configured service desk identifier to the numeric service desk id. * @@ -337,7 +349,11 @@ async function resolveServiceDeskId( }) if (!response.ok) { - throw new Error(`Failed to resolve service desk "${trimmed}": ${response.status}`) + throw listingRequestError( + `Failed to resolve service desk "${trimmed}"`, + response.status, + isJsmScopeUnavailableStatus(response.status) + ) } const data = (await response.json()) as { id?: string } @@ -427,8 +443,15 @@ async function fetchComments( export const jsmConnector: ConnectorConfig = { ...jsmConnectorMeta, + /** + * A member whose token reaches no Atlassian site, or only sites other than + * the configured one, or who may not view the configured service desk, lists + * nothing: a complete listing of nothing, not an error. + */ isListingScopeUnavailableError: (error) => - isListingScopeUnavailableError(error) || error instanceof AtlassianSiteNotAccessibleError, + isListingScopeUnavailableError(error) || + error instanceof AtlassianSiteNotAccessibleError || + error instanceof AtlassianSiteNotMatchedError, listDocuments: async ( accessToken: string, @@ -507,7 +530,11 @@ export const jsmConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list JSM requests', { status: response.status, error: errorText }) - throw listingRequestError('Failed to list JSM requests', response.status) + throw listingRequestError( + 'Failed to list JSM requests', + response.status, + isJsmScopeUnavailableStatus(response.status) + ) } const data = (await response.json()) as JsmPage diff --git a/apps/sim/connectors/monday/monday.test.ts b/apps/sim/connectors/monday/monday.test.ts index 525523704b7..7ab350c5cff 100644 --- a/apps/sim/connectors/monday/monday.test.ts +++ b/apps/sim/connectors/monday/monday.test.ts @@ -160,6 +160,80 @@ describe('monday listDocuments', () => { }) }) +describe('monday listDocuments with configured boards the caller cannot reach', () => { + it('reports the sole configured board coming back absent as the scope being unavailable', async () => { + mockMonday([{ body: { data: { boards: [] } } }]) + const syncContext: Record = {} + + const error = await mondayConnector + .listDocuments('token', { boardIds: '1' }, undefined, syncContext) + .catch((caught: unknown) => caught) + + expect(mondayConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('skips only the unreachable board when another configured board is reachable', async () => { + mockMonday([ + { body: { data: { boards: [] } } }, + { + body: { + data: { + boards: [ + { id: '2', name: 'Board Two', items_page: { cursor: null, items: [item('a')] } }, + ], + }, + }, + }, + ]) + const syncContext: Record = {} + + const first = await mondayConnector.listDocuments( + 'token', + { boardIds: '1,2' }, + undefined, + syncContext + ) + expect(first.documents).toHaveLength(0) + expect(first.hasMore).toBe(true) + + const second = await mondayConnector.listDocuments( + 'token', + { boardIds: '1,2' }, + first.nextCursor, + syncContext + ) + expect(second.documents.map((doc) => doc.externalId)).toEqual(['a']) + expect(second.hasMore).toBe(false) + }) + + it('reports every configured board coming back absent once the last one is walked', async () => { + mockMonday([{ body: { data: { boards: [] } } }, { body: { data: { boards: [] } } }]) + const syncContext: Record = {} + + const first = await mondayConnector.listDocuments( + 'token', + { boardIds: '1,2' }, + undefined, + syncContext + ) + expect(first.hasMore).toBe(true) + + const error = await mondayConnector + .listDocuments('token', { boardIds: '1,2' }, first.nextCursor, syncContext) + .catch((caught: unknown) => caught) + expect(mondayConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('does not doubt an enumerated board list, which only ever holds reachable boards', async () => { + mockMonday([{ body: { data: { boards: [] } } }]) + const syncContext: Record = {} + + const result = await mondayConnector.listDocuments('token', {}, undefined, syncContext) + expect(result.documents).toHaveLength(0) + expect(result.hasMore).toBe(false) + }) +}) + describe('monday content extraction', () => { it('falls back to display_value for columns that do not populate text', async () => { mockMonday([ diff --git a/apps/sim/connectors/monday/monday.ts b/apps/sim/connectors/monday/monday.ts index 87050cec2e1..0a50a7819e5 100644 --- a/apps/sim/connectors/monday/monday.ts +++ b/apps/sim/connectors/monday/monday.ts @@ -5,7 +5,12 @@ import { backoffWithJitter } from '@sim/utils/retry' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { mondayConnectorMeta } from '@/connectors/monday/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' const logger = createLogger('MondayConnector') @@ -449,6 +454,8 @@ async function resolveBoardIds( export const mondayConnector: ConnectorConfig = { ...mondayConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -505,6 +512,15 @@ export const mondayConnector: ConnectorConfig = { { ids: [board.id], limit: pageLimit } ) itemsPage = data.boards?.[0]?.items_page ?? null + /** + * `boards(ids:)` filters to the boards the token can see, so a configured + * board the caller cannot reach comes back absent rather than as an error. + * Counted so that a listing which reached none of its configured boards + * can say so below instead of passing as a board that happens to be empty. + */ + if (!data.boards?.length && syncContext) { + syncContext.unreachableBoardCount = ((syncContext.unreachableBoardCount as number) ?? 0) + 1 + } } const items = itemsPage?.items ?? [] @@ -558,6 +574,24 @@ export const mondayConnector: ConnectorConfig = { hasMore = true } + /** + * The caller reached none of the boards this connector is configured for. + * That is the configured scope being unavailable to them — under a member's + * own token a complete listing of nothing, so their access is withdrawn — + * not a set of boards that all happen to be empty. Enumerated boards need no + * such check: the enumeration only ever returns boards the token can see. + */ + if ( + !hasMore && + parseMultiValue(sourceConfig.boardIds).length > 0 && + syncContext?.unreachableBoardCount === boards.length + ) { + throw new ConnectorListingScopeUnavailableError( + `monday.com returned none of the configured boards (${boards.map((b) => b.id).join(', ')}); the account cannot reach them`, + 404 + ) + } + return { documents, nextCursor, hasMore } }, diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index 973a70972a9..7fe5c17a103 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -12,7 +12,6 @@ import { assertMicrosoftGraphNextLink, CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, - ConnectorListingScopeUnavailableError, connectorFileExtension, decodeMicrosoftGraphTraversalCursor, encodeMicrosoftGraphTraversalCursor, @@ -24,6 +23,7 @@ import { isSkippedDocument, type MicrosoftGraphTraversalState, markSkipped, + microsoftGraphListingError, parseMicrosoftGraphDriveItemList, parseOptionalUnlimitedSafeInteger, parseTagDate, @@ -81,19 +81,6 @@ function parseMaxFiles(value: unknown): number { ) } -/** - * The error a failed Graph listing request throws. Graph reports a folder the - * caller cannot reach as 404 (`itemNotFound`) or 403 (`accessDenied`); either - * is a complete listing of nothing for that caller, while anything else is a - * fault the sync engines retry. - */ -function graphListingError(message: string, status: number): Error { - const described = `${message}: ${status}` - return status === 403 || status === 404 - ? new ConnectorListingScopeUnavailableError(described, status) - : new Error(described) -} - interface OneDriveItem { id: string name: string @@ -280,7 +267,7 @@ export const onedriveConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw graphListingError('Failed to list OneDrive files', response.status) + throw microsoftGraphListingError('Failed to list OneDrive files', response.status) } const data = parseMicrosoftGraphDriveItemList(await response.json(), 'OneDrive') diff --git a/apps/sim/connectors/salesforce/meta.ts b/apps/sim/connectors/salesforce/meta.ts index 648c23d443b..0340bad70dc 100644 --- a/apps/sim/connectors/salesforce/meta.ts +++ b/apps/sim/connectors/salesforce/meta.ts @@ -21,6 +21,8 @@ export const salesforceConnectorMeta: ConnectorMeta = { requiredScopes: ['api', 'refresh_token', 'openid'], }, + /** Every synced object carries `LastModifiedDate`, which the listing filters on. */ + supportsIncrementalSync: true, permissionScopedListing: { capFieldIds: ['maxRecords'] }, configFields: [ { diff --git a/apps/sim/connectors/salesforce/salesforce.test.ts b/apps/sim/connectors/salesforce/salesforce.test.ts new file mode 100644 index 00000000000..6bf56c4ceba --- /dev/null +++ b/apps/sim/connectors/salesforce/salesforce.test.ts @@ -0,0 +1,124 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ SalesforceIcon: () => null })) + +import { salesforceConnector } from '@/connectors/salesforce/salesforce' + +const INSTANCE_URL = 'https://org.example.com/services/data/v62.0/' + +/** Answers every query with `status` and `body`, recording the requested URLs. */ +function mockQuery(status: number, body: unknown) { + const urls: string[] = [] + mockFetchWithRetry.mockImplementation(async (url: string) => { + urls.push(url) + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response + }) + return urls +} + +function syncContext(): Record { + return { instanceUrl: INSTANCE_URL } +} + +async function listingError(sourceConfig: Record): Promise { + return salesforceConnector + .listDocuments('token', sourceConfig, undefined, syncContext()) + .catch((caught: unknown) => caught) +} + +function soqlOf(url: string): string { + return decodeURIComponent(new URL(url).searchParams.get('q') ?? '') +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('salesforce listing scope classification', () => { + it('treats an object the caller may not read (400 INVALID_TYPE) as the scope being unavailable', async () => { + mockQuery(400, [ + { message: "sObject type 'Case' is not supported.", errorCode: 'INVALID_TYPE' }, + ]) + const error = await listingError({ objectType: 'Case' }) + expect(salesforceConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('treats an explicit denial (403 INSUFFICIENT_ACCESS) the same way', async () => { + mockQuery(403, [{ message: 'denied', errorCode: 'INSUFFICIENT_ACCESS_OR_READONLY' }]) + const error = await listingError({ objectType: 'Account' }) + expect(salesforceConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('leaves a malformed query and server faults for the sync engines to retry', async () => { + mockQuery(400, [{ message: 'unexpected token', errorCode: 'MALFORMED_QUERY' }]) + const malformed = await listingError({ objectType: 'Account' }) + expect(malformed).toBeInstanceOf(Error) + expect(salesforceConnector.isListingScopeUnavailableError?.(malformed)).toBe(false) + + mockQuery(500, 'Internal Server Error') + const fault = await listingError({ objectType: 'Account' }) + expect(fault).toBeInstanceOf(Error) + expect(salesforceConnector.isListingScopeUnavailableError?.(fault)).toBe(false) + }) +}) + +describe('salesforce incremental listing', () => { + it('advertises incremental sync', () => { + expect(salesforceConnector.supportsIncrementalSync).toBe(true) + }) + + it('lists the whole object when no watermark is given', async () => { + const urls = mockQuery(200, { records: [] }) + await salesforceConnector.listDocuments( + 'token', + { objectType: 'Case' }, + undefined, + syncContext() + ) + expect(soqlOf(urls[0])).toBe( + 'SELECT Id,Subject,Description,Status,LastModifiedDate,CaseNumber FROM Case ORDER BY LastModifiedDate DESC' + ) + }) + + it('filters on LastModifiedDate with an unquoted UTC literal after a watermark', async () => { + const urls = mockQuery(200, { records: [] }) + await salesforceConnector.listDocuments( + 'token', + { objectType: 'Case' }, + undefined, + syncContext(), + new Date('2026-09-01T12:34:56.789Z') + ) + expect(soqlOf(urls[0])).toContain( + ' FROM Case WHERE LastModifiedDate >= 2026-09-01T12:34:56Z ORDER BY' + ) + }) + + it('appends the watermark to the mandatory Knowledge Article filters', async () => { + const urls = mockQuery(200, { records: [] }) + await salesforceConnector.listDocuments( + 'token', + { objectType: 'KnowledgeArticleVersion' }, + undefined, + syncContext(), + new Date('2026-09-01T00:00:00Z') + ) + expect(soqlOf(urls[0])).toContain( + "WHERE PublishStatus='Online' AND IsLatestVersion=true AND Language='en_US' AND LastModifiedDate >= 2026-09-01T00:00:00Z ORDER BY" + ) + }) +}) diff --git a/apps/sim/connectors/salesforce/salesforce.ts b/apps/sim/connectors/salesforce/salesforce.ts index ef075fc004c..522e290d834 100644 --- a/apps/sim/connectors/salesforce/salesforce.ts +++ b/apps/sim/connectors/salesforce/salesforce.ts @@ -4,7 +4,12 @@ import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/document import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' import { salesforceConnectorMeta } from '@/connectors/salesforce/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + isListingScopeUnavailableError, + listingRequestError, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('SalesforceConnector') @@ -84,9 +89,52 @@ const OBJECT_FIELDS: Record = { * user-selectable locale — rather than relying on that hedge holding for the * abstract KnowledgeArticleVersion view. */ -function buildWhereClause(objectType: string, language: string): string { - if (objectType !== 'KnowledgeArticleVersion') return '' - return ` WHERE PublishStatus='Online' AND IsLatestVersion=true AND Language='${language}'` +function buildWhereClause(objectType: string, language: string, lastSyncAt?: Date): string { + const conditions: string[] = [] + if (objectType === 'KnowledgeArticleVersion') { + conditions.push("PublishStatus='Online'", 'IsLatestVersion=true', `Language='${language}'`) + } + if (lastSyncAt) conditions.push(`LastModifiedDate >= ${toSoqlDateTime(lastSyncAt)}`) + return conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '' +} + +/** + * A SOQL dateTime literal: ISO 8601 in UTC, unquoted, and without the + * fractional seconds SOQL does not accept. + */ +function toSoqlDateTime(date: Date): string { + return date.toISOString().replace(/\.\d{3}Z$/, 'Z') +} + +/** The `errorCode` values in a Salesforce REST error body (a JSON array of errors). */ +function parseSalesforceErrorCodes(errorText: string): string[] { + try { + const parsed: unknown = JSON.parse(errorText) + if (!Array.isArray(parsed)) return [] + return parsed.flatMap((entry: unknown) => + typeof entry === 'object' && + entry !== null && + typeof (entry as { errorCode?: unknown }).errorCode === 'string' + ? [(entry as { errorCode: string }).errorCode] + : [] + ) + } catch { + return [] + } +} + +/** + * Whether a failed query means the caller cannot read the configured object at + * all. Salesforce hides an object from a user who may not read it, so the query + * fails with 400 `INVALID_TYPE` rather than returning nothing, and an explicit + * denial is 403 `INSUFFICIENT_ACCESS`; either is a complete listing of nothing + * for that caller, while anything else is a fault the sync engines retry. + */ +function isSalesforceAccessDenied(status: number, errorText: string): boolean { + if (status !== 400 && status !== 403) return false + return parseSalesforceErrorCodes(errorText).some( + (code) => code === 'INVALID_TYPE' || code.startsWith('INSUFFICIENT_ACCESS') + ) } /** @@ -326,11 +374,14 @@ function recordToDocument( export const salesforceConnector: ConnectorConfig = { ...salesforceConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, - syncContext?: Record + syncContext?: Record, + lastSyncAt?: Date ): Promise => { const objectType = sourceConfig.objectType as string const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0 @@ -347,7 +398,11 @@ export const salesforceConnector: ConnectorConfig = { if (cursor) { url = `${toOrigin(instanceUrl)}${cursor}` } else { - const whereClause = buildWhereClause(objectType, resolveArticleLanguage(sourceConfig)) + const whereClause = buildWhereClause( + objectType, + resolveArticleLanguage(sourceConfig), + lastSyncAt + ) /** * No SOQL `LIMIT`: it bounds the total result set rather than the batch, * so it would end the sync after a single page. Paging is driven by @@ -359,7 +414,10 @@ export const salesforceConnector: ConnectorConfig = { url = `${instanceUrl}query?q=${encodeURIComponent(soql)}` } - logger.info(`Listing Salesforce ${objectType}`, { cursor: cursor || 'initial' }) + logger.info(`Listing Salesforce ${objectType}`, { + cursor: cursor || 'initial', + incremental: Boolean(lastSyncAt), + }) const response = await fetchWithRetry(url, { method: 'GET', @@ -375,7 +433,11 @@ export const salesforceConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to query Salesforce ${objectType}: ${response.status}`) + throw listingRequestError( + `Failed to query Salesforce ${objectType}`, + response.status, + isSalesforceAccessDenied(response.status, errorText) + ) } const data = await response.json() diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index 2accd1caf4a..9d14a62f949 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -25,6 +25,7 @@ import { isSkippedDocument, type MicrosoftGraphTraversalState, markSkipped, + microsoftGraphListingError, parseMicrosoftGraphDriveItemList, parseOptionalUnlimitedSafeInteger, parseTagDate, @@ -76,19 +77,6 @@ function parseMaxFiles(value: unknown): number { ) } -/** - * The error a failed Graph request for the configured site, library, or folder - * throws. Graph reports a scope the caller cannot reach as 404 (`itemNotFound`) - * or 403 (`accessDenied`); either is a complete listing of nothing for that - * caller, while anything else is a fault the sync engines retry. - */ -function graphListingError(message: string, status: number, detail?: string): Error { - const described = detail ? `${message}: ${status} – ${detail}` : `${message}: ${status}` - return status === 403 || status === 404 - ? new ConnectorListingScopeUnavailableError(described, status) - : new Error(described) -} - /** Microsoft Graph drive item shape (subset of fields we use). */ interface DriveItem { id: string @@ -231,7 +219,7 @@ async function resolveSiteId( if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw graphListingError( + throw microsoftGraphListingError( `Failed to resolve SharePoint site "${siteUrl}"`, response.status, errorText @@ -340,7 +328,7 @@ async function listFolderItems( if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw graphListingError('Failed to list folder items', response.status, errorText) + throw microsoftGraphListingError('Failed to list folder items', response.status, errorText) } const data = parseMicrosoftGraphDriveItemList(await response.json(), 'SharePoint') @@ -420,7 +408,7 @@ async function getItemByPath( if (response.status === 404) return null if (!response.ok) { - throw graphListingError('Failed to resolve folder path', response.status) + throw microsoftGraphListingError('Failed to resolve folder path', response.status) } return (await response.json()) as DriveItem @@ -444,7 +432,7 @@ async function listChildFolders( const response = await graphGet(url, accessToken, retryOptions) if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw graphListingError('Failed to list folder contents', response.status, errorText) + throw microsoftGraphListingError('Failed to list folder contents', response.status, errorText) } const rawData: unknown = await response.json() @@ -581,7 +569,7 @@ export async function resolveFolderTarget( retryOptions ) if (!defaultDriveResponse.ok) { - throw graphListingError( + throw microsoftGraphListingError( `Failed to open the default document library for site "${siteUrl}"`, defaultDriveResponse.status ) @@ -695,7 +683,7 @@ async function listSiteDrives( const response = await graphGet(url, accessToken, retryOptions) if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw graphListingError( + throw microsoftGraphListingError( 'Failed to list SharePoint document libraries', response.status, errorText diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 4eb5b2f5473..4d25f214c13 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -732,3 +732,33 @@ export function listingRequestError( export function isListingScopeUnavailableError(error: unknown): boolean { return error instanceof ConnectorListingScopeUnavailableError } + +/** + * The error a failed Microsoft Graph listing request throws. Graph reports a + * drive, site, or folder the caller cannot reach as 404 (`itemNotFound`) or + * 403 (`accessDenied`); either is a complete listing of nothing for that + * caller, while anything else is a fault the sync engines retry. + */ +export function microsoftGraphListingError( + message: string, + status: number, + detail?: string +): Error { + const described = detail ? `${message}: ${status} – ${detail}` : `${message}: ${status}` + return status === 403 || status === 404 + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + +/** + * `syncContext` entry the members-mode crawl sets on every listing it runs + * under one member's own token. A connector that walks several scopes reads + * it to tell that a scope the caller cannot reach is simply absent from that + * member's complete listing, where the same failure under a shared credential + * is a cap or an error. + */ +export const PER_MEMBER_LISTING_CONTEXT = { perMemberListing: true } as const + +export function isPerMemberListing(syncContext: Record | undefined): boolean { + return syncContext?.perMemberListing === true +} diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index b26cc7e853a..bae9fc443b4 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -83,6 +83,7 @@ import type { SyncResult, SyncSkipReason, } from '@/connectors/types' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' const logger = createLogger('ConnectorMemberSyncEngine') @@ -1399,7 +1400,11 @@ export async function executeMemberSync( const member = await claimNextMember(run) if (!member) break result.membersClaimed += 1 - const syncContext: Record = { syncRunId: runId, memberId: member.id } + const syncContext: Record = { + syncRunId: runId, + memberId: member.id, + ...PER_MEMBER_LISTING_CONTEXT, + } syncContexts.set(member.id, syncContext) const listed = await listForMember({ From 84123acd07d3253e5bf4032cccea34df3d167061 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 02:10:43 -0700 Subject: [PATCH 42/76] feat(search): make a search a shareable link The composer's Search-mode query lives in the URL as q, so a search can be bookmarked or sent; opening such a link restores the query and Search mode. --- .../app/workspace/[workspaceId]/home/home.tsx | 24 ++++++++++++++++--- .../[workspaceId]/home/search-params.ts | 11 +++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 13527b2e5dd..a38f075b0cd 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -49,7 +49,11 @@ import { resolveResourceEventPresentation, resolveResourceSelectionUpdate, } from '@/app/workspace/[workspaceId]/home/resource-view-policy' -import { resourceParam, resourceUrlKeys } from '@/app/workspace/[workspaceId]/home/search-params' +import { + resourceParam, + resourceUrlKeys, + searchQueryParam, +} from '@/app/workspace/[workspaceId]/home/search-params' import { useFolders } from '@/hooks/queries/folders' import { useMarkMothershipChatRead } from '@/hooks/queries/mothership-chats' import { useWorkflows } from '@/hooks/queries/workflows' @@ -155,7 +159,21 @@ export function Home({ chatId, userName, userId }: HomeProps) { const posthogRef = useRef(posthog) posthogRef.current = posthog const [initialPrompt, setInitialPrompt] = useState('') - const [searchQuery, setSearchQuery] = useState('') + /** The search query lives in the URL so a search is a shareable link; null between searches. */ + const [searchQueryValue, setSearchQueryParam] = useQueryState(searchQueryParam.key, { + ...searchQueryParam.parser, + ...resourceUrlKeys, + }) + const searchQuery = searchQueryValue ?? '' + const setSearchQuery = useCallback( + (query: string) => void setSearchQueryParam(query || null), + [setSearchQueryParam] + ) + /** A link that carries a query opens in Search mode with the query in the box. */ + const [initialSearchQuery] = useState(searchQuery) + useEffect(() => { + if (initialSearchQuery) useMothershipModeStore.getState().setMode('search') + }, [initialSearchQuery]) const composerMode = useMothershipModeStore((state) => state.mode) const hasCheckedLandingStorageRef = useRef(false) const initialViewInputRef = useRef(null) @@ -705,7 +723,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { > Date: Wed, 2 Sep 2026 02:15:41 -0700 Subject: [PATCH 43/76] test(knowledge): pin member-access availability in the v1 search route test --- apps/sim/app/api/v1/knowledge/search/route.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index 659739bd027..40747a907a4 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -44,6 +44,11 @@ const SYSTEM_BILLING_ATTRIBUTION = { payerSubscription: null, } +/** The route's defaults depend on member-access availability; pin it so the local flag cannot change the expectations. */ +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: async () => false, +})) + vi.mock('@/lib/knowledge/search/queries', () => ({ executeKnowledgeSearch: mockExecuteKnowledgeSearch, getDocumentMetadataByIds: mockGetDocumentMetadataByIds, From d6968e4d15f2aee90bf7a032cd86baf071934da1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 02:24:09 -0700 Subject: [PATCH 44/76] refactor(search): apply the cleanup passes to the Sim Search surface Drop three memos over cheap derivations, make the search setter's callers honest about their dependencies, ignore a whitespace-only query from a link, give a dropdown setup field its hint, and drop the query from the URL when the composer leaves Search. --- .../knowledge-search-results.tsx | 29 +++++++------------ .../search-sources/search-sources.tsx | 13 ++++----- .../search-sources/source-setup-modal.tsx | 1 + .../mode-switcher/mode-switcher.test.tsx | 20 ++++++++++++- .../mode-switcher/mode-switcher.tsx | 6 ++++ .../app/workspace/[workspaceId]/home/home.tsx | 8 ++--- .../[workspaceId]/home/search-params.ts | 7 +++-- 7 files changed, 50 insertions(+), 34 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 2687438dbe9..1e30a08c2e9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -118,14 +118,10 @@ export function KnowledgeSearchResults({ * The list also carries the viewer's legacy personal bases, which have no * workspace; a search names one workspace and refuses a base outside it. */ - const knowledgeBaseIds = useMemo( - () => - knowledgeBases - .filter((kb) => kb.workspaceId === workspaceId) - .slice(0, MAX_SEARCHED_KNOWLEDGE_BASES) - .map((kb) => kb.id), - [knowledgeBases, workspaceId] - ) + const knowledgeBaseIds = knowledgeBases + .filter((kb) => kb.workspaceId === workspaceId) + .slice(0, MAX_SEARCHED_KNOWLEDGE_BASES) + .map((kb) => kb.id) const { data: results, isPending, @@ -134,16 +130,13 @@ export function KnowledgeSearchResults({ } = useWorkspaceKnowledgeSearch(workspaceId, knowledgeBaseIds, query) const { data: memberConnectors = [] } = useWorkspaceMemberConnectors(workspaceId) /** Every per-member connector still indexing for the viewer, in any base the search spans. */ - const indexing = useMemo( - () => [ - ...new Set( - memberConnectors - .filter(isIndexing) - .map((connection) => connectorName(connection.connectorType)) - ), - ], - [memberConnectors] - ) + const indexing = [ + ...new Set( + memberConnectors + .filter(isIndexing) + .map((connection) => connectorName(connection.connectorType)) + ), + ] const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) const sourceTypes = useMemo( () => [...new Set(documents.map((result) => result.connectorType ?? 'upload'))], diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx index 7446053dd75..fbbb1d935b1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx @@ -169,14 +169,11 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { }) const [setupConnector, setSetupConnector] = useState(null) - /** Connected sources first, then the rest alphabetically. */ - const ordered = useMemo(() => { - const rank = (connector: SearchConnector) => - connectionByType.get(connector.type)?.viewerMembership === 'connected' ? 0 : 1 - return [...PERSONAL_SEARCH_CONNECTORS].sort( - (a, b) => rank(a) - rank(b) || a.meta.name.localeCompare(b.meta.name) - ) - }, [connectionByType]) + const rank = (connector: SearchConnector) => + connectionByType.get(connector.type)?.viewerMembership === 'connected' ? 0 : 1 + const ordered = [...PERSONAL_SEARCH_CONNECTORS].sort( + (a, b) => rank(a) - rank(b) || a.meta.name.localeCompare(b.meta.name) + ) const startConnect = (connector: SearchConnector) => { const connection = connectionByType.get(connector.type) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx index 71695341983..e322856f152 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx @@ -57,6 +57,7 @@ export function SourceSetupModal({ label: option.label, }))} placeholder={field.placeholder} + hint={field.description} required /> ) : ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx index f44dbb101a5..f118138a58c 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx @@ -5,11 +5,15 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCaptureEvent } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn() })) +const { mockCaptureEvent, mockSetSearchQuery } = vi.hoisted(() => ({ + mockCaptureEvent: vi.fn(), + mockSetSearchQuery: vi.fn(), +})) vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), })) +vi.mock('nuqs', () => ({ useQueryState: () => [null, mockSetSearchQuery] })) vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) @@ -94,6 +98,20 @@ describe('ModeSwitcher', () => { workspace_id: 'workspace-1', mode: 'search', }) + expect(mockSetSearchQuery).not.toHaveBeenCalled() + }) + + it('drops the search query from the URL when leaving Search', () => { + useMothershipModeStore.getState().setMode('search') + mount() + openMenu() + + act(() => { + items()[0].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) + + expect(useMothershipModeStore.getState().mode).toBe('build') + expect(mockSetSearchQuery).toHaveBeenCalledWith(null, { history: 'replace', scroll: false }) }) it('does not report re-selecting the active mode', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx index 60d58ada236..cb82323125e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -11,8 +11,10 @@ import { } from '@sim/emcn' import { Check } from '@sim/emcn/icons' import { useParams } from 'next/navigation' +import { useQueryState } from 'nuqs' import { usePostHog } from 'posthog-js/react' import { captureEvent } from '@/lib/posthog/client' +import { searchQueryParam } from '@/app/workspace/[workspaceId]/home/search-params' import { MOTHERSHIP_MODES, type MothershipMode, @@ -37,9 +39,13 @@ export const ModeSwitcher = memo(function ModeSwitcher() { const mode = useMothershipModeStore((state) => state.mode) const setMode = useMothershipModeStore((state) => state.setMode) + const [, setSearchQueryParam] = useQueryState(searchQueryParam.key, searchQueryParam.parser) + + /** Leaving Search drops the query from the URL, so a clean URL always means no search is showing. */ const handleSelect = (next: MothershipMode) => { if (next === mode) return setMode(next) + if (next !== 'search') void setSearchQueryParam(null, { history: 'replace', scroll: false }) captureEvent(posthog, 'chat_mode_changed', { workspace_id: workspaceId, mode: next }) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index a38f075b0cd..c1a08225a40 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -475,11 +475,11 @@ export function Home({ chatId, userName, userId }: HomeProps) { prepareResourceViewForAgentTurn() sendMessage(trimmed || 'Analyze the attached file(s).', fileAttachments, contexts) }, - [workspaceId, chatId, prepareResourceViewForAgentTurn, sendMessage] + [workspaceId, chatId, prepareResourceViewForAgentTurn, sendMessage, setSearchQuery] ) /** An emptied search box returns to the sources; nothing else reads the cleared query. */ - const clearSearch = useCallback(() => setSearchQuery(''), []) + const clearSearch = useCallback(() => setSearchQuery(''), [setSearchQuery]) /** Summarize on a result: hand the document to the agent in Build mode. */ const handleSummarize = useCallback( @@ -488,7 +488,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { setSearchQuery('') handleSubmit(prompt) }, - [handleSubmit] + [handleSubmit, setSearchQuery] ) /** * A chat that already exists opens in Build: its transcript is a @@ -498,7 +498,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { useEffect(() => { if (chatId) useMothershipModeStore.getState().setMode('build') }, [chatId]) - const showSearchResults = composerMode === 'search' && searchQuery.length > 0 + const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 const searchResults = showSearchResults ? ( Date: Wed, 2 Sep 2026 02:39:22 -0700 Subject: [PATCH 45/76] refactor(search): apply the simplify passes to the Sim Search surface One availability gate for every members-mode refusal, the shared role gate behind the admin-first message, setup fields precomputed on the catalog, the connect flow owned by the enrollment hook and shared by the strip and the Search page, the latest attempt derived instead of cross-resets, a trailing adornment slot on the chip in place of a loader wrapper, the processor's source access as one object with the connector-row rule in one place, and the first connect's independent lookups run together after its synchronous checks. --- apps/sim/app/api/files/authorization.ts | 9 ++- .../knowledge-search-results.tsx | 12 +-- .../home/components/search-sources/index.ts | 2 +- .../search-sources/search-sources.tsx | 76 ++++++++----------- .../search-sources/source-setup-modal.tsx | 28 ++++--- .../member-connectors-section.tsx | 8 +- .../[workspaceId]/search/search.test.tsx | 37 +++++++-- .../workspace/[workspaceId]/search/search.tsx | 54 +++++-------- apps/sim/hooks/use-member-enrollment.ts | 42 +++++++--- apps/sim/lib/knowledge/access/availability.ts | 12 +++ .../knowledge/application/connector-access.ts | 9 +-- .../knowledge/application/sim-search.test.ts | 16 +++- .../lib/knowledge/application/sim-search.ts | 39 +++++----- .../document-processing-source.test.ts | 11 +-- .../knowledge/documents/document-processor.ts | 8 +- .../documents/pdf-ocr-triage.test.ts | 5 +- apps/sim/lib/knowledge/documents/service.ts | 34 +++++---- .../orchestration/connector-access.test.ts | 16 +++- .../orchestration/connector-access.ts | 9 +-- apps/sim/lib/sim-search/connectors.test.ts | 4 + apps/sim/lib/sim-search/connectors.ts | 27 +++++-- packages/emcn/src/components/chip/chip.tsx | 9 ++- 22 files changed, 262 insertions(+), 205 deletions(-) diff --git a/apps/sim/app/api/files/authorization.ts b/apps/sim/app/api/files/authorization.ts index db10d3b7820..dbac5ed031e 100644 --- a/apps/sim/app/api/files/authorization.ts +++ b/apps/sim/app/api/files/authorization.ts @@ -491,10 +491,13 @@ async function verifyCopilotFileAccess( * signal only: it reflects whether the file is still part of a live KB, not who * owns it (ownership comes from the binding). */ +/** A reader once resolved: a person's or the workspace's tokens, or the system reading its own rows. */ +type ResolvedKnowledgeFileAccess = KnowledgeAccessScope | SystemAccessScope + async function hasActiveKbDocumentForKey( cloudKey: string, workspaceId: string, - access: KnowledgeAccessScope | SystemAccessScope + access: ResolvedKnowledgeFileAccess ): Promise { const rows = await db .select({ id: document.id }) @@ -525,13 +528,13 @@ async function hasActiveKbDocumentForKey( * else — an internal token, a tool running with the workflow owner's id — * reads as the workspace, never as the person whose id it happens to carry. */ -export type KnowledgeFileAccess = 'user' | KnowledgeAccessScope | SystemAccessScope +export type KnowledgeFileAccess = 'user' | ResolvedKnowledgeFileAccess async function resolveKnowledgeFileAccess( knowledgeAccess: KnowledgeFileAccess | undefined, userId: string, workspaceId: string -): Promise { +): Promise { if (knowledgeAccess === 'user') return resolveUserKnowledgeAccessScope(userId, workspaceId) return knowledgeAccess ?? WORKSPACE_ACCESS_SCOPE } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 1e30a08c2e9..4468cce2d59 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -3,10 +3,10 @@ import { useMemo, useState } from 'react' import { Button, Chip } from '@sim/emcn' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' +import { connectorDisplayName } from '@/lib/sim-search/connectors' import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import { isIndexing } from '@/app/workspace/[workspaceId]/home/components/search-sources' -import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { useWorkspaceMemberConnectors } from '@/hooks/queries/kb/connectors' import { useKnowledgeBasesQuery, useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' @@ -58,7 +58,7 @@ function toSource(result: WorkspaceKnowledgeSearchResult): SourceTagData | null url: result.sourceUrl, title: result.documentName ?? undefined, siteName: result.connectorType - ? connectorName(result.connectorType) + ? connectorDisplayName(result.connectorType) : result.knowledgeBaseName || undefined, connectorType: result.connectorType ?? undefined, snippet: toSnippet(result.content), @@ -66,10 +66,6 @@ function toSource(result: WorkspaceKnowledgeSearchResult): SourceTagData | null } } -function connectorName(connectorType: string): string { - return CONNECTOR_META_REGISTRY[connectorType]?.name ?? connectorType -} - /** * Arrow keys walk the result links, the way a search page does; Enter on a * focused link opens it natively. Focus stops at either end. @@ -134,7 +130,7 @@ export function KnowledgeSearchResults({ ...new Set( memberConnectors .filter(isIndexing) - .map((connection) => connectorName(connection.connectorType)) + .map((connection) => connectorDisplayName(connection.connectorType)) ), ] const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) @@ -203,7 +199,7 @@ export function KnowledgeSearchResults({ active={sourceFilter === type} onClick={() => setSourceFilter(sourceFilter === type ? null : type)} > - {type === 'upload' ? 'Uploads' : connectorName(type)} + {type === 'upload' ? 'Uploads' : connectorDisplayName(type)} ))} · diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts index 9ff66d4c312..daff1550541 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/index.ts @@ -1 +1 @@ -export { isIndexing, SearchSources, simSearchConnectionsByType } from './search-sources' +export { isIndexing, SearchSources } from './search-sources' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx index fbbb1d935b1..38806d49d18 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx @@ -1,15 +1,14 @@ 'use client' -import { useMemo, useState } from 'react' +import { useMemo } from 'react' import { Chip } from '@sim/emcn' import { Loader, Plus } from '@sim/emcn/icons' import { canConnectPersonally, - isSearchConnectorAvailable, - personalSetupFields, SEARCH_CONNECTORS, type SearchConnector, SIM_SEARCH_KNOWLEDGE_BASE_NAME, + searchConnectorUnavailableReason, } from '@/lib/sim-search/connectors' import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' @@ -23,20 +22,14 @@ import { CONNECTABLE_MEMBERSHIPS, useMemberEnrollment } from '@/hooks/use-member import { usePermissionConfig } from '@/hooks/use-permission-config' const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] -const MEMBER_ACCESS_UNAVAILABLE = 'Per-member access is not available in this workspace' /** The sources a person can connect themselves, alphabetical. */ const PERSONAL_SEARCH_CONNECTORS = SEARCH_CONNECTORS.filter((connector) => canConnectPersonally(connector.meta) ) -/** A chip's trailing icon is a component, so the spinning loader needs a wrapper to carry `animate`. */ -function SpinningLoader({ className }: { className?: string }) { - return -} - /** The Sim Search connection per source, keyed by connector type. */ -export function simSearchConnectionsByType( +function simSearchConnectionsByType( connectors: readonly WorkspaceMemberConnector[] ): Map { const byType = new Map() @@ -108,6 +101,7 @@ function SourceChip({ const title = unavailableReason ?? (connected ? `${connector.meta.name}: ${state}` : `Connect ${connector.meta.name}`) + const busy = waiting || isIndexing(connection) return ( } - rightIcon={waiting || isIndexing(connection) ? SpinningLoader : actionable ? Plus : undefined} + rightIcon={!busy && actionable ? Plus : undefined} + rightAdornment={ + busy ? : undefined + } > {connector.meta.name} @@ -163,50 +160,42 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { [memberConnectors] ) const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) - const { connect, connectSource, isAwaiting, isPending, error } = useMemberEnrollment({ - membershipQueryKeys, - connectedConnectorIds, - }) - const [setupConnector, setSetupConnector] = useState(null) + const { + connectSource, + connectSearchSource, + setupConnector, + closeSetup, + isAwaiting, + isPending, + error, + } = useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds }) - const rank = (connector: SearchConnector) => - connectionByType.get(connector.type)?.viewerMembership === 'connected' ? 0 : 1 - const ordered = [...PERSONAL_SEARCH_CONNECTORS].sort( - (a, b) => rank(a) - rank(b) || a.meta.name.localeCompare(b.meta.name) - ) - - const startConnect = (connector: SearchConnector) => { - const connection = connectionByType.get(connector.type) - if (connection) { - connect(connection.knowledgeBaseId, connection.connectorId) - return - } - if (personalSetupFields(connector.meta).length > 0) { - setSetupConnector(connector) - return - } - connectSource(workspaceId, connector.type) - } + /** Connected sources first; the catalog is already alphabetical, so the partition keeps the order. */ + const isConnected = (connector: SearchConnector) => + connectionByType.get(connector.type)?.viewerMembership === 'connected' + const ordered = [ + ...PERSONAL_SEARCH_CONNECTORS.filter(isConnected), + ...PERSONAL_SEARCH_CONNECTORS.filter((connector) => !isConnected(connector)), + ] return (
{ordered.map((connector) => { const connection = connectionByType.get(connector.type) - const unavailableReason = !isSearchConnectorAvailable(connector, integrationAvailability) - ? `${connector.meta.name} is unavailable in this deployment` - : memberAccessAvailable - ? null - : MEMBER_ACCESS_UNAVAILABLE return ( startConnect(connector)} + onConnect={() => connectSearchSource(workspaceId, connector, connection)} /> ) })} @@ -215,10 +204,7 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { {setupConnector && ( { - if (!open) setSetupConnector(null) - }} + onClose={closeSetup} onConnect={(sourceConfig) => connectSource(workspaceId, setupConnector.type, sourceConfig) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx index e322856f152..3537ebec9ed 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx @@ -9,12 +9,10 @@ import { ChipModalHeader, } from '@sim/emcn' import type { SearchConnector } from '@/lib/sim-search/connectors' -import type { ConnectorConfigField } from '@/connectors/types' interface SourceSetupModalProps { connector: SearchConnector - fields: readonly ConnectorConfigField[] - onOpenChange: (open: boolean) => void + onClose: () => void /** Connects the source with the filled-in fields; the caller opens the OAuth tab in this click. */ onConnect: (sourceConfig: Record) => void } @@ -23,26 +21,26 @@ interface SourceSetupModalProps { * The few fields a source needs before its first connect, such as a site and * a space. Everyone after the first person clicks straight through. */ -export function SourceSetupModal({ - connector, - fields, - onOpenChange, - onConnect, -}: SourceSetupModalProps) { +export function SourceSetupModal({ connector, onClose, onConnect }: SourceSetupModalProps) { + const fields = connector.setupFields const [values, setValues] = useState>({}) const complete = fields.every((field) => values[field.id]?.trim()) const submit = () => { if (!complete) return onConnect(Object.fromEntries(fields.map((field) => [field.id, values[field.id]?.trim() ?? '']))) - onOpenChange(false) + onClose() } return ( - - onOpenChange(false)}> - Connect {connector.meta.name} - + { + if (!open) onClose() + }} + srTitle={`Connect ${connector.meta.name}`} + > + Connect {connector.meta.name} {fields.map((field) => field.type === 'dropdown' ? ( @@ -76,7 +74,7 @@ export function SourceSetupModal({ )} onOpenChange(false)} + onCancel={onClose} primaryAction={{ label: 'Connect', onClick: submit, disabled: !complete }} /> diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx index 520370b7907..43ee669f90c 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx @@ -2,6 +2,7 @@ import { useMemo } from 'react' import { Button } from '@sim/emcn' +import { connectorDisplayName } from '@/lib/sim-search/connectors' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' import { RESOURCE_LIST_STACK, @@ -19,11 +20,6 @@ import { const SHARED_WITH_YOU_LABEL = 'Shared with you' -/** The name a per-member connector shows, from its registry entry. */ -export function memberConnectorName(connector: WorkspaceMemberConnector): string { - return CONNECTOR_META_REGISTRY[connector.connectorType]?.name ?? connector.connectorType -} - interface MemberConnectorsSectionProps { workspaceId: string /** The per-member connectors to show, already narrowed by the page's search. */ @@ -59,7 +55,7 @@ export function MemberConnectorsSection({ workspaceId, connectors }: MemberConne
{connectors.map((connector) => { const meta = CONNECTOR_META_REGISTRY[connector.connectorType] - const name = memberConnectorName(connector) + const name = connectorDisplayName(connector.connectorType) const waiting = isAwaiting(connector.connectorId) const state = describeMembership({ diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx index 347bd2e6a22..e29ad85e5ee 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx @@ -62,15 +62,28 @@ vi.mock('@/lib/sim-search/connectors', () => { serviceName: name, serviceIcon: icon, blockType: type, + setupFields: [], }) + const isSearchConnectorAvailable = ( + candidate: { blockType: string }, + availability: ReadonlyMap + ) => availability.get(candidate.blockType)?.oauthAvailable ?? true return { SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', canConnectPersonally: (meta: { permissionScopedListing?: unknown }) => Boolean(meta.permissionScopedListing), - isSearchConnectorAvailable: ( - candidate: { blockType: string }, - availability: ReadonlyMap - ) => availability.get(candidate.blockType)?.oauthAvailable ?? true, + connectorDisplayName: (connectorType: string) => connectorType, + isSearchConnectorAvailable, + searchConnectorUnavailableReason: ( + candidate: { blockType: string; meta: { name: string } }, + availability: ReadonlyMap, + memberAccessAvailable: boolean + ) => + !isSearchConnectorAvailable(candidate, availability) + ? `${candidate.meta.name} is unavailable in this deployment` + : memberAccessAvailable + ? null + : 'Per-member access is not available in this workspace', SEARCH_CONNECTORS: [ connector('google_drive', 'Google Drive', 'Sync Drive files', true), connector('confluence', 'Confluence', 'Sync Confluence pages', false), @@ -116,6 +129,16 @@ vi.mock('@/hooks/use-member-enrollment', async () => { useMemberEnrollment: () => ({ connect: mockConnect, connectSource: mockConnectSource, + connectSearchSource: ( + workspaceId: string, + connector: { type: string }, + connection: { knowledgeBaseId: string; connectorId: string } | undefined + ) => + connection + ? mockConnect(connection.knowledgeBaseId, connection.connectorId) + : mockConnectSource(workspaceId, connector.type), + setupConnector: null, + closeSetup: () => {}, isAwaiting: () => false, isPending: false, error: null, @@ -167,7 +190,7 @@ describe('Search', () => { const text = container?.textContent ?? '' expect(text).toContain('Connected · 12 documents') expect(text).toContain('Set up by a workspace admin from a knowledge base.') - expect(text).toContain('Unavailable in this deployment. Contact your administrator.') + expect(text).toContain('Slack is unavailable in this deployment') expect(text).toContain('Sales') }) @@ -189,9 +212,7 @@ describe('Search', () => { expect(sectionLabels()).toEqual(['Sim Search Connectors']) const text = container?.textContent ?? '' - expect(text).toContain( - 'Per-member access is not available in this workspace. Contact your administrator.' - ) + expect(text).toContain('Per-member access is not available in this workspace') expect(text).not.toContain('Connected · 12 documents') expect(buttons().find((button) => button.textContent === 'Connect')).toBeUndefined() }) diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx index 34009c338f6..f456582a604 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -1,17 +1,17 @@ 'use client' -import { useMemo, useRef, useState } from 'react' +import { useMemo, useRef } from 'react' import { Button, ChipInput } from '@sim/emcn' import { Search as SearchIcon } from '@sim/emcn/icons' import { useParams } from 'next/navigation' import { useQueryState } from 'nuqs' import { canConnectPersonally, - isSearchConnectorAvailable, - personalSetupFields, + connectorDisplayName, SEARCH_CONNECTORS, type SearchConnector, SIM_SEARCH_KNOWLEDGE_BASE_NAME, + searchConnectorUnavailableReason, } from '@/lib/sim-search/connectors' import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/components' import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' @@ -19,10 +19,7 @@ import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/c import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' -import { - MemberConnectorsSection, - memberConnectorName, -} from '@/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section' +import { MemberConnectorsSection } from '@/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section' import { connectorSearchParam, connectorSearchUrlKeys, @@ -46,9 +43,6 @@ import { usePermissionConfig } from '@/hooks/use-permission-config' const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] const CONNECTORS_LABEL = 'Sim Search Connectors' const NEEDS_KNOWLEDGE_BASE_SETUP = 'Set up by a workspace admin from a knowledge base.' -const UNAVAILABLE = 'Unavailable in this deployment. Contact your administrator.' -const MEMBER_ACCESS_UNAVAILABLE = - 'Per-member access is not available in this workspace. Contact your administrator.' /** What a source row says once the viewer's own indexing has settled. */ function connectedDescription(connector: WorkspaceMemberConnector): string { @@ -179,8 +173,15 @@ export function Search() { [memberConnectors] ) const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) - const [setupConnector, setSetupConnector] = useState(null) - const { connect, connectSource, isAwaiting, isPending, error } = useMemberEnrollment({ + const { + connectSource, + connectSearchSource, + setupConnector, + closeSetup, + isAwaiting, + isPending, + error, + } = useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds, }) @@ -195,7 +196,7 @@ export function Search() { : SEARCH_CONNECTORS const visibleSharedConnectors = normalizedSearch ? sharedConnectors.filter((connector) => - [memberConnectorName(connector), connector.knowledgeBaseName].some((text) => + [connectorDisplayName(connector.connectorType), connector.knowledgeBaseName].some((text) => text.toLowerCase().includes(normalizedSearch) ) ) @@ -226,29 +227,19 @@ export function Search() { {visibleConnectors.map((connector) => { const connection = connectionByType.get(connector.type) - const unavailableReason = !isSearchConnectorAvailable( - connector, - integrationAvailability - ) - ? UNAVAILABLE - : memberAccessAvailable - ? null - : MEMBER_ACCESS_UNAVAILABLE return ( - connection - ? connect(connection.knowledgeBaseId, connection.connectorId) - : personalSetupFields(connector.meta).length > 0 - ? setSetupConnector(connector) - : connectSource(workspaceId, connector.type) - } + onConnect={() => connectSearchSource(workspaceId, connector, connection)} /> ) })} @@ -266,10 +257,7 @@ export function Search() { {setupConnector && ( { - if (!open) setSetupConnector(null) - }} + onClose={closeSetup} onConnect={(sourceConfig) => connectSource(workspaceId, setupConnector.type, sourceConfig) } diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index 69375631755..cf99314925a 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -4,11 +4,13 @@ import { useEffect, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { type QueryKey, useQueryClient } from '@tanstack/react-query' import type { MemberSyncStatus } from '@/lib/knowledge/types' +import type { SearchConnector } from '@/lib/sim-search/connectors' import { memberConnectorKeys, useConnectSimSearchConnector, useStartConnectorMemberEnrollment, type ViewerConnectorMembership, + type WorkspaceMemberConnector, } from '@/hooks/queries/kb/connectors' const logger = createLogger('MemberEnrollment') @@ -164,13 +166,8 @@ export function useMemberEnrollment({ }) } - /** - * Each path clears the other's failure first: the surface shows one error, - * and a stale one from the other path would outlive a success on this one. - */ const connect = (knowledgeBaseId: string, connectorId: string) => openEnrollment(({ onSuccess, onError }) => { - sourceConnection.reset() enrollment.mutate( { knowledgeBaseId, connectorId }, { @@ -194,7 +191,6 @@ export function useMemberEnrollment({ sourceConfig?: Record ) => openEnrollment(({ onSuccess, onError }) => { - enrollment.reset() sourceConnection.mutate( { workspaceId, connectorType, sourceConfig }, { @@ -207,15 +203,43 @@ export function useMemberEnrollment({ ) }) + const [setupConnector, setSetupConnector] = useState(null) + + /** + * One click on a Sim Search source: enroll in its connector when someone + * already connected it, ask for its setup fields when it needs them, and + * otherwise create it and enroll in one step. + */ + const connectSearchSource = ( + workspaceId: string, + connector: SearchConnector, + connection: WorkspaceMemberConnector | undefined + ) => { + if (connection) { + connect(connection.knowledgeBaseId, connection.connectorId) + return + } + if (connector.setupFields.length > 0) { + setSetupConnector(connector) + return + } + connectSource(workspaceId, connector.type) + } + const isAwaiting = (connectorId: string) => awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId) - const error = enrollment.error ?? sourceConnection.error + /** The surface reports the latest attempt, whichever path made it. */ + const latest = + enrollment.submittedAt >= sourceConnection.submittedAt ? enrollment : sourceConnection return { connect, connectSource, + connectSearchSource, + setupConnector, + closeSetup: () => setSetupConnector(null), isAwaiting, - isPending: enrollment.isPending || sourceConnection.isPending, - error: popupBlocked ? POPUP_BLOCKED_MESSAGE : (error?.message ?? null), + isPending: latest.isPending, + error: popupBlocked ? POPUP_BLOCKED_MESSAGE : (latest.error?.message ?? null), } } diff --git a/apps/sim/lib/knowledge/access/availability.ts b/apps/sim/lib/knowledge/access/availability.ts index b46d1b0a25c..6496b0e4b9d 100644 --- a/apps/sim/lib/knowledge/access/availability.ts +++ b/apps/sim/lib/knowledge/access/availability.ts @@ -3,6 +3,7 @@ import { type WorkspaceOwnerSubscriptionAccess, } from '@/lib/billing/core/workspace-access' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' /** @@ -39,3 +40,14 @@ export async function isKnowledgeMemberAccessAvailable( context.ownerBilling ?? (await getWorkspaceOwnerSubscriptionAccess(context.workspaceId)) return isCredentialGroupsAvailable({ workspaceId: context.workspaceId, ownerBilling }) } + +/** Refuses with the one message every members-mode gate uses when the feature is off for the workspace. */ +export async function requireKnowledgeMemberAccessAvailable( + context: KnowledgeMemberAccessContext +): Promise { + if (await isKnowledgeMemberAccessAvailable(context)) return + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) +} diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index d0ded5301cf..3451da3499c 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -3,7 +3,7 @@ import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/princip import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' -import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeAttributedUserId, @@ -56,12 +56,7 @@ export const startKnowledgeConnectorMemberEnrollment = defineAuthorizedKnowledge if (connector.accessMode !== 'members' || !connector.credentialGroupId) { throw new OrchestrationError('validation', 'This connector does not sync per member') } - if (!(await isKnowledgeMemberAccessAvailable({ workspaceId }))) { - throw new OrchestrationError( - 'validation', - 'Per-member access is not available for this workspace' - ) - } + await requireKnowledgeMemberAccessAvailable({ workspaceId }) const url = await createViewerConnectorEnrollmentLink({ userId, workspaceId, diff --git a/apps/sim/lib/knowledge/application/sim-search.test.ts b/apps/sim/lib/knowledge/application/sim-search.test.ts index b85a0622dd9..e6add769674 100644 --- a/apps/sim/lib/knowledge/application/sim-search.test.ts +++ b/apps/sim/lib/knowledge/application/sim-search.test.ts @@ -37,9 +37,19 @@ vi.mock('@/lib/knowledge/application/contexts', () => ({ resolveKnowledgeWorkspaceContext: mocks.resolveWorkspace, })) -vi.mock('@/lib/knowledge/access/availability', () => ({ - isKnowledgeMemberAccessAvailable: mocks.isMemberAccessAvailable, -})) +vi.mock('@/lib/knowledge/access/availability', async () => { + const { OrchestrationError } = await import('@/lib/core/orchestration/types') + return { + isKnowledgeMemberAccessAvailable: mocks.isMemberAccessAvailable, + requireKnowledgeMemberAccessAvailable: async (context: { workspaceId: string }) => { + if (await mocks.isMemberAccessAvailable(context)) return + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) + }, + } +}) vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ createKnowledgeBase: { execute: mocks.createKnowledgeBase }, diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts index 056ece81a8b..628467812fd 100644 --- a/apps/sim/lib/knowledge/application/sim-search.ts +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -1,14 +1,14 @@ import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' -import { - permissionSatisfies, - resolveEffectiveWorkspacePermission, -} from '@sim/platform-authz/workspace' import { and, asc, eq, isNull, sql } from 'drizzle-orm' +import { + InsufficientWorkspacePermissionsError, + requireCurrentHumanRole, +} from '@/lib/core/application/workspace-authorization' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx } from '@/lib/db/types' -import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' import { createKnowledgeConnector } from '@/lib/knowledge/application/connectors' @@ -99,12 +99,10 @@ async function requireSimSearchSetupAdmin( context: KnowledgeWorkspaceContext, sourceName: string ): Promise { - const permission = await resolveEffectiveWorkspacePermission( - userId, - context.workspaceId, - context.workspaceOrganizationId - ) - if (!permissionSatisfies(permission, 'admin')) { + try { + await requireCurrentHumanRole(userId, context, 'admin') + } catch (error) { + if (!(error instanceof InsufficientWorkspacePermissionsError)) throw error throw new OrchestrationError( 'forbidden', `${sourceName} is not connected in this workspace yet. Ask a workspace admin to connect ${sourceName} first; after that everyone connects their own account.` @@ -141,19 +139,8 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ const workspaceId = context.workspaceId let target = await findSimSearchConnector(db, workspaceId, input.connectorType) if (!target) { - /** - * Judged before anything is created: the connector creation below checks - * the same availability, but only after the knowledge base exists. - */ - if (!(await isKnowledgeMemberAccessAvailable({ workspaceId }))) { - throw new OrchestrationError( - 'validation', - 'Per-member access is not available for this workspace' - ) - } const userId = resolvePrincipalSubjectUserId(principal) if (!userId) throw new OrchestrationError('forbidden', 'Sign in to connect your account') - await requireSimSearchSetupAdmin(userId, context, meta.name) const sourceConfig = input.sourceConfig ?? {} const missing = missingSetupFields(meta, sourceConfig) if (missing.length > 0) { @@ -162,6 +149,14 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ `${meta.name} needs ${missing.map((field) => field.title).join(' and ')} to connect` ) } + /** + * Judged before anything is created: the connector creation below checks + * the same availability, but only after the knowledge base exists. + */ + await Promise.all([ + requireKnowledgeMemberAccessAvailable({ workspaceId }), + requireSimSearchSetupAdmin(userId, context, meta.name), + ]) target = await db.transaction(async (tx) => { await tx.execute( sql`select set_config('lock_timeout', ${`${SIM_SEARCH_SETUP_LOCK_TIMEOUT_MS}ms`}, true)` diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 9d1886eb7ef..435c0c38116 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -219,10 +219,9 @@ describe('knowledge document processing source', () => { 1024, 200, 100, - PERSISTED_CONTEXT.uploadedBy, + { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: undefined }, null, undefined, - undefined, undefined ) expect(mockGenerateEmbeddings).not.toHaveBeenCalled() @@ -250,11 +249,10 @@ describe('knowledge document processing source', () => { 1024, 200, 100, - PERSISTED_CONTEXT.uploadedBy, + { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: SYSTEM_ACCESS_SCOPE }, null, undefined, - undefined, - SYSTEM_ACCESS_SCOPE + undefined ) }) @@ -276,10 +274,9 @@ describe('knowledge document processing source', () => { 1024, 200, 100, - PERSISTED_CONTEXT.uploadedBy, + { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: undefined }, null, undefined, - undefined, undefined ) }) diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 8172f9e755f..a1a48008db5 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -201,7 +201,7 @@ async function applyStrategy( * Who a source-file read runs as: the actor for authorization and OCR * attribution, plus how a knowledge-base file identifies its reader. */ -type SourceFileAccess = Pick +export type SourceFileAccess = Pick export async function processDocument( fileUrl: string, @@ -210,11 +210,10 @@ export async function processDocument( chunkSize = 1024, chunkOverlap = 200, minCharactersPerChunk = 100, - userId?: string, + access: SourceFileAccess = {}, workspaceId?: string | null, strategy?: ChunkingStrategy, - strategyOptions?: StrategyOptions, - knowledgeAccess?: DownloadFileFromUrlOptions['knowledgeAccess'] + strategyOptions?: StrategyOptions ): Promise<{ chunks: Chunk[] metadata: { @@ -229,7 +228,6 @@ export async function processDocument( } }> { logger.info('Processing document', { mimeType }) - const access: SourceFileAccess = { userId, knowledgeAccess } try { const parseResult = await parseDocument(fileUrl, filename, mimeType, access, workspaceId) diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts index b450f60897d..e8dc4c4bbde 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts @@ -57,7 +57,10 @@ function ocrPages(count: number, markdown = 'Recognised page') { function parse() { return runWithKnowledgeModelInputProvenance( undefined, - () => processDocument(PDF_URL, 'Contract.pdf', 'application/pdf', 1024, 200, 1, 'user-1'), + () => + processDocument(PDF_URL, 'Contract.pdf', 'application/pdf', 1024, 200, 1, { + userId: 'user-1', + }), { opaqueInputSafe: true } ) } diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 75566c4ae4b..9e0e1db6038 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -90,7 +90,10 @@ import { toPermanentDocumentProcessingError, UsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' -import { processDocument } from '@/lib/knowledge/documents/document-processor' +import { + processDocument, + type SourceFileAccess, +} from '@/lib/knowledge/documents/document-processor' import { failStaleDocumentProcessingClaim, recordUndispatchedDocumentFailure, @@ -1331,6 +1334,18 @@ function queueGenerationConditions( * invocation against the document's retry budget. Direct callers omit it and * therefore cannot refund an attempt they never charged. */ +/** + * Who the processor reads a document's source file as. Always the actor, not + * the payer: authorizing as the KB owner would let a writer ingest an internal + * file only the owner can read. A connector-owned row was written by the sync + * from bytes it fetched, not from a caller-supplied URL, so it is read as the + * system: in members mode the row stays hidden until the sync materializes who + * observed it, and the actor's own scope would deny the read. + */ +function sourceFileAccessFor(connectorId: string | null, actorUserId: string): SourceFileAccess { + return { userId: actorUserId, knowledgeAccess: connectorId ? SYSTEM_ACCESS_SCOPE : undefined } +} + export async function processDocumentAsync( knowledgeBaseId: string, documentId: string, @@ -1575,23 +1590,10 @@ export async function processDocumentAsync( kbConfig.maxSize, kbConfig.overlap, kbConfig.minSize, - /** - * Authorize source-file processing as the actor, not the payer. Using - * the KB owner would let a writer ingest an internal file that only the - * owner can read. - */ - documentActorUserId, + sourceFileAccessFor(ctx.connectorId, documentActorUserId), ctx.workspaceId, rawConfig?.strategy, - rawConfig?.strategyOptions, - /** - * A connector-owned row was written by the sync from bytes it fetched, - * not from a caller-supplied URL, so the processor reads it as the - * system: in members mode the row stays hidden until the sync - * materializes who observed it, and the actor's own scope would deny - * the read. Uploads keep the actor's authorization above. - */ - ctx.connectorId ? SYSTEM_ACCESS_SCOPE : undefined + rawConfig?.strategyOptions ) assertDocumentChunkCountWithinLimit(processed.chunks.length) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts index 17865e23eb1..3c91e25009f 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts @@ -47,9 +47,19 @@ vi.mock('@/lib/knowledge/connectors/member-access', () => ({ vi.mock('@/lib/credential-groups/credentials', () => ({ loadCredentialGroupCredentialListContext: mocks.loadGroup, })) -vi.mock('@/lib/knowledge/access/availability', () => ({ - isKnowledgeMemberAccessAvailable: mocks.memberAccessAvailable, -})) +vi.mock('@/lib/knowledge/access/availability', async () => { + const { OrchestrationError } = await import('@/lib/core/orchestration/types') + return { + isKnowledgeMemberAccessAvailable: mocks.memberAccessAvailable, + requireKnowledgeMemberAccessAvailable: async (context: { workspaceId: string }) => { + if (await mocks.memberAccessAvailable(context)) return + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) + }, + } +}) vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ provisionKnowledgeConnectorMembersBinding: mocks.provision, })) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index fbe32cc2f9f..c400541e173 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -8,7 +8,7 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attr import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { loadCredentialGroupCredentialListContext } from '@/lib/credential-groups/credentials' -import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' import { grantKnowledgeConnectorCredentialAccess, @@ -84,12 +84,7 @@ export async function resolveKnowledgeConnectorMembersBinding(input: { * Judged by the workspace alone, as the member engine is: a person's own * flag clause must not open a mode the engine will then refuse to run. */ - if (!(await isKnowledgeMemberAccessAvailable({ workspaceId: input.workspaceId }))) { - throw new OrchestrationError( - 'validation', - 'Per-member access is not available for this workspace' - ) - } + await requireKnowledgeMemberAccessAvailable({ workspaceId: input.workspaceId }) if (!input.connectorMeta.permissionScopedListing) { throw new OrchestrationError( 'validation', diff --git a/apps/sim/lib/sim-search/connectors.test.ts b/apps/sim/lib/sim-search/connectors.test.ts index 1f8dd35edd8..0aec5f4aa92 100644 --- a/apps/sim/lib/sim-search/connectors.test.ts +++ b/apps/sim/lib/sim-search/connectors.test.ts @@ -12,6 +12,7 @@ vi.mock('@/connectors/registry', () => { id: 'jsm', name: 'Jira Service Management', auth: { mode: 'oauth', provider: 'jira' }, + configFields: [], icon, }, jira: { @@ -34,18 +35,21 @@ vi.mock('@/connectors/registry', () => { id: 'gmail', name: 'Gmail', auth: { mode: 'oauth', provider: 'google-email' }, + configFields: [], icon, }, unknown: { id: 'unknown', name: 'Unknown', auth: { mode: 'oauth', provider: 'not-a-service' }, + configFields: [], icon, }, salesforce: { id: 'salesforce', name: 'Salesforce', auth: { mode: 'oauth', provider: 'salesforce' }, + configFields: [], icon, }, }, diff --git a/apps/sim/lib/sim-search/connectors.ts b/apps/sim/lib/sim-search/connectors.ts index 716a9ee1c89..9842a4d0e53 100644 --- a/apps/sim/lib/sim-search/connectors.ts +++ b/apps/sim/lib/sim-search/connectors.ts @@ -42,6 +42,8 @@ export interface SearchConnector { * the first catalog integration on the provider, else the connector type. */ blockType: string + /** Required config a person supplies on the source's first connect; empty for one-click sources. */ + setupFields: readonly ConnectorConfigField[] } /** @@ -69,6 +71,7 @@ export const SEARCH_CONNECTORS: readonly SearchConnector[] = Object.entries(CONN serviceName: service.name, serviceIcon: service.icon as ComponentType<{ className?: string }>, blockType: getIntegrationsForCredentialProvider(service.providerId)[0]?.type ?? type, + setupFields: personalSetupFields(meta), }, ] }) @@ -100,12 +103,26 @@ export function personalSetupFields(meta: ConnectorMeta): ConnectorConfigField[] /** The setup fields a source config leaves empty. */ export function missingSetupFields( meta: ConnectorMeta, - sourceConfig: Record + sourceConfig: Record ): ConnectorConfigField[] { - return personalSetupFields(meta).filter((field) => { - const value = sourceConfig[field.id] - return typeof value !== 'string' || value.trim() === '' - }) + return personalSetupFields(meta).filter((field) => !sourceConfig[field.id]?.trim()) +} + +/** The name a connector shows, from its registry entry. */ +export function connectorDisplayName(connectorType: string): string { + return CONNECTOR_META_REGISTRY[connectorType]?.name ?? connectorType +} + +/** Why a source cannot be connected on this surface right now; null when it can. */ +export function searchConnectorUnavailableReason( + connector: SearchConnector, + integrationAvailability: ReadonlyMap, + memberAccessAvailable: boolean +): string | null { + if (!isSearchConnectorAvailable(connector, integrationAvailability)) { + return `${connector.meta.name} is unavailable in this deployment` + } + return memberAccessAvailable ? null : 'Per-member access is not available in this workspace' } /** diff --git a/packages/emcn/src/components/chip/chip.tsx b/packages/emcn/src/components/chip/chip.tsx index f06411da8a6..07ce3077459 100644 --- a/packages/emcn/src/components/chip/chip.tsx +++ b/packages/emcn/src/components/chip/chip.tsx @@ -102,6 +102,8 @@ interface ChipBaseProps extends Omit, 'variant leftAdornment?: ReactNode /** Icon component rendered after the label. */ rightIcon?: ChipIcon + /** Custom content rendered after the label, such as a spinning loader. Takes precedence over `rightIcon`. */ + rightAdornment?: ReactNode children?: ReactNode } @@ -115,6 +117,7 @@ function ChipContent({ leftIcon: LeftIcon, leftAdornment, rightIcon: RightIcon, + rightAdornment, children, }: ChipBaseProps) { const isInverse = variant === 'primary' || variant === 'destructive' @@ -130,7 +133,7 @@ function ChipContent({ ) : children != null && children !== false ? ( {children} ) : null} - {RightIcon ? : null} + {rightAdornment ?? (RightIcon ? : null)} ) } @@ -152,6 +155,7 @@ const Chip = forwardRef(function Chip( leftIcon, leftAdornment, rightIcon, + rightAdornment, children, type, ...props @@ -170,6 +174,7 @@ const Chip = forwardRef(function Chip( leftIcon={leftIcon} leftAdornment={leftAdornment} rightIcon={rightIcon} + rightAdornment={rightAdornment} > {children} @@ -195,6 +200,7 @@ const ChipLink = forwardRef(function ChipLink( leftIcon, leftAdornment, rightIcon, + rightAdornment, children, ...props }, @@ -211,6 +217,7 @@ const ChipLink = forwardRef(function ChipLink( leftIcon={leftIcon} leftAdornment={leftAdornment} rightIcon={rightIcon} + rightAdornment={rightAdornment} > {children} From 0c5a0302243ef209d51cd055d1b16ab766451d02 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 03:02:09 -0700 Subject: [PATCH 46/76] fix(connectors): keep per-member listings complete across partial scopes and blank caps Teams, SharePoint, and OneDrive skip one of several channels or a descendant folder a member cannot reach instead of failing the whole listing, which the members-mode crawl would read as the member reaching nothing. Gmail and Outlook keep their default cap for a null, empty, or whitespace field rather than lifting it, through a shared parseDefaultedUnlimitedSafeInteger. The Confluence incremental CQL clause is fixed on the first page so a cursor is never paired with a query that crossed a minute boundary. --- .../connectors/confluence/confluence.test.ts | 65 ++++++++- apps/sim/connectors/confluence/confluence.ts | 19 ++- apps/sim/connectors/gmail/gmail.test.ts | 18 +++ apps/sim/connectors/gmail/gmail.ts | 16 +-- apps/sim/connectors/microsoft-excel/meta.ts | 6 +- .../microsoft-teams/microsoft-teams.test.ts | 74 ++++++++++ .../microsoft-teams/microsoft-teams.ts | 129 ++++++++++++------ apps/sim/connectors/onedrive/onedrive.test.ts | 42 ++++++ apps/sim/connectors/onedrive/onedrive.ts | 16 ++- apps/sim/connectors/outlook/outlook.test.ts | 16 +++ apps/sim/connectors/outlook/outlook.ts | 6 +- .../connectors/sharepoint/sharepoint.test.ts | 47 +++++++ apps/sim/connectors/sharepoint/sharepoint.ts | 20 ++- apps/sim/connectors/utils.test.ts | 45 ++++++ apps/sim/connectors/utils.ts | 33 +++++ 15 files changed, 491 insertions(+), 61 deletions(-) diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index 70f06538d2b..383a8ca8164 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AtlassianSiteNotAccessibleError, AtlassianSiteNotMatchedError, @@ -408,3 +408,66 @@ describe('preserveConfluenceCallouts', () => { expect(result).toContain('[WARNING] Do NOT use this form for: GitLab') }) }) + +describe('confluence incremental CQL listing', () => { + const fetchMock = + vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise>() + + function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + } + + function cqlOfCall(index: number): string | null { + return new URL(String(fetchMock.mock.calls[index][0])).searchParams.get('cql') + } + + beforeEach(() => { + vi.useFakeTimers() + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() + }) + + it('keeps one lastModified clause across pages that straddle a minute boundary', async () => { + const lastSyncAt = new Date('2026-09-01T11:30:00Z') + const config = { domain: 'example.atlassian.net', spaceKey: 'ENG' } + const syncContext: Record = { cloudId: 'cloud-1' } + fetchMock + .mockResolvedValueOnce( + jsonResponse({ + results: [], + _links: { next: '/wiki/rest/api/content/search?cursor=page-2&cql=ignored' }, + }) + ) + .mockResolvedValueOnce(jsonResponse({ results: [] })) + + vi.setSystemTime(new Date('2026-09-01T12:00:59Z')) + const first = await confluenceConnector.listDocuments( + 'token', + config, + undefined, + syncContext, + lastSyncAt + ) + expect(first.nextCursor).toBe('page-2') + + vi.setSystemTime(new Date('2026-09-01T12:01:01Z')) + await confluenceConnector.listDocuments( + 'token', + config, + first.nextCursor, + syncContext, + lastSyncAt + ) + + expect(cqlOfCall(0)).toContain('lastModified >= now("-31m")') + expect(cqlOfCall(1)).toBe(cqlOfCall(0)) + }) +}) diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index e1b5a4ce798..6ad3f97e199 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -698,6 +698,23 @@ export function buildLastModifiedClause(lastSyncAt: Date, now: Date): string { return `lastModified >= now("-${minutes}m")` } +/** + * The `lastModified` clause every page of one listing shares. The clause is a + * window relative to the server clock, so recomputing it on a later page that + * crosses a minute boundary would pair the cursor `_links.next` issued with a + * query it was not issued for; the first page fixes it for the run. + */ +export function resolveLastModifiedClause( + lastSyncAt: Date, + syncContext: Record | undefined +): string { + const fixed = syncContext?.cqlLastModifiedClause + if (typeof fixed === 'string') return fixed + const clause = buildLastModifiedClause(lastSyncAt, new Date()) + if (syncContext) syncContext.cqlLastModifiedClause = clause + return clause +} + /** * Page size for CQL search. The endpoint defaults to 25 and documents no hard * maximum, so this stays conservatively below the fixed system limits it warns @@ -749,7 +766,7 @@ async function listDocumentsViaCql( cql += ` AND label in (${labelList})` } - if (lastSyncAt) cql += ` AND ${buildLastModifiedClause(lastSyncAt, new Date())}` + if (lastSyncAt) cql += ` AND ${resolveLastModifiedClause(lastSyncAt, syncContext)}` const fetchedSoFar = (syncContext?.totalDocsFetched as number) ?? 0 const remaining = maxPages > 0 ? maxPages - fetchedSoFar : Number.POSITIVE_INFINITY diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts index 74023e74f51..2362fa0011f 100644 --- a/apps/sim/connectors/gmail/gmail.test.ts +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -12,6 +12,7 @@ vi.mock('@/lib/knowledge/documents/utils', () => ({ vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) import { gmailConnector } from '@/connectors/gmail/gmail' +import { DEFAULT_MAX_THREADS } from '@/connectors/gmail/meta' function threads(count: number, prefix: string) { return Array.from({ length: count }, (_, i) => ({ id: `${prefix}-${i}`, historyId: '1' })) @@ -86,3 +87,20 @@ describe('gmail listDocuments with maxThreads 0 (unlimited, a per-member sync)', expect(syncContext.listingCapped).toBe(true) }) }) + +describe('gmail listDocuments with a blank maxThreads', () => { + it.each([null, '', ' '])('keeps the default cap for %j', async (maxThreads) => { + mockPages([]) + const syncContext: Record = { totalThreadsFetched: DEFAULT_MAX_THREADS } + + const result = await gmailConnector.listDocuments( + 'token', + { maxThreads }, + undefined, + syncContext + ) + + expect(result.hasMore).toBe(false) + expect(mockFetchWithRetry).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index c8ea8fd70b7..a3254c8a338 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -6,8 +6,8 @@ import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/ import { htmlToPlainText, joinTagArray, + parseDefaultedUnlimitedSafeInteger, parseMultiValue, - parseOptionalUnlimitedSafeInteger, parseTagDate, } from '@/connectors/utils' @@ -452,14 +452,12 @@ export const gmailConnector: ConnectorConfig = { labelIndex = resolved } const searchQuery = buildSearchQuery(sourceConfig, labelIndex) - /** Absent means the default cap; an explicit 0 (a per-member sync) means unlimited. */ - const maxThreads = - sourceConfig.maxThreads === undefined - ? DEFAULT_MAX_THREADS - : parseOptionalUnlimitedSafeInteger( - sourceConfig.maxThreads, - 'maxThreads must be a non-negative integer' - ) + /** A blank field keeps the default cap; an explicit 0 (a per-member sync) means unlimited. */ + const maxThreads = parseDefaultedUnlimitedSafeInteger( + sourceConfig.maxThreads, + DEFAULT_MAX_THREADS, + 'maxThreads must be a non-negative integer' + ) const totalFetched = (syncContext?.totalThreadsFetched as number) ?? 0 if (maxThreads > 0 && totalFetched >= maxThreads) { diff --git a/apps/sim/connectors/microsoft-excel/meta.ts b/apps/sim/connectors/microsoft-excel/meta.ts index 142a0786071..463993e4070 100644 --- a/apps/sim/connectors/microsoft-excel/meta.ts +++ b/apps/sim/connectors/microsoft-excel/meta.ts @@ -14,7 +14,11 @@ export const microsoftExcelConnectorMeta: ConnectorMeta = { requiredScopes: ['Files.ReadWrite'], }, - /** Every worksheet of the one workbook is listed; nothing caps the listing. */ + /** + * No config field caps the listing: every worksheet of the one workbook is + * listed. The `MAX_WORKSHEETS` memory bound flags `listingCapped` when it + * bites, which the members-mode crawl reads as an incomplete listing. + */ permissionScopedListing: { capFieldIds: [] }, configFields: [ { diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts index 5dba2c181b1..306289f09c5 100644 --- a/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts @@ -12,6 +12,7 @@ vi.mock('@/lib/knowledge/documents/utils', () => ({ vi.mock('@/components/icons', () => ({ MicrosoftTeamsIcon: () => null })) import { microsoftTeamsConnector } from '@/connectors/microsoft-teams/microsoft-teams' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' const GRAPH = 'https://graph.microsoft.com/v1.0' const TEAM_ID = 'team-1' @@ -80,3 +81,76 @@ describe('microsoft teams listing scope', () => { expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(false) }) }) + +describe('microsoft teams per-member listing of several channels', () => { + const messagesUrl = (channelId: string) => + `${GRAPH}/teams/${TEAM_ID}/channels/${channelId}/messages?$top=50&$expand=replies` + + function teamsMessage(id: string) { + return { + id, + messageType: 'message', + createdDateTime: '2026-01-01T00:00:00Z', + from: { user: { id: 'u1', displayName: 'Ada' } }, + body: { contentType: 'text', content: `hello from ${id}` }, + } + } + + /** General is readable, Private answers 403 on its messages, Secret is not listed at all. */ + function mockChannels() { + mockGraph({ + [CHANNELS_URL]: { + body: { + value: [ + { id: 'c1', displayName: 'General' }, + { id: 'c2', displayName: 'Private' }, + ], + }, + }, + [messagesUrl('c1')]: { body: { value: [teamsMessage('m1')] } }, + [messagesUrl('c2')]: { status: 403, body: {} }, + }) + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('skips the channels the member cannot reach and keeps the rest', async () => { + mockChannels() + + const result = await microsoftTeamsConnector.listDocuments( + 'token', + { teamId: TEAM_ID, channel: ['General', 'Private', 'Secret'] }, + undefined, + { ...PER_MEMBER_LISTING_CONTEXT } + ) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['c1']) + expect(result.hasMore).toBe(false) + }) + + it('still fails a shared listing when one of several channels is unreachable', async () => { + mockChannels() + + const error = await microsoftTeamsConnector + .listDocuments('token', { teamId: TEAM_ID, channel: ['General', 'Private'] }, undefined, {}) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('reads a sole unreachable channel as the whole scope', async () => { + mockChannels() + + const error = await microsoftTeamsConnector + .listDocuments('token', { teamId: TEAM_ID, channel: 'Private' }, undefined, { + ...PER_MEMBER_LISTING_CONTEXT, + }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) +}) diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts index c458ff5460e..1ae19ae97ef 100644 --- a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts @@ -11,6 +11,7 @@ import { computeContentHash, htmlToPlainText, isListingScopeUnavailableError, + isPerMemberListing, parseMultiValue, parseTagDate, } from '@/connectors/utils' @@ -312,23 +313,73 @@ async function resolveChannel( return null } +/** + * Graph answers 403 for a team or private channel the caller is not a member + * of and 404 for one it will not show them; a channel the caller's channel + * list does not resolve is reported the same way. + */ +function isChannelScopeUnavailableError(error: unknown): boolean { + return ( + isListingScopeUnavailableError(error) || + (error instanceof GraphApiError && (error.status === 403 || error.status === 404)) + ) +} + +/** Lists one configured channel as a document, or null when it holds no messages. */ +async function listChannel( + accessToken: string, + teamId: string, + channelInput: string, + maxMessages: number +): Promise { + const channel = await resolveChannel(accessToken, teamId, channelInput) + if (!channel) { + throw new ConnectorListingScopeUnavailableError(`Channel not found: ${channelInput}`, 404) + } + + const { threads, messageCount, lastActivityTs } = await fetchChannelMessages( + accessToken, + teamId, + channel.id, + maxMessages + ) + + const content = formatMessages(threads) + if (!content.trim()) { + logger.info(`No messages found in channel: ${channel.displayName}`) + return null + } + + const contentHash = await computeContentHash(content) + + const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}` + + return { + externalId: channel.id, + title: channel.displayName, + content, + mimeType: 'text/plain', + sourceUrl, + contentHash, + metadata: { + channelName: channel.displayName, + messageCount, + lastActivity: lastActivityTs || undefined, + description: channel.description || undefined, + }, + } +} + export const microsoftTeamsConnector: ConnectorConfig = { ...microsoftTeamsConnectorMeta, - /** - * Graph answers 403 for a team or private channel the caller is not a member - * of and 404 for one it will not show them; a channel the caller's channel - * list does not resolve is reported the same way. - */ - isListingScopeUnavailableError: (error) => - isListingScopeUnavailableError(error) || - (error instanceof GraphApiError && (error.status === 403 || error.status === 404)), + isListingScopeUnavailableError: isChannelScopeUnavailableError, listDocuments: async ( accessToken: string, sourceConfig: Record, _cursor?: string, - _syncContext?: Record + syncContext?: Record ): Promise => { const teamId = sourceConfig.teamId as string const channelInputs = parseMultiValue(sourceConfig.channel) @@ -350,42 +401,32 @@ export const microsoftTeamsConnector: ConnectorConfig = { const documents: ExternalDocument[] = [] for (const channelInput of channelInputs) { - const channel = await resolveChannel(accessToken, teamId, channelInput) - if (!channel) { - throw new ConnectorListingScopeUnavailableError(`Channel not found: ${channelInput}`, 404) - } - - const { threads, messageCount, lastActivityTs } = await fetchChannelMessages( - accessToken, - teamId, - channel.id, - maxMessages - ) - - const content = formatMessages(threads) - if (!content.trim()) { - logger.info(`No messages found in channel: ${channel.displayName}`) - continue + let document: ExternalDocument | null + try { + document = await listChannel(accessToken, teamId, channelInput, maxMessages) + } catch (error) { + /** + * One of several channels a member cannot reach is absent from their + * listing, not the end of it: move on to the next channel so the rest + * of their access survives. A sole unreachable channel is the whole + * scope, which the members-mode crawl reads as a complete listing of + * nothing, and a shared credential still fails the sync rather than + * silently dropping the channel. + */ + if ( + channelInputs.length > 1 && + isPerMemberListing(syncContext) && + isChannelScopeUnavailableError(error) + ) { + logger.warn('Skipping a Microsoft Teams channel the member cannot reach', { + channel: channelInput, + error: getErrorMessage(error), + }) + continue + } + throw error } - - const contentHash = await computeContentHash(content) - - const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}` - - documents.push({ - externalId: channel.id, - title: channel.displayName, - content, - mimeType: 'text/plain', - sourceUrl, - contentHash, - metadata: { - channelName: channel.displayName, - messageCount, - lastActivity: lastActivityTs || undefined, - description: channel.description || undefined, - }, - }) + if (document) documents.push(document) } // All selected channels are emitted in a single page; no pagination needed diff --git a/apps/sim/connectors/onedrive/onedrive.test.ts b/apps/sim/connectors/onedrive/onedrive.test.ts index 3439d7b31b1..7ce1f708fd7 100644 --- a/apps/sim/connectors/onedrive/onedrive.test.ts +++ b/apps/sim/connectors/onedrive/onedrive.test.ts @@ -16,6 +16,7 @@ import { onedriveConnector } from '@/connectors/onedrive/onedrive' import { encodeMicrosoftGraphTraversalCursor, MICROSOFT_GRAPH_MAX_PENDING_FOLDERS, + PER_MEMBER_LISTING_CONTEXT, } from '@/connectors/utils' const GRAPH = 'https://graph.microsoft.com/v1.0' @@ -430,6 +431,47 @@ describe('onedrive listing scope', () => { } ) + it('skips a subfolder the member cannot reach and keeps their listing complete', async () => { + mockGraph({ + [ROOT_URL]: { + body: { value: [file('f1', 'a.txt'), folder('open', 'open'), folder('locked', 'locked')] }, + }, + [childrenUrl('locked')]: { status: 403, body: {} }, + [childrenUrl('open')]: { body: { value: [file('f2', 'b.md')] } }, + }) + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const result = await onedriveConnector.listDocuments('token', {}, undefined, syncContext) + + expect(result.documents.map((d) => d.externalId)).toEqual(['f1', 'f2']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('still fails a shared listing on a subfolder it cannot reach', async () => { + mockGraph({ + [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), folder('locked', 'locked')] } }, + }) + + const error = await onedriveConnector + .listDocuments('token', {}, undefined, {}) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('reads an unreachable root as the whole scope under a per-member listing', async () => { + mockGraph({ [ROOT_URL]: { status: 403, body: {} } }) + + const error = await onedriveConnector + .listDocuments('token', {}, undefined, { ...PER_MEMBER_LISTING_CONTEXT }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + it('keeps any other listing failure retryable', async () => { mockGraph({ [ROOT_URL]: { status: 500, body: {} } }) diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index 7fe5c17a103..dced987dbaf 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -20,6 +20,7 @@ import { isIndexableConnectorFile, isListingScopeUnavailableError, isMicrosoftGraphDriveItem, + isSkippableMicrosoftGraphFolderError, isSkippedDocument, type MicrosoftGraphTraversalState, markSkipped, @@ -267,7 +268,20 @@ export const onedriveConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw microsoftGraphListingError('Failed to list OneDrive files', response.status) + const error = microsoftGraphListingError('Failed to list OneDrive files', response.status) + const isRootFolder = state.currentFolder === undefined + if (!isSkippableMicrosoftGraphFolderError(error, syncContext, isRootFolder)) throw error + logger.warn('Skipping a OneDrive folder the member cannot reach', { + folderId: state.currentFolder, + status: response.status, + }) + if (state.folderStack.length === 0) { + done = true + break + } + state.currentFolder = state.folderStack.pop()! + state.nextLink = undefined + continue } const data = parseMicrosoftGraphDriveItemList(await response.json(), 'OneDrive') diff --git a/apps/sim/connectors/outlook/outlook.test.ts b/apps/sim/connectors/outlook/outlook.test.ts index e33c2fa1e26..7452c2969c3 100644 --- a/apps/sim/connectors/outlook/outlook.test.ts +++ b/apps/sim/connectors/outlook/outlook.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { DEFAULT_MAX_CONVERSATIONS } from '@/connectors/outlook/meta' import { DELETED_ITEMS_FOLDER, isAllMailSync, @@ -499,6 +500,21 @@ describe('listDocuments conversation cap', () => { expect(syncContext.listingCapped).toBe(true) }) + it('keeps the default cap when the field holds only whitespace', async () => { + routeFetch([inboxMessagesRoute(DEFAULT_MAX_CONVERSATIONS + 1)]) + + const syncContext: Record = {} + const result = await outlookConnector.listDocuments( + 'token', + { folder: 'inbox', maxConversations: ' ' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(DEFAULT_MAX_CONVERSATIONS) + expect(syncContext.listingCapped).toBe(true) + }) + it('lists every conversation when the cap is 0', async () => { routeFetch([inboxMessagesRoute(3)]) diff --git a/apps/sim/connectors/outlook/outlook.ts b/apps/sim/connectors/outlook/outlook.ts index 344a311e364..78f183883d6 100644 --- a/apps/sim/connectors/outlook/outlook.ts +++ b/apps/sim/connectors/outlook/outlook.ts @@ -7,7 +7,7 @@ import { htmlToPlainText, isListingScopeUnavailableError, listingRequestError, - parseOptionalUnlimitedSafeInteger, + parseDefaultedUnlimitedSafeInteger, parseTagDate, } from '@/connectors/utils' @@ -613,9 +613,9 @@ function formatConversation( * per-member listing is complete. */ function parseMaxConversations(value: unknown): number { - if (value === undefined || value === null || value === '') return DEFAULT_MAX_CONVERSATIONS - return parseOptionalUnlimitedSafeInteger( + return parseDefaultedUnlimitedSafeInteger( value, + DEFAULT_MAX_CONVERSATIONS, 'Max conversations must be a positive safe integer, or 0 for unlimited' ) } diff --git a/apps/sim/connectors/sharepoint/sharepoint.test.ts b/apps/sim/connectors/sharepoint/sharepoint.test.ts index 12b885ced96..9263246515b 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.test.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.test.ts @@ -22,6 +22,7 @@ import { appendPendingMicrosoftGraphFolders, encodeMicrosoftGraphTraversalCursor, MICROSOFT_GRAPH_MAX_PENDING_FOLDERS, + PER_MEMBER_LISTING_CONTEXT, } from '@/connectors/utils' const GRAPH = 'https://graph.microsoft.com/v1.0' @@ -479,6 +480,52 @@ describe('listDocuments', () => { expect(syncContext.listingCapped).toBeUndefined() }) + it('skips a subfolder the member cannot reach and keeps their listing complete', async () => { + mockGraph({ + ...childrenRoute(DEFAULT_DRIVE_ID, null, [ + file('f1', 'a.txt'), + folder('open', 'Open'), + folder('locked', 'Locked'), + ]), + [`${GRAPH}/drives/${DEFAULT_DRIVE_ID}/items/locked/children?$top=200&$select=${ITEM_SELECT}`]: + { status: 403, body: {} }, + ...childrenRoute(DEFAULT_DRIVE_ID, 'open', [file('f2', 'b.txt')]), + }) + const syncContext = { ...listContext(), ...PER_MEMBER_LISTING_CONTEXT } + + const result = await list(undefined, syncContext) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['f1', 'f2']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('still fails a shared listing on a subfolder it cannot reach', async () => { + mockGraph({ + ...childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt'), folder('locked', 'Locked')]), + }) + + const error = await list(undefined, listContext()).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('reads an unreachable root as the whole scope under a per-member listing', async () => { + mockGraph({ + [`${GRAPH}/drives/${DEFAULT_DRIVE_ID}/root/children?$top=200&$select=${ITEM_SELECT}`]: { + status: 403, + body: {}, + }, + }) + const syncContext = { ...listContext(), ...PER_MEMBER_LISTING_CONTEXT } + + const error = await list(undefined, syncContext).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + it('drains subfolders within a single call instead of one folder per page', async () => { mockGraph({ ...childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt'), folder('sub', 'Sub')]), diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index 9d14a62f949..27f21bd6fab 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -22,6 +22,7 @@ import { isIndexableConnectorFile, isListingScopeUnavailableError, isMicrosoftGraphDriveItem, + isSkippableMicrosoftGraphFolderError, isSkippedDocument, type MicrosoftGraphTraversalState, markSkipped, @@ -865,7 +866,24 @@ export const sharepointConnector: ConnectorConfig = { let cappedWithItemsLeft = false for (let request = 0; request < MAX_LIST_REQUESTS_PER_CALL; request++) { - const data = await listFolderItems(accessToken, driveId, state.currentFolder, state.nextLink) + let data: Awaited> + try { + data = await listFolderItems(accessToken, driveId, state.currentFolder, state.nextLink) + } catch (error) { + const isRootFolder = state.currentFolder === rootFolderId + if (!isSkippableMicrosoftGraphFolderError(error, syncContext, isRootFolder)) throw error + logger.warn('Skipping a SharePoint folder the member cannot reach', { + folderId: state.currentFolder, + error: getErrorMessage(error), + }) + if (state.folderStack.length === 0) { + stopPaging = true + break + } + state.currentFolder = state.folderStack.pop()! + state.nextLink = undefined + continue + } // Separate files and subfolders const subfolders: string[] = [] diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index 4fd76920f55..353d715eaa0 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -64,17 +64,21 @@ import { appendPendingMicrosoftGraphFolders, assertMicrosoftGraphNextLink, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, decodeMicrosoftGraphTraversalCursor, encodeMicrosoftGraphTraversalCursor, extractConnectorText, hasIndexablePayload, htmlToPlainText, isIndexableConnectorFile, + isSkippableMicrosoftGraphFolderError, isSkippedDocument, MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES, MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES, MICROSOFT_GRAPH_MAX_PENDING_FOLDERS, markSkipped, + PER_MEMBER_LISTING_CONTEXT, + parseDefaultedUnlimitedSafeInteger, pipelineParsedMimeType, readBodyWithLimit, sizeLimitSkipReason, @@ -1594,3 +1598,44 @@ describe('hasIndexablePayload', () => { expect(hasIndexablePayload({ content: ' ' })).toBe(false) }) }) + +describe('parseDefaultedUnlimitedSafeInteger', () => { + const ERROR = 'bad cap' + + it.each([undefined, null, '', ' '])('keeps the default for a blank field (%j)', (value) => { + expect(parseDefaultedUnlimitedSafeInteger(value, 500, ERROR)).toBe(500) + }) + + it('reads an explicit 0 as unlimited', () => { + expect(parseDefaultedUnlimitedSafeInteger(0, 500, ERROR)).toBe(0) + expect(parseDefaultedUnlimitedSafeInteger('0', 500, ERROR)).toBe(0) + }) + + it('parses a set cap and rejects a malformed one', () => { + expect(parseDefaultedUnlimitedSafeInteger(' 200 ', 500, ERROR)).toBe(200) + expect(() => parseDefaultedUnlimitedSafeInteger('many', 500, ERROR)).toThrow(ERROR) + expect(() => parseDefaultedUnlimitedSafeInteger(-1, 500, ERROR)).toThrow(ERROR) + }) +}) + +describe('isSkippableMicrosoftGraphFolderError', () => { + const unreachable = new ConnectorListingScopeUnavailableError('folder', 403) + const perMember = { ...PER_MEMBER_LISTING_CONTEXT } + + it('skips an unreachable descendant folder under a per-member listing', () => { + expect(isSkippableMicrosoftGraphFolderError(unreachable, perMember, false)).toBe(true) + }) + + it('never skips the configured root', () => { + expect(isSkippableMicrosoftGraphFolderError(unreachable, perMember, true)).toBe(false) + }) + + it('never skips under a shared credential', () => { + expect(isSkippableMicrosoftGraphFolderError(unreachable, {}, false)).toBe(false) + expect(isSkippableMicrosoftGraphFolderError(unreachable, undefined, false)).toBe(false) + }) + + it('never skips a fault the engine should retry', () => { + expect(isSkippableMicrosoftGraphFolderError(new Error('500'), perMember, false)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 4d25f214c13..857bac814a7 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -56,6 +56,22 @@ export function parseOptionalUnlimitedSafeInteger(value: unknown, errorMessage: return parsed } +/** + * Parses a connector cap that keeps `defaultValue` when the field is blank — + * absent, null, or a string of nothing but whitespace — and otherwise reads + * like {@link parseOptionalUnlimitedSafeInteger}, where 0 lifts the cap. A + * per-member sync writes that explicit 0; a form left empty must not. + */ +export function parseDefaultedUnlimitedSafeInteger( + value: unknown, + defaultValue: number, + errorMessage: string +): number { + if (value === undefined || value === null) return defaultValue + if (typeof value === 'string' && value.trim() === '') return defaultValue + return parseOptionalUnlimitedSafeInteger(value, errorMessage) +} + const MICROSOFT_GRAPH_ORIGIN = 'https://graph.microsoft.com' export interface MicrosoftGraphTraversalState { @@ -762,3 +778,20 @@ export const PER_MEMBER_LISTING_CONTEXT = { perMemberListing: true } as const export function isPerMemberListing(syncContext: Record | undefined): boolean { return syncContext?.perMemberListing === true } + +/** + * Whether a folder request that failed while walking a Microsoft Graph drive + * can be left out of the listing: under a member's own token a descendant + * folder Graph reports as unreachable (403, 404) is simply not shared with + * them, so their listing stays complete without it and their access to its + * files is withdrawn. The configured root is the whole scope, which the + * members-mode crawl reads as a complete listing of nothing, and a shared + * credential never skips: dropping the folder's files would read as deletions. + */ +export function isSkippableMicrosoftGraphFolderError( + error: unknown, + syncContext: Record | undefined, + isRootFolder: boolean +): boolean { + return !isRootFolder && isListingScopeUnavailableError(error) && isPerMemberListing(syncContext) +} From 240e321f254a8056805563459aa3b15e14feda95 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 03:05:44 -0700 Subject: [PATCH 47/76] fix(knowledge): address review threads on search surface, lease-guarded dispatch, and selected-output principal - Guard the processing-queue write and dispatch with the sync lease so a reclaimed run neither marks nor enqueues processing - Pass the executing principal into the final-frame selected-output context - Read JSM statusDate as the source modification time - Keep a first-connected Sim Search source non-actionable until its membership row appears - Render the Build/Search switcher only for the Home composer - Keep every Search-mode submit out of the agent; attachment-only submits do nothing - Skip the member-connector fetch and 'still indexing' note when per-member access is off, and scope it to the searched bases - Queue the workspace member-connector list on a members-mode sync trigger - Log a refused clipboard write on the source card instead of leaving it unhandled --- .../knowledge-search-results/index.ts | 6 +- .../knowledge-search-results.test.tsx | 67 ++++++++++ .../knowledge-search-results.tsx | 47 +++++-- .../components/source-card/source-card.tsx | 25 +++- .../mothership-chat/mothership-chat.tsx | 4 + .../search-sources/search-sources.tsx | 5 +- .../home/components/user-input/user-input.tsx | 9 +- .../app/workspace/[workspaceId]/home/home.tsx | 11 +- apps/sim/hooks/queries/kb/connectors.test.ts | 76 +++++++++++ apps/sim/hooks/queries/kb/connectors.ts | 18 ++- apps/sim/hooks/use-member-enrollment.test.tsx | 119 ++++++++++++++++++ apps/sim/hooks/use-member-enrollment.ts | 40 +++++- .../connectors/source-modified-at.test.ts | 1 + .../connectors/source-modified-at.ts | 2 + .../sim/lib/knowledge/connectors/sync-lock.ts | 24 ++++ .../knowledge/connectors/sync-persistence.ts | 26 +--- .../knowledge/connectors/sync-primitives.ts | 7 +- .../documents/processing-queue.test.ts | 66 ++++++++++ apps/sim/lib/knowledge/documents/service.ts | 24 +++- .../streaming/streaming-principal.test.ts | 82 ++++++++++++ apps/sim/lib/workflows/streaming/streaming.ts | 1 + 21 files changed, 601 insertions(+), 59 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx create mode 100644 apps/sim/hooks/use-member-enrollment.test.tsx create mode 100644 apps/sim/lib/workflows/streaming/streaming-principal.test.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts index d716af02066..a170379c468 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts @@ -1 +1,5 @@ -export { groupResultsByDocument, KnowledgeSearchResults } from './knowledge-search-results' +export { + groupResultsByDocument, + indexingSourceNames, + KnowledgeSearchResults, +} from './knowledge-search-results' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx new file mode 100644 index 00000000000..0be852d394d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -0,0 +1,67 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it, vi } from 'vitest' +import type { WorkspaceMemberConnector } from '@/hooks/queries/kb/connectors' + +vi.mock('@/hooks/queries/kb/connectors', () => ({ useWorkspaceMemberConnectors: vi.fn() })) +vi.mock('@/hooks/queries/kb/knowledge', () => ({ + useKnowledgeBasesQuery: vi.fn(), + useWorkspaceKnowledgeSearch: vi.fn(), +})) +vi.mock('@/app/workspace/[workspaceId]/providers/workspace-host-provider', () => ({ + useWorkspaceHostContext: vi.fn(), +})) +vi.mock( + '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card', + () => ({ SourceCard: () => null }) +) + +import { indexingSourceNames } from '@/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results' + +function memberConnector( + overrides: Partial = {} +): WorkspaceMemberConnector { + return { + knowledgeBaseId: 'kb-1', + knowledgeBaseName: 'Sim Search', + connectorId: 'connector-1', + connectorType: 'google_drive', + memberSyncStatus: 'running', + viewerMembership: 'connected', + viewerDocumentCount: 0, + ...overrides, + } +} + +describe('indexingSourceNames', () => { + it('names each source still indexing for the viewer once, in the searched bases only', () => { + const names = indexingSourceNames( + [ + memberConnector({ connectorId: 'a', connectorType: 'google_drive' }), + memberConnector({ + connectorId: 'b', + connectorType: 'google_drive', + knowledgeBaseId: 'kb-2', + }), + memberConnector({ connectorId: 'c', connectorType: 'slack', memberSyncStatus: 'pending' }), + memberConnector({ connectorId: 'd', connectorType: 'notion', knowledgeBaseId: 'kb-3' }), + ], + ['kb-1', 'kb-2'] + ) + + expect(names).toEqual(['Google Drive', 'Slack']) + }) + + it('ignores sources that are idle or not connected for the viewer', () => { + expect( + indexingSourceNames( + [ + memberConnector({ connectorId: 'a', memberSyncStatus: 'idle' }), + memberConnector({ connectorId: 'b', viewerMembership: 'invited' }), + ], + ['kb-1'] + ) + ).toEqual([]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 4468cce2d59..8461a7905c2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -7,9 +7,15 @@ import { connectorDisplayName } from '@/lib/sim-search/connectors' import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import { isIndexing } from '@/app/workspace/[workspaceId]/home/components/search-sources' -import { useWorkspaceMemberConnectors } from '@/hooks/queries/kb/connectors' +import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' +import { + useWorkspaceMemberConnectors, + type WorkspaceMemberConnector, +} from '@/hooks/queries/kb/connectors' import { useKnowledgeBasesQuery, useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' +const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] + /** A search spans at most this many knowledge bases. */ const MAX_SEARCHED_KNOWLEDGE_BASES = 20 /** Characters of the matching chunk shown under a result. */ @@ -47,6 +53,25 @@ export function groupResultsByDocument( return grouped } +/** + * The names of the sources still indexing for the viewer among the bases the + * search spans, each once. A base outside the search cannot grow its results, + * so its indexing is not the reader's concern here. + */ +export function indexingSourceNames( + memberConnectors: readonly WorkspaceMemberConnector[], + knowledgeBaseIds: readonly string[] +): string[] { + const searched = new Set(knowledgeBaseIds) + return [ + ...new Set( + memberConnectors + .filter((connection) => searched.has(connection.knowledgeBaseId) && isIndexing(connection)) + .map((connection) => connectorDisplayName(connection.connectorType)) + ), + ] +} + /** * A result as the source card renders it: the row's second line names the * source app, or the knowledge base for an upload. A document without a @@ -124,15 +149,17 @@ export function KnowledgeSearchResults({ isFetching, error, } = useWorkspaceKnowledgeSearch(workspaceId, knowledgeBaseIds, query) - const { data: memberConnectors = [] } = useWorkspaceMemberConnectors(workspaceId) - /** Every per-member connector still indexing for the viewer, in any base the search spans. */ - const indexing = [ - ...new Set( - memberConnectors - .filter(isIndexing) - .map((connection) => connectorDisplayName(connection.connectorType)) - ), - ] + const { features } = useWorkspaceHostContext() + /** + * Judged by the workspace, as the server judges it: with per-member access + * off, member-scoped documents are hidden, so no source is indexing anything + * the viewer will see, and the list is not worth asking for. + */ + const memberAccessAvailable = features?.knowledgeMemberAccess === true + const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS } = useWorkspaceMemberConnectors( + memberAccessAvailable ? workspaceId : undefined + ) + const indexing = indexingSourceNames(memberConnectors, knowledgeBaseIds) const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) const sourceTypes = useMemo( () => [...new Set(documents.map((result) => result.connectorType ?? 'upload'))], diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index 1add016d1a3..b9773a8c5cf 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -3,6 +3,8 @@ import { type ReactNode, useState } from 'react' import { Button, cn, Tooltip } from '@sim/emcn' import { Check, Link as LinkIcon } from '@sim/emcn/icons' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { formatDate } from '@sim/utils/formatting' import { faviconUrl } from '@/lib/core/utils/favicon' import { @@ -17,6 +19,8 @@ import { import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import { BrandIcon } from '@/blocks/brand-icon' +const logger = createLogger('SourceCard') + /** Query terms shorter than this are too common to bold. */ const MIN_HIGHLIGHT_TERM_LENGTH = 3 /** How long the copied state shows on the copy-link action. */ @@ -63,7 +67,11 @@ interface CopyLinkActionProps { url: string } -/** Copies the document's link; confirms with a check for a moment. */ +/** + * Copies the document's link; confirms with a check for a moment. The check + * only shows once the clipboard accepted the write: a page denied clipboard + * access is left at "Copy link" rather than claiming a copy that never landed. + */ function CopyLinkAction({ url }: CopyLinkActionProps) { const [copied, setCopied] = useState(false) return ( @@ -74,10 +82,17 @@ function CopyLinkAction({ url }: CopyLinkActionProps) { size='sm' aria-label='Copy link' onClick={() => { - void navigator.clipboard.writeText(url).then(() => { - setCopied(true) - window.setTimeout(() => setCopied(false), COPIED_FEEDBACK_MS) - }) + navigator.clipboard.writeText(url).then( + () => { + setCopied(true) + window.setTimeout(() => setCopied(false), COPIED_FEEDBACK_MS) + }, + (error: unknown) => { + logger.warn('Copying the document link failed', { + error: getErrorMessage(error), + }) + } + ) }} > {copied ? ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx index 5f4ece490d4..c44cab19127 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx @@ -70,6 +70,8 @@ interface MothershipChatProps { fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[] ) => void + /** Whether the composer offers Search mode; only the Home composer answers a search. */ + canSearch?: boolean /** Off in Search mode, where the query stays put so the person can refine it. */ clearOnSubmit?: boolean /** Fires when the composer's text goes from something to nothing. */ @@ -324,6 +326,7 @@ export function MothershipChat({ isReconnecting = false, isLoading = false, onSubmit, + canSearch = false, clearOnSubmit, onCleared, onStopGeneration, @@ -844,6 +847,7 @@ export function MothershipChat({ key={draftScopeKey} ref={userInputRef} onSubmit={onSubmit} + canSearch={canSearch} clearOnSubmit={clearOnSubmit} onCleared={onCleared} isSending={isStreamActive} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx index 38806d49d18..64dd5200441 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx @@ -166,6 +166,7 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { setupConnector, closeSetup, isAwaiting, + isAwaitingSource, isPending, error, } = useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds }) @@ -193,7 +194,9 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { integrationAvailability, memberAccessAvailable )} - waiting={connection ? isAwaiting(connection.connectorId) : false} + waiting={ + connection ? isAwaiting(connection.connectorId) : isAwaitingSource(connector.type) + } disabled={isPending} onConnect={() => connectSearchSource(workspaceId, connector, connection)} /> diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index b7f833cd588..9cf844cf597 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -70,6 +70,12 @@ interface UserInputProps { onStopGeneration: () => void isInitialView?: boolean onSendQueuedHead?: () => void + /** + * Whether the composer offers Search mode. Only the Home composer answers a + * search with documents; the workflow copilot always talks to the agent, so + * it must not show a mode it cannot honour. + */ + canSearch?: boolean /** * Whether the text is cleared once submitted. A search keeps its query in * the box, the way a search bar does, so it can be read and refined against @@ -107,6 +113,7 @@ const UserInputImpl = forwardRef(function UserI isInitialView = true, onSendQueuedHead, onEditQueuedTail, + canSearch = false, clearOnSubmit = true, onCleared, }, @@ -712,7 +719,7 @@ const UserInputImpl = forwardRef(function UserI
- + {canSearch && } {isSttSupported && ( ({ cancelQueries: vi.fn(), getQueryData: vi.fn(), setQueryData: vi.fn(), + setQueriesData: vi.fn(), invalidateQueries: vi.fn(), })) @@ -24,6 +25,7 @@ vi.mock('@tanstack/react-query', () => ({ cancelQueries: mocks.cancelQueries, getQueryData: mocks.getQueryData, setQueryData: mocks.setQueryData, + setQueriesData: mocks.setQueriesData, invalidateQueries: mocks.invalidateQueries, })), })) @@ -41,14 +43,31 @@ import { CONNECTOR_SYNC_POLL_INTERVAL_MS, connectorKeys, isConnectorSyncingOrPending, + memberConnectorKeys, useConnectorDetail, useConnectorDocuments, useConnectorList, useTriggerSync, + type WorkspaceMemberConnector, } from '@/hooks/queries/kb/connectors' const KB_ID = 'kb-1' +function makeMemberConnector( + overrides: Partial = {} +): WorkspaceMemberConnector { + return { + knowledgeBaseId: KB_ID, + knowledgeBaseName: 'Sim Search', + connectorId: 'connector-1', + connectorType: 'hubspot', + memberSyncStatus: 'idle', + viewerMembership: 'connected', + viewerDocumentCount: 0, + ...overrides, + } +} + function makeConnector(overrides: Partial = {}): ConnectorData { return { id: 'connector-1', @@ -250,6 +269,63 @@ describe('useTriggerSync optimistic state', () => { expect(rolledBack?.find((c) => c.id === 'connector-1')?.status).toBe('active') expect(rolledBack?.find((c) => c.id === 'connector-2')?.status).toBe('pending') }) + + /** + * The Search surface reads the member sync status from the workspace + * member-connector list, which has no poll of its own, so a members-mode + * trigger patches that cache too and a refused trigger refetches it. + */ + it('queues a members connector in the workspace member-connector list as well', async () => { + const existing = [ + makeConnector({ id: 'connector-1', accessMode: 'members', memberSyncStatus: 'idle' }), + ] + mocks.getQueryData.mockReturnValue(existing) + + useTriggerSync() + const options = capturedMutationOptions() + const context = await options.onMutate({ knowledgeBaseId: KB_ID, connectorId: 'connector-1' }) + + expect(mocks.setQueriesData).toHaveBeenCalledWith( + { queryKey: memberConnectorKeys.lists() }, + expect.any(Function) + ) + const patchMemberList = mocks.setQueriesData.mock.calls.at(-1)?.[1] as ( + connectors: WorkspaceMemberConnector[] | undefined + ) => WorkspaceMemberConnector[] | undefined + const memberList = [ + makeMemberConnector({ connectorId: 'connector-1', memberSyncStatus: 'idle' }), + makeMemberConnector({ connectorId: 'connector-2', memberSyncStatus: 'idle' }), + ] + expect(patchMemberList(memberList)?.map((c) => c.memberSyncStatus)).toEqual(['pending', 'idle']) + expect(patchMemberList(undefined)).toBeUndefined() + + options.onError( + new Error('boom'), + { knowledgeBaseId: KB_ID, connectorId: 'connector-1' }, + context + ) + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: memberConnectorKeys.lists(), + }) + }) + + it('leaves the workspace member-connector list alone for a workspace connector', async () => { + mocks.getQueryData.mockReturnValue([makeConnector({ status: 'active' })]) + + useTriggerSync() + const options = capturedMutationOptions() + const context = await options.onMutate({ knowledgeBaseId: KB_ID, connectorId: 'connector-1' }) + options.onError( + new Error('boom'), + { knowledgeBaseId: KB_ID, connectorId: 'connector-1' }, + context + ) + + expect(mocks.setQueriesData).not.toHaveBeenCalled() + expect(mocks.invalidateQueries).not.toHaveBeenCalledWith({ + queryKey: memberConnectorKeys.lists(), + }) + }) }) interface ConnectorDocumentsPage { diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 028a148f332..f24791bd5c7 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -205,7 +205,10 @@ function optimisticallySetConnectorStatus( /** * The optimistic "queued" write for a sync trigger, on whichever engine the * connector runs: a members connector queues a member run, so its content - * status must not flip. Returns what to restore if the trigger is refused. + * status must not flip. The Search surface reads the same member sync status + * from the workspace member-connector list, so that cache is queued too; it + * has no poll to reconcile it, and the write cannot stop one, so it is patched + * rather than refetched. Returns what to restore if the trigger is refused. */ function optimisticallyQueueSync( queryClient: QueryClient, @@ -220,6 +223,15 @@ function optimisticallyQueueSync( setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { memberSyncStatus: 'pending', }) + queryClient.setQueriesData( + { queryKey: memberConnectorKeys.lists() }, + (connectors) => + connectors?.map((connector) => + connector.connectorId === connectorId + ? { ...connector, memberSyncStatus: 'pending' } + : connector + ) + ) return { memberSyncStatus: cached.memberSyncStatus } } setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { status: 'pending' }) @@ -507,6 +519,10 @@ export function useTriggerSync() { setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, previous) } queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + /** The member-connector list took the same optimistic `pending`; a refetch is its rollback. */ + if (previous && 'memberSyncStatus' in previous) { + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) + } }, /** * Deliberately no invalidation on success. The route answers without diff --git a/apps/sim/hooks/use-member-enrollment.test.tsx b/apps/sim/hooks/use-member-enrollment.test.tsx new file mode 100644 index 00000000000..58773f49259 --- /dev/null +++ b/apps/sim/hooks/use-member-enrollment.test.tsx @@ -0,0 +1,119 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + enrollmentMutate: vi.fn(), + sourceConnectionMutate: vi.fn(), + invalidateQueries: vi.fn(), +})) + +vi.mock('@tanstack/react-query', () => ({ + useQueryClient: () => ({ invalidateQueries: mocks.invalidateQueries }), +})) +vi.mock('@/hooks/queries/kb/connectors', () => ({ + memberConnectorKeys: { lists: () => ['member-connectors', 'list'] }, + useStartConnectorMemberEnrollment: () => ({ + mutate: mocks.enrollmentMutate, + submittedAt: 0, + isPending: false, + error: null, + }), + useConnectSimSearchConnector: () => ({ + mutate: mocks.sourceConnectionMutate, + submittedAt: 0, + isPending: false, + error: null, + }), +})) + +import { useMemberEnrollment } from '@/hooks/use-member-enrollment' + +type Enrollment = ReturnType + +let latest: Enrollment | null = null +let root: Root | null = null +let container: HTMLDivElement | null = null + +function Harness({ connected }: { connected: ReadonlySet }) { + latest = useMemberEnrollment({ membershipQueryKeys: [], connectedConnectorIds: connected }) + return null +} + +function mount(connected: ReadonlySet = new Set()) { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function enrollment(): Enrollment { + if (!latest) throw new Error('Hook did not render') + return latest +} + +beforeEach(() => { + vi.clearAllMocks() + vi.spyOn(window, 'open').mockReturnValue({ + location: { href: '' }, + close: vi.fn(), + } as unknown as Window) +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null + latest = null + vi.restoreAllMocks() +}) + +describe('useMemberEnrollment', () => { + /** + * The connect that creates a Sim Search source's connector returns its id, + * but the membership list has no row for it until it refetches, so the + * source is awaited by type until then and by id once the row exists. + */ + it('awaits a first-connected source by type until its membership row appears', () => { + mount() + act(() => enrollment().connectSource('workspace-1', 'google_drive')) + + const [, handlers] = mocks.sourceConnectionMutate.mock.calls[0] + act(() => + handlers.onSuccess({ url: 'https://example.test/enroll', connectorId: 'connector-1' }) + ) + + expect(enrollment().isAwaitingSource('google_drive')).toBe(true) + expect(enrollment().isAwaitingSource('slack')).toBe(false) + expect(enrollment().isAwaiting('connector-1')).toBe(true) + }) + + it('stops awaiting a source once the viewer is connected to its connector', () => { + mount() + act(() => enrollment().connectSource('workspace-1', 'google_drive')) + const [, handlers] = mocks.sourceConnectionMutate.mock.calls[0] + act(() => + handlers.onSuccess({ url: 'https://example.test/enroll', connectorId: 'connector-1' }) + ) + + act(() => root?.render()) + + expect(enrollment().isAwaitingSource('google_drive')).toBe(false) + expect(enrollment().isAwaiting('connector-1')).toBe(false) + }) + + it('does not report an enrollment in an existing connector as an awaited source', () => { + mount() + act(() => enrollment().connect('kb-1', 'connector-1')) + const [, handlers] = mocks.enrollmentMutate.mock.calls[0] + act(() => handlers.onSuccess({ url: 'https://example.test/enroll' })) + + expect(enrollment().isAwaiting('connector-1')).toBe(true) + expect(enrollment().isAwaitingSource('google_drive')).toBe(false) + }) +}) diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index cf99314925a..e1a6ee1c590 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -83,6 +83,16 @@ export function describeMembership({ } } +/** An enrollment tab this surface opened that has not connected yet. */ +interface AwaitingEnrollment { + since: number + /** + * The Sim Search source whose connect created the connector, so the source + * can be told it is awaited before its membership row exists to look it up by. + */ + connectorType: string | null +} + interface UseMemberEnrollmentProps { /** Queries this surface reads memberships from, refreshed while a connection is awaited. */ membershipQueryKeys: readonly QueryKey[] @@ -109,7 +119,9 @@ export function useMemberEnrollment({ const queryClient = useQueryClient() const enrollment = useStartConnectorMemberEnrollment() const sourceConnection = useConnectSimSearchConnector() - const [awaitingSince, setAwaitingSince] = useState>(() => new Map()) + const [awaitingSince, setAwaitingSince] = useState>( + () => new Map() + ) const [popupBlocked, setPopupBlocked] = useState(false) useEffect(() => { @@ -129,7 +141,7 @@ export function useMemberEnrollment({ setAwaitingSince((current) => { const next = new Map( [...current].filter( - ([id, since]) => + ([id, { since }]) => !connectedRef.current.has(id) && now - since < AWAITING_CONNECTION_TIMEOUT_MS ) ) @@ -146,7 +158,7 @@ export function useMemberEnrollment({ /** Opens the tab inside the click, then sends it wherever `start` mints. */ const openEnrollment = ( start: (handlers: { - onSuccess: (url: string, connectorId: string) => void + onSuccess: (url: string, connectorId: string, connectorType?: string) => void onError: () => void }) => void ) => { @@ -158,9 +170,14 @@ export function useMemberEnrollment({ tab.opener = null setPopupBlocked(false) start({ - onSuccess: (url, connectorId) => { + onSuccess: (url, connectorId, connectorType) => { tab.location.href = url - setAwaitingSince((current) => new Map(current).set(connectorId, Date.now())) + setAwaitingSince((current) => + new Map(current).set(connectorId, { + since: Date.now(), + connectorType: connectorType ?? null, + }) + ) }, onError: () => tab.close(), }) @@ -194,7 +211,7 @@ export function useMemberEnrollment({ sourceConnection.mutate( { workspaceId, connectorType, sourceConfig }, { - onSuccess: ({ url, connectorId }) => onSuccess(url, connectorId), + onSuccess: ({ url, connectorId }) => onSuccess(url, connectorId, connectorType), onError: (err) => { onError() logger.error('Failed to connect a Sim Search source', { error: err.message }) @@ -229,6 +246,16 @@ export function useMemberEnrollment({ const isAwaiting = (connectorId: string) => awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId) + /** + * Whether a Sim Search source is awaited by the connect that created its + * connector: the membership list has no row for it until it refetches, so + * the source cannot be looked up by connector id yet. + */ + const isAwaitingSource = (connectorType: string) => + [...awaitingSince].some( + ([id, awaiting]) => awaiting.connectorType === connectorType && !connectedConnectorIds.has(id) + ) + /** The surface reports the latest attempt, whichever path made it. */ const latest = enrollment.submittedAt >= sourceConnection.submittedAt ? enrollment : sourceConnection @@ -239,6 +266,7 @@ export function useMemberEnrollment({ setupConnector, closeSetup: () => setSetupConnector(null), isAwaiting, + isAwaitingSource, isPending: latest.isPending, error: popupBlocked ? POPUP_BLOCKED_MESSAGE : (latest.error?.message ?? null), } diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts index ed7c1a21729..cc3726d4b73 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts @@ -15,6 +15,7 @@ describe('resolveSourceModifiedAt', () => { ['updatedAt', '2026-08-20T12:00:00Z'], ['updated', '2026-08-20T12:00:00Z'], ['lastUpdated', '2026-08-20 12:00:00Z'], + ['statusDate', '2026-08-20T12:00:00.000+0000'], ])('reads %s', (key, value) => { expect(resolveSourceModifiedAt({ [key]: value }, NOW)?.toISOString()).toBe( '2026-08-20T12:00:00.000Z' diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.ts index f39ac86dc61..90fcdec2fb1 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.ts @@ -14,6 +14,8 @@ const SOURCE_MODIFIED_AT_KEYS = [ 'updatedAt', 'updated_at', 'updated', + /** JSM requests: the time the request last changed status, the list endpoint's only change signal. */ + 'statusDate', ] as const /** Earlier than any plausible document; guards against epoch-zero placeholders. */ diff --git a/apps/sim/lib/knowledge/connectors/sync-lock.ts b/apps/sim/lib/knowledge/connectors/sync-lock.ts index 7f71a93c376..4cd4bb13ef6 100644 --- a/apps/sim/lib/knowledge/connectors/sync-lock.ts +++ b/apps/sim/lib/knowledge/connectors/sync-lock.ts @@ -181,6 +181,30 @@ export interface SyncRunLease { beatLive: () => Promise } +/** The lease a document write proves before it lands, as the run that makes it holds it. */ +export type SyncWriteLease = Pick + +/** + * Proves, inside the write's own transaction, that the run still owns the + * connector. A heartbeat taken before the batch only says the lease was held + * then; the hydration and storage work between it and the row write can + * outlast the lease. The share lock keeps the scheduler's reclaim from landing + * until this write commits, and a row that no longer matches aborts the write + * instead of landing stale content over the replacement run's. + */ +export async function assertSyncLeaseHeldInTx( + tx: Pick, + connectorId: string, + lease: SyncWriteLease +): Promise { + const [held] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(lease.stillHeld()) + .for('share') + if (!held) throw new SyncLockLostException(connectorId) +} + /** * The lease of the content sync engine, held through `syncLockToken`. The * heartbeat clock is seeded at lock acquisition, which opened `syncLockLeaseAt`. diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index b7532ac49aa..7fb7f537c22 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -9,7 +9,7 @@ import type { DbOrTx } from '@/lib/db/types' import { textArrayLiteral } from '@/lib/knowledge/access/predicate' import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' -import { SyncLockLostException, type SyncRunLease } from '@/lib/knowledge/connectors/sync-lock' +import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' import type { DocumentData } from '@/lib/knowledge/documents/service' import { StorageService } from '@/lib/uploads' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' @@ -149,30 +149,6 @@ async function isKnowledgeBaseActiveInTx( return rows.length > 0 } -/** The lease a document write proves before it lands, as the run that makes it holds it. */ -export type SyncWriteLease = Pick - -/** - * Proves, inside the write's own transaction, that the run still owns the - * connector. A heartbeat taken before the batch only says the lease was held - * then; the hydration and storage work between it and the row write can - * outlast the lease. The share lock keeps the scheduler's reclaim from landing - * until this write commits, and a row that no longer matches aborts the write - * instead of landing stale content over the replacement run's. - */ -async function assertSyncLeaseHeldInTx( - tx: KnowledgeBaseLockingTx, - connectorId: string, - lease: SyncWriteLease -): Promise { - const [held] = await tx - .select({ id: knowledgeConnector.id }) - .from(knowledgeConnector) - .where(lease.stillHeld()) - .for('share') - if (!held) throw new SyncLockLostException(connectorId) -} - /** * Resolves tag values from connector metadata using the connector's mapTags function. * Translates semantic keys returned by mapTags to actual DB slots using the diff --git a/apps/sim/lib/knowledge/connectors/sync-primitives.ts b/apps/sim/lib/knowledge/connectors/sync-primitives.ts index 83ca0b3d566..f1b3938dbd1 100644 --- a/apps/sim/lib/knowledge/connectors/sync-primitives.ts +++ b/apps/sim/lib/knowledge/connectors/sync-primitives.ts @@ -1769,11 +1769,13 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise { connector.knowledgeBaseId, {}, generateId(), - billingAttribution + billingAttribution, + { connectorId, stillHeld: input.lease.stillHeld } ) result.processingDispatch.accepted += dispatch.accepted result.processingDispatch.failed += dispatch.failed } catch (error) { + if (error instanceof SyncLockLostException) throw error result.processingDispatch.failed += batchDocs.length logger.warn('Failed to enqueue batch for processing — will retry on next sync', { connectorId, @@ -2300,7 +2302,8 @@ export async function sweepStuckDocuments(input: SweepStuckDocumentsInput): Prom knowledgeBaseId, {}, generateId(), - billingAttribution + billingAttribution, + { connectorId, stillHeld: input.lease.stillHeld } ) result.processingDispatch.accepted += dispatch.accepted result.processingDispatch.failed += dispatch.failed diff --git a/apps/sim/lib/knowledge/documents/processing-queue.test.ts b/apps/sim/lib/knowledge/documents/processing-queue.test.ts index 9e5fe8dda47..f5da3cbfafc 100644 --- a/apps/sim/lib/knowledge/documents/processing-queue.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-queue.test.ts @@ -20,6 +20,7 @@ import { markInsideTriggerRun, resetInsideTriggerRunForTests, } from '@/lib/core/config/trigger-runtime' +import { SyncLockLostException } from '@/lib/knowledge/connectors/sync-lock' import { DOCUMENT_PROCESSING_STALE_THRESHOLD_MS } from '@/lib/knowledge/documents/processing-timeouts.server' import { QUEUED_DISPATCH_GRACE_MS } from '@/lib/knowledge/documents/types' @@ -1060,3 +1061,68 @@ describe('processDocumentsWithQueue attempt refund', () => { ).toBe(false) }) }) + +describe('processDocumentsWithQueue under a connector sync lease', () => { + const lease = { + connectorId: 'connector-1', + stillHeld: () => ({ type: 'lease' }) as never, + } + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockBatchTrigger.mockResolvedValue({ batchId: 'batch-1' }) + mockResolveTriggerRegion.mockResolvedValue('us-east-1') + for (const key of Object.keys(env)) { + delete (env as Record)[key] + } + Object.assign(env, { ...defaultMockEnv, TRIGGER_SECRET_KEY: 'trigger-secret' }) + }) + + /** + * The document writes proved the lease in their own transactions; the queue + * write is a later one. A run reclaimed in between must not install a + * processing generation, spend an attempt, or dispatch beside the + * replacement run's own dispatch for the same document. + */ + it('neither marks nor dispatches processing once the lease was reclaimed', async () => { + dbChainMockFns.for.mockResolvedValueOnce([]) + + await expect( + processDocumentsWithQueue( + [DOCUMENT], + 'knowledge-base-1', + {}, + 'request-1', + BILLING_ATTRIBUTION, + lease + ) + ).rejects.toBeInstanceOf(SyncLockLostException) + + expect(dbChainMockFns.where).toHaveBeenCalledWith(lease.stillHeld()) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(mockBatchTrigger).not.toHaveBeenCalled() + }) + + it('queues and dispatches while the lease is still held', async () => { + dbChainMockFns.for.mockResolvedValueOnce([{ id: 'connector-1' }]) + dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }]) + dbChainMockFns.limit.mockResolvedValueOnce([ + { userId: 'knowledge-owner', workspaceId: 'workspace-1' }, + ]) + + await expect( + processDocumentsWithQueue( + [DOCUMENT], + 'knowledge-base-1', + {}, + 'request-1', + BILLING_ATTRIBUTION, + lease + ) + ).resolves.toEqual({ requested: 1, accepted: 1, failed: 0, failedDocumentIds: [] }) + + expect(dbChainMockFns.where).toHaveBeenCalledWith(lease.stillHeld()) + expect(mockBatchTrigger).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 9e0e1db6038..cab2ea2f4f7 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -82,6 +82,7 @@ import { SYSTEM_ACCESS_SCOPE, type SystemAccessScope, } from '@/lib/knowledge/access/types' +import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' import { assertDocumentChunkCountWithinLimit, isPermanentDocumentProcessingError, @@ -820,14 +821,27 @@ async function isDocumentAcceptedWithoutDispatch( return accepted.length > 0 } +/** + * The sync run a connector dispatch proves before it queues processing. The + * document writes prove the lease in their own transactions, but the queue + * write is a later transaction: a run reclaimed in between would otherwise + * install a processing generation, spend an attempt, and dispatch a worker + * beside the replacement run's own dispatch for the same document. + */ +export interface ProcessingDispatchLease extends SyncWriteLease { + connectorId: string +} + async function markDocumentsQueued( documentIds: string[], knowledgeBaseId: string, queueToken: string, - queuedAt: Date + queuedAt: Date, + lease: ProcessingDispatchLease | undefined ): Promise { const legacyAdoptionCutoff = new Date(queuedAt.getTime() - QUEUED_DISPATCH_GRACE_MS) return db.transaction(async (tx) => { + if (lease) await assertSyncLeaseHeldInTx(tx, lease.connectorId, lease) const claimed = await tx .update(document) .set({ @@ -1016,14 +1030,16 @@ async function bestEffortWithdrawDocumentsQueued( * available, or in-process otherwise. Throws only when every dispatch fails; * partial failures are returned and recovered by the next sync's stuck-doc * pass. A successful Trigger.dev hand-off is only an accepted child run, not a - * claim about its eventual processing outcome. + * claim about its eventual processing outcome. A connector sync passes its + * lease, and the queue write then lands only while the run still holds it. */ export async function processDocumentsWithQueue( createdDocuments: DocumentData[], knowledgeBaseId: string, processingOptions: ProcessingOptions, requestId: string, - billingAttribution: BillingAttributionSnapshot | undefined + billingAttribution: BillingAttributionSnapshot | undefined, + lease?: ProcessingDispatchLease ): Promise { const seenDocumentIds = new Set() const uniqueDocuments = createdDocuments.filter((createdDocument) => { @@ -1042,7 +1058,7 @@ export async function processDocumentsWithQueue( generations: queuedGenerations, acceptedWithoutDispatchIds, unresolvedIds, - } = await markDocumentsQueued(documentIds, knowledgeBaseId, requestId, queuedAt) + } = await markDocumentsQueued(documentIds, knowledgeBaseId, requestId, queuedAt, lease) const generationByDocumentId = new Map( queuedGenerations.map((generation) => [generation.documentId, generation]) ) diff --git a/apps/sim/lib/workflows/streaming/streaming-principal.test.ts b/apps/sim/lib/workflows/streaming/streaming-principal.test.ts new file mode 100644 index 00000000000..b74432043c7 --- /dev/null +++ b/apps/sim/lib/workflows/streaming/streaming-principal.test.ts @@ -0,0 +1,82 @@ +/** + * @vitest-environment node + */ +import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createStreamingResponse } from '@/lib/workflows/streaming/streaming' + +const { mockNavigatePathAsync } = vi.hoisted(() => ({ + mockNavigatePathAsync: vi.fn(), +})) + +vi.mock('@/executor/variables/resolvers/reference-async.server', () => ({ + navigatePathAsync: mockNavigatePathAsync, +})) + +const principal = { + kind: 'session', + userId: 'user-1', +} as unknown as WorkflowExecutionPrincipal + +async function drain(stream: ReadableStream): Promise { + const reader = stream.getReader() + try { + while (!(await reader.read()).done) { + /* consume */ + } + } finally { + reader.releaseLock() + } +} + +describe('selected output principal', () => { + beforeEach(() => { + mockNavigatePathAsync.mockReset() + mockNavigatePathAsync.mockImplementation(async (value: unknown, path: string[]) => + path.reduce( + (current, part) => (current as Record | undefined)?.[part], + value + ) + ) + }) + + /** + * A block that completes without the selected path streams nothing, so the + * output is materialized from the final result instead. Both reads must run + * as the principal behind the run, or a member-only knowledge-base file in + * the output is read as nobody on the final-frame path. + */ + it('reads a selected output as the executing principal on the chunk and final paths', async () => { + const stream = await createStreamingResponse({ + requestId: 'request-1', + principal, + streamConfig: { selectedOutputs: ['agent_content'] }, + executeFn: async ({ onBlockComplete }) => { + await onBlockComplete('agent', {}) + const output = { content: 'Done' } + return { + success: true, + output, + logs: [ + { + blockId: 'agent', + output, + startedAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + durationMs: 1, + success: true, + }, + ], + } as never + }, + }) + + await drain(stream) + + expect(mockNavigatePathAsync).toHaveBeenCalledTimes(2) + for (const [, path, context] of mockNavigatePathAsync.mock.calls) { + expect(path).toEqual(['content']) + expect(context.executionContext.principal).toBe(principal) + } + }) +}) diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index 49dca891fd0..7c57eb35586 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -886,6 +886,7 @@ export async function createStreamingResponse( fileKeys: result.metadata?.fileKeys ?? options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, redactToolPayloads: streamConfig.isSecureMode === true, } ) From de8c0703f06d8fb6ca30378533a7f60a97eb8706 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 03:25:47 -0700 Subject: [PATCH 48/76] fix(connectors): validate Gmail's thread cap with the sync parser and reject invalid Date metadata --- apps/sim/connectors/gmail/gmail.test.ts | 11 +++++++++++ apps/sim/connectors/gmail/gmail.ts | 13 +++++++++---- .../knowledge/connectors/source-modified-at.test.ts | 4 ++++ .../lib/knowledge/connectors/source-modified-at.ts | 2 +- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts index 2362fa0011f..7f4922325dc 100644 --- a/apps/sim/connectors/gmail/gmail.test.ts +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -104,3 +104,14 @@ describe('gmail listDocuments with a blank maxThreads', () => { expect(mockFetchWithRetry).not.toHaveBeenCalled() }) }) + +describe('gmail validateConfig maxThreads', () => { + it('refuses what the sync parser would refuse, before any request', async () => { + for (const maxThreads of ['1.5', 'abc', '-1']) { + const result = await gmailConnector.validateConfig('token', { maxThreads }) + expect(result.valid).toBe(false) + expect(result.error).toBe('Max threads must be a non-negative whole number') + } + expect(mockFetchWithRetry).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index a3254c8a338..b25f6243a8c 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -565,10 +565,15 @@ export const gmailConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const maxThreads = sourceConfig.maxThreads as string | undefined - - if (maxThreads && (Number.isNaN(Number(maxThreads)) || Number(maxThreads) <= 0)) { - return { valid: false, error: 'Max threads must be a positive number' } + /** The same parser the sync uses, so a value that saves is a value that syncs. */ + try { + parseDefaultedUnlimitedSafeInteger( + sourceConfig.maxThreads, + DEFAULT_MAX_THREADS, + 'Max threads must be a non-negative whole number' + ) + } catch (error) { + return { valid: false, error: getErrorMessage(error) } } try { diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts index cc3726d4b73..46ac71a8ca1 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts @@ -46,6 +46,10 @@ describe('resolveSourceModifiedAt', () => { ).toBe('2026-08-01T00:00:00.000Z') }) + it('rejects an invalid Date instance', () => { + expect(resolveSourceModifiedAt({ modifiedTime: new Date('not a date') })).toBeNull() + }) + it('rejects placeholders and far-future values', () => { expect(resolveSourceModifiedAt({ modifiedTime: 0 }, NOW)).toBeNull() expect(resolveSourceModifiedAt({ modifiedTime: '1970-01-01T00:00:00Z' }, NOW)).toBeNull() diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.ts index 90fcdec2fb1..5c97903d545 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.ts @@ -24,7 +24,7 @@ const EARLIEST_PLAUSIBLE_MS = Date.UTC(1990, 0, 1) const FUTURE_TOLERANCE_MS = 24 * 60 * 60 * 1000 function toDate(value: unknown): Date | null { - if (value instanceof Date) return value + if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value if (typeof value === 'number' && Number.isFinite(value)) { /** Seconds-since-epoch values are far too small to be milliseconds after 1990. */ return new Date(value < 1e11 ? value * 1000 : value) From 209f32838759c005b6cb1a299997a98b33a24f82 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 03:39:01 -0700 Subject: [PATCH 49/76] fix(knowledge): prove the lease inside every ACL rewrite batch and harden source-modified-time parsing --- .../connectors/member-observations.test.ts | 32 ++++++++++++++ .../connectors/member-observations.ts | 43 +++++++++++++------ .../connectors/member-sync-engine.ts | 5 ++- .../connectors/source-modified-at.test.ts | 9 +++- .../connectors/source-modified-at.ts | 16 ++++--- .../orchestration/connector-access.test.ts | 38 +++++++++++----- .../orchestration/connector-access.ts | 2 + 7 files changed, 112 insertions(+), 33 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/member-observations.test.ts b/apps/sim/lib/knowledge/connectors/member-observations.test.ts index 85163b0e03d..d3471f2563a 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.test.ts @@ -9,10 +9,12 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ })) import { + rewriteConnectorAcls, staleMemberWindowMs, sweepStaleMemberObservations, } from '@/lib/knowledge/connectors/member-observations' import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' +import { SyncLockLostException } from '@/lib/knowledge/connectors/sync-lock' const NOW = new Date('2026-09-01T12:00:00Z') const STALE_MEMBER = { id: 'm-1', connectorId: 'c-1', syncIntervalMinutes: 60 } @@ -88,3 +90,33 @@ describe('sweepStaleMemberObservations', () => { expect(dbChainMockFns.delete).not.toHaveBeenCalled() }) }) + +describe('rewriteConnectorAcls', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('proves the lease inside each batch transaction before rewriting', async () => { + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'd-1' }]) + + await expect( + rewriteConnectorAcls('c-1', [], { lease: { stillHeld: () => 'held' as never } }) + ).resolves.toBe(true) + + expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() + expect(dbChainMockFns.for).toHaveBeenCalledWith('share') + expect(dbChainMockFns.update).toHaveBeenCalledWith(schemaMock.document) + }) + + it('stops without writing once the lease is gone', async () => { + queueTableRows(schemaMock.knowledgeConnector, []) + + await expect( + rewriteConnectorAcls('c-1', [], { lease: { stillHeld: () => 'lost' as never } }) + ).rejects.toBeInstanceOf(SyncLockLostException) + + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index f75def9484d..ac5a5f6cd55 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -27,9 +27,11 @@ import { MEMBER_TOMBSTONE_PURGE_DAYS, } from '@/lib/knowledge/connectors/sync-limits' import { + assertSyncLeaseHeldInTx, connectorIsLive, MEMBER_LOCKABLE_CONNECTOR_STATUSES, type SyncRunLease, + type SyncWriteLease, } from '@/lib/knowledge/connectors/sync-lock' import { type ConnectorSyncDeletionGuard, @@ -184,7 +186,17 @@ const ACCESS_REWRITE_BATCH_SIZE = 1000 export async function rewriteConnectorAcls( connectorId: string, target: readonly string[], - options: { deadlineAt?: number; beforeBatch?: () => Promise } = {} + options: { + deadlineAt?: number + beforeBatch?: () => Promise + /** + * The lease the caller holds on the connector, proved inside each batch's + * transaction: a heartbeat before the batch only says the lease was held + * then, and a run reclaimed mid-rewrite must not land an empty ACL over + * what its replacement has since materialised. + */ + lease?: SyncWriteLease + } = {} ): Promise { const mismatch = target.length === 0 @@ -192,20 +204,23 @@ export async function rewriteConnectorAcls( : sql`${document.acl} <> ${textArrayLiteral(target)}` for (;;) { await options.beforeBatch?.() - const rewritten = await db - .update(document) - .set({ acl: [...target] }) - .where( - eq( - document.id, - sql`ANY(ARRAY( - SELECT ${document.id} FROM ${document} - WHERE ${document.connectorId} = ${connectorId} AND ${mismatch} - LIMIT ${ACCESS_REWRITE_BATCH_SIZE} - ))` + const rewritten = await db.transaction(async (tx) => { + if (options.lease) await assertSyncLeaseHeldInTx(tx, connectorId, options.lease) + return tx + .update(document) + .set({ acl: [...target] }) + .where( + eq( + document.id, + sql`ANY(ARRAY( + SELECT ${document.id} FROM ${document} + WHERE ${document.connectorId} = ${connectorId} AND ${mismatch} + LIMIT ${ACCESS_REWRITE_BATCH_SIZE} + ))` + ) ) - ) - .returning({ id: document.id }) + .returning({ id: document.id }) + }) if (rewritten.length < ACCESS_REWRITE_BATCH_SIZE) return true if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) return false } diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index f77802a1f78..1b6bebf82f7 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -467,7 +467,10 @@ async function insertMemberSyncLog(runId: string, connectorId: string, startedAt * makes it visible again. */ async function finishPendingAccessRewrite(run: MemberSyncRun): Promise { - await rewriteConnectorAcls(run.connectorId, EMPTY_ACL, { beforeBatch: run.lease.beatIfDue }) + await rewriteConnectorAcls(run.connectorId, EMPTY_ACL, { + beforeBatch: run.lease.beatIfDue, + lease: run.lease, + }) await db .update(knowledgeConnector) .set({ accessRewritePending: false, updatedAt: new Date() }) diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts index 46ac71a8ca1..610788e4cb8 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts @@ -46,8 +46,15 @@ describe('resolveSourceModifiedAt', () => { ).toBe('2026-08-01T00:00:00.000Z') }) - it('rejects an invalid Date instance', () => { + it('rejects an invalid Date instance and a number outside the Date range', () => { expect(resolveSourceModifiedAt({ modifiedTime: new Date('not a date') })).toBeNull() + expect(resolveSourceModifiedAt({ modifiedTime: 1e20 })).toBeNull() + }) + + it('reads the last activity a chat space or channel reports', () => { + expect(resolveSourceModifiedAt({ lastActivity: '2026-08-30T10:00:00Z' })?.toISOString()).toBe( + '2026-08-30T10:00:00.000Z' + ) }) it('rejects placeholders and far-future values', () => { diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.ts index 5c97903d545..2741408b4b8 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.ts @@ -16,6 +16,8 @@ const SOURCE_MODIFIED_AT_KEYS = [ 'updated', /** JSM requests: the time the request last changed status, the list endpoint's only change signal. */ 'statusDate', + /** Google Chat spaces and Teams channels: the latest message time, the listing's only change signal. */ + 'lastActivity', ] as const /** Earlier than any plausible document; guards against epoch-zero placeholders. */ @@ -23,16 +25,18 @@ const EARLIEST_PLAUSIBLE_MS = Date.UTC(1990, 0, 1) /** A source clock a day ahead is skew; further ahead is a placeholder. */ const FUTURE_TOLERANCE_MS = 24 * 60 * 60 * 1000 +/** A `Date` only when it holds a real instant; a finite number outside the Date range yields an invalid one. */ +function validDate(date: Date): Date | null { + return Number.isNaN(date.getTime()) ? null : date +} + function toDate(value: unknown): Date | null { - if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value + if (value instanceof Date) return validDate(value) if (typeof value === 'number' && Number.isFinite(value)) { /** Seconds-since-epoch values are far too small to be milliseconds after 1990. */ - return new Date(value < 1e11 ? value * 1000 : value) - } - if (typeof value === 'string' && value.trim()) { - const parsed = new Date(value) - return Number.isNaN(parsed.getTime()) ? null : parsed + return validDate(new Date(value < 1e11 ? value * 1000 : value)) } + if (typeof value === 'string' && value.trim()) return validDate(new Date(value)) return null } diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts index 3c91e25009f..58eabc3ae2e 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts @@ -20,6 +20,11 @@ const mocks = vi.hoisted(() => ({ dispatchMemberSync: vi.fn(), memberAccessAvailable: vi.fn(), provision: vi.fn(), + rewriteAcls: vi.fn(), +})) + +vi.mock('@/lib/knowledge/connectors/member-observations', () => ({ + rewriteConnectorAcls: mocks.rewriteAcls, })) vi.mock('@sim/audit', () => ({ @@ -242,6 +247,7 @@ describe('performUpdateKnowledgeConnectorAccess', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() + mocks.rewriteAcls.mockResolvedValue(true) mocks.grant.mockResolvedValue(undefined) mocks.revoke.mockResolvedValue(undefined) mocks.dispatchSync.mockResolvedValue({ queued: true }) @@ -277,14 +283,21 @@ describe('performUpdateKnowledgeConnectorAccess', () => { queueGroupRow('option-1') dbChainMockFns.returning .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) - /** The rewrite finds nothing left, the flip lands under the lease, then the release. */ - .mockResolvedValueOnce([]) + /** The flip lands under the lease, then the release. */ .mockResolvedValueOnce([{ id: 'c-1' }]) .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, nextMemberSyncAt: new Date() }]) const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) expect(outcome).toMatchObject({ success: true, changed: true }) + /** The rewrite hides every document and proves the switch lease inside each batch. */ + expect(mocks.rewriteAcls).toHaveBeenCalledWith( + 'c-1', + [], + expect.objectContaining({ + lease: expect.objectContaining({ stillHeld: expect.any(Function) }), + }) + ) expect(mocks.grant).toHaveBeenCalledWith( { workspaceId: 'ws-1', @@ -294,7 +307,6 @@ describe('performUpdateKnowledgeConnectorAccess', () => { }, 'admin-1' ) - expect(dbChainMockFns.set).toHaveBeenCalledWith(expect.objectContaining({ acl: [] })) /** The flip is written inside the group's row lock. */ expect(dbChainMockFns.for).toHaveBeenCalledWith('update') expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() @@ -325,9 +337,9 @@ describe('performUpdateKnowledgeConnectorAccess', () => { it('refuses the flip, and undoes the grant, when the option is gone by the time the group is locked', async () => { queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) queueGroupRow() - dbChainMockFns.returning - .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) - .mockResolvedValueOnce([]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }, + ]) const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) @@ -348,7 +360,6 @@ describe('performUpdateKnowledgeConnectorAccess', () => { queueGroupRow('option-2') dbChainMockFns.returning .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) - .mockResolvedValueOnce([]) .mockResolvedValueOnce([{ id: 'c-1' }]) .mockResolvedValueOnce([ { ...MEMBERS_CONNECTOR, credentialGroupId: 'group-2', credentialGroupOptionId: 'option-2' }, @@ -384,16 +395,22 @@ describe('performUpdateKnowledgeConnectorAccess', () => { queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) dbChainMockFns.returning .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) - /** The flip lands under the lease, then the rewrite finds nothing left, then the release. */ + /** The flip lands under the lease, then the release. */ .mockResolvedValueOnce([{ id: 'c-1' }]) - .mockResolvedValueOnce([]) .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, credentialId: 'cred-2' }]) const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) expect(outcome).toMatchObject({ success: true, changed: true }) expect(dbChainMockFns.delete).toHaveBeenCalled() - expect(dbChainMockFns.set).toHaveBeenCalledWith(expect.objectContaining({ acl: ['ws'] })) + /** Workspace access is restored under the switch lease, proved inside each batch. */ + expect(mocks.rewriteAcls).toHaveBeenCalledWith( + 'c-1', + ['ws'], + expect.objectContaining({ + lease: expect.objectContaining({ stillHeld: expect.any(Function) }), + }) + ) expect(mocks.revoke).toHaveBeenCalledWith( { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, 'admin-1' @@ -509,7 +526,6 @@ describe('performUpdateKnowledgeConnectorAccess', () => { queueGroupRow('option-1') dbChainMockFns.returning .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'paused' }]) - .mockResolvedValueOnce([]) .mockResolvedValueOnce([{ id: 'c-1' }]) .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'paused' }]) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index c400541e173..5cfbd6773da 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -335,6 +335,7 @@ export async function performUpdateKnowledgeConnectorAccess( try { const rewritten = await rewriteConnectorAcls(connectorId, EMPTY_ACL, { deadlineAt: deadlineAt, + lease: { stillHeld: () => switchLeaseHeld(connectorId, switchId) }, }) /** * The flip lands under the group's row lock, which the group's option @@ -475,6 +476,7 @@ export async function performUpdateKnowledgeConnectorAccess( }) const rewritten = await rewriteConnectorAcls(connectorId, WORKSPACE_ACL, { deadlineAt: deadlineAt, + lease: { stillHeld: () => switchLeaseHeld(connectorId, switchId) }, }) if (existing.credentialGroupId) { await revokeKnowledgeConnectorCredentialAccess( From dde49209513a26f3b16bf4310414fce9ff8f13b5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 03:48:23 -0700 Subject: [PATCH 50/76] fix(knowledge): claim a member only under a proved lease and read email conversation dates --- .../connectors/member-sync-engine.ts | 31 ++++++++++++------- .../connectors/source-modified-at.test.ts | 6 ++++ .../connectors/source-modified-at.ts | 2 ++ 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 1b6bebf82f7..c1c94cfd8e7 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -52,6 +52,7 @@ import { MEMBER_SYNC_SOFT_BUDGET_SECONDS, } from '@/lib/knowledge/connectors/sync-limits' import { + assertSyncLeaseHeldInTx, createMemberSyncLease, holdsMemberSyncLockToken, MEMBER_LOCKABLE_CONNECTOR_STATUSES, @@ -660,15 +661,22 @@ async function reconcileMembership( * member be aborted at the deadline without touching the others. */ async function claimNextMember(run: MemberSyncRun): Promise { - const [claimed] = await db - .update(knowledgeConnectorMember) - .set({ lastStartedAt: new Date(), updatedAt: new Date() }) - .where( - and( - eq(knowledgeConnectorMember.connectorId, run.connectorId), - eq( - knowledgeConnectorMember.id, - sql`( + /** + * Proved under the lease: a run reclaimed while it slept must not stamp + * `lastStartedAt`, which would hide the member from its replacement's + * selection and defer that member's access updates to a later run. + */ + const [claimed] = await db.transaction(async (tx) => { + await assertSyncLeaseHeldInTx(tx, run.connectorId, run.lease) + return tx + .update(knowledgeConnectorMember) + .set({ lastStartedAt: new Date(), updatedAt: new Date() }) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + eq( + knowledgeConnectorMember.id, + sql`( SELECT ${knowledgeConnectorMember.id} FROM ${knowledgeConnectorMember} WHERE ${knowledgeConnectorMember.connectorId} = ${run.connectorId} AND ${knowledgeConnectorMember.status} = 'active' @@ -678,10 +686,11 @@ async function claimNextMember(run: MemberSyncRun): Promise { LIMIT 1 FOR UPDATE SKIP LOCKED )` + ) ) ) - ) - .returning() + .returning() + }) return claimed ?? null } diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts index 610788e4cb8..492ab74fc31 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts @@ -51,6 +51,12 @@ describe('resolveSourceModifiedAt', () => { expect(resolveSourceModifiedAt({ modifiedTime: 1e20 })).toBeNull() }) + it('reads the newest message time an email conversation reports', () => { + expect( + resolveSourceModifiedAt({ lastMessageDate: '2026-08-29T09:30:00Z' })?.toISOString() + ).toBe('2026-08-29T09:30:00.000Z') + }) + it('reads the last activity a chat space or channel reports', () => { expect(resolveSourceModifiedAt({ lastActivity: '2026-08-30T10:00:00Z' })?.toISOString()).toBe( '2026-08-30T10:00:00.000Z' diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.ts index 2741408b4b8..47ecf923231 100644 --- a/apps/sim/lib/knowledge/connectors/source-modified-at.ts +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.ts @@ -18,6 +18,8 @@ const SOURCE_MODIFIED_AT_KEYS = [ 'statusDate', /** Google Chat spaces and Teams channels: the latest message time, the listing's only change signal. */ 'lastActivity', + /** Gmail and Outlook conversations: the newest message's time. */ + 'lastMessageDate', ] as const /** Earlier than any plausible document; guards against epoch-zero placeholders. */ From fff2aa60e969eda66884c59779f74aca8251ad0e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 10:16:33 -0700 Subject: [PATCH 51/76] feat(search): show the matching passage and the author on each result A result's snippet is the window around the first query term, skipping the header block an email carries, so the row shows why the document matched; its meta line names the source, the person behind the document from its author-like tag, and the date. The agent's citations carry the author too. --- apps/sim/app/api/knowledge/search/route.ts | 2 + .../knowledge-search-results.tsx | 17 ++--- .../components/source-card/source-card.tsx | 8 ++- .../components/special-tags/special-tags.tsx | 3 + .../sim/lib/api/contracts/knowledge/search.ts | 2 + .../tools/server/knowledge/knowledge-base.ts | 4 +- apps/sim/lib/knowledge/search/author.test.ts | 23 +++++++ apps/sim/lib/knowledge/search/author.ts | 34 +++++++++ apps/sim/lib/knowledge/search/snippet.test.ts | 59 ++++++++++++++++ apps/sim/lib/knowledge/search/snippet.ts | 69 +++++++++++++++++++ 10 files changed, 206 insertions(+), 15 deletions(-) create mode 100644 apps/sim/lib/knowledge/search/author.test.ts create mode 100644 apps/sim/lib/knowledge/search/author.ts create mode 100644 apps/sim/lib/knowledge/search/snippet.test.ts create mode 100644 apps/sim/lib/knowledge/search/snippet.ts diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index 3f50b2f384c..96d78f3d1d7 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -7,6 +7,7 @@ import { import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { searchKnowledge } from '@/lib/knowledge/application/search' +import { sourceAuthor } from '@/lib/knowledge/search/author' export const POST = defineInternalJsonRoute({ contract: searchWorkspaceKnowledgeContract, @@ -37,6 +38,7 @@ export const POST = defineInternalJsonRoute({ sourceUrl: result.sourceUrl, connectorType: result.connectorType, sourceModifiedAt: result.sourceModifiedAt?.toISOString() ?? null, + author: sourceAuthor(result.metadata), content: result.content, chunkIndex: result.chunkIndex, similarity: result.similarity, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 8461a7905c2..aaa83c04a4d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -3,6 +3,7 @@ import { useMemo, useState } from 'react' import { Button, Chip } from '@sim/emcn' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' +import { matchSnippet } from '@/lib/knowledge/search/snippet' import { connectorDisplayName } from '@/lib/sim-search/connectors' import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' @@ -18,8 +19,6 @@ const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] /** A search spans at most this many knowledge bases. */ const MAX_SEARCHED_KNOWLEDGE_BASES = 20 -/** Characters of the matching chunk shown under a result. */ -const SNIPPET_LENGTH = 280 /** Filters appear only once a list is long and mixed enough for them to help. */ const FILTERS_MIN_RESULTS = 10 const DAY_MS = 24 * 60 * 60 * 1000 @@ -31,11 +30,6 @@ const UPDATED_WINDOWS = [ ] as const type UpdatedWindow = (typeof UPDATED_WINDOWS)[number]['id'] -function toSnippet(content: string): string { - const flat = content.replace(/\s+/g, ' ').trim() - return flat.length > SNIPPET_LENGTH ? `${flat.slice(0, SNIPPET_LENGTH).trimEnd()}…` : flat -} - /** * One card per document, keeping the best-ranked chunk of each: the list is * already in rank order, so the first chunk seen for a document is its best. @@ -77,7 +71,7 @@ export function indexingSourceNames( * source app, or the knowledge base for an upload. A document without a * source URL cannot be opened. */ -function toSource(result: WorkspaceKnowledgeSearchResult): SourceTagData | null { +function toSource(result: WorkspaceKnowledgeSearchResult, query: string): SourceTagData | null { if (!result.sourceUrl) return null return { url: result.sourceUrl, @@ -86,7 +80,8 @@ function toSource(result: WorkspaceKnowledgeSearchResult): SourceTagData | null ? connectorDisplayName(result.connectorType) : result.knowledgeBaseName || undefined, connectorType: result.connectorType ?? undefined, - snippet: toSnippet(result.content), + snippet: matchSnippet(result.content, query), + author: result.author ?? undefined, updatedAt: result.sourceModifiedAt ?? undefined, } } @@ -251,7 +246,7 @@ export function KnowledgeSearchResults({ ) : (
{visible.map((result) => { - const source = toSource(result) + const source = toSource(result, query) return source ? (

- {toSnippet(result.content)} + {matchSnippet(result.content, query)}

) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index b9773a8c5cf..11b6ad93858 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -129,9 +129,11 @@ export function SourceCard({ source, query, onSummarize }: SourceCardProps) { ? BRAND_ICON_BY_BASE_TYPE.get(source.connectorType) : undefined const updatedAt = parseUpdatedAt(source.updatedAt) - const meta = [sourceLabel(source), updatedAt ? `Updated ${formatDate(updatedAt)}` : null].filter( - (part): part is string => Boolean(part) - ) + const meta = [ + sourceLabel(source), + source.author?.trim() || null, + updatedAt ? formatDate(updatedAt) : null, + ].filter((part): part is string => Boolean(part)) return (
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index f44ac7e935b..307b4f767f0 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -347,6 +347,8 @@ export interface SourceTagData { snippet?: string /** When the source last changed the document, as an ISO timestamp. */ updatedAt?: string + /** The person behind the document, as the source names them. */ + author?: string } export type ContentSegment = @@ -583,6 +585,7 @@ function isSourceTagData(value: unknown): value is SourceTagData { if (value.connectorType !== undefined && typeof value.connectorType !== 'string') return false if (value.snippet !== undefined && typeof value.snippet !== 'string') return false if (value.updatedAt !== undefined && typeof value.updatedAt !== 'string') return false + if (value.author !== undefined && typeof value.author !== 'string') return false return true } diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index cd6abd94584..d2529f30fb4 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -164,6 +164,8 @@ export const workspaceKnowledgeSearchResultSchema = z.object({ sourceUrl: z.string().nullable(), connectorType: z.string().nullable(), sourceModifiedAt: z.string().nullable(), + /** The person behind the document, from its author-like tag; null when the source names none. */ + author: z.string().nullable(), content: z.string(), chunkIndex: z.number(), similarity: z.number(), diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index 56d6c48151e..14e8c1dc1fb 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -52,6 +52,7 @@ import { KNOWLEDGE_TAG_DISPLAY_NAME_MAX_LENGTH, MAX_KNOWLEDGE_BATCH_ITEMS, } from '@/lib/knowledge/constants' +import { sourceAuthor } from '@/lib/knowledge/search/author' import { captureServerEvent } from '@/lib/posthog/server' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' @@ -65,7 +66,7 @@ const DEFAULT_QUERY_TOP_K = 10 * a source URL is quoted by name instead. */ const KNOWLEDGE_CITATION_INSTRUCTION = - 'Cite each result you use inline, right after the sentence it supports, as {"url":,"title":,"siteName":,"connectorType":,"snippet":,"updatedAt":}; omit the tag for a result whose sourceUrl is null and name the document instead.' + 'Cite each result you use inline, right after the sentence it supports, as {"url":,"title":,"siteName":,"connectorType":,"snippet":,"updatedAt":,"author":}; omit the tag for a result whose sourceUrl is null and name the document instead.' /** * Resolves an environment-variable reference passed as a connector API key. @@ -468,6 +469,7 @@ export const knowledgeBaseServerTool: BaseServerTool { + it('prefers the sender of an email and drops the address', () => { + expect(sourceAuthor({ From: '"Ada Lovelace" ', Owner: 'Someone' })).toBe( + 'Ada Lovelace' + ) + }) + + it('falls through the author-like names in order', () => { + expect(sourceAuthor({ Assignee: 'Grace', Owner: 'Alan' })).toBe('Alan') + expect(sourceAuthor({ Reporter: 'Grace' })).toBe('Grace') + }) + + it('returns null when nothing names a person', () => { + expect(sourceAuthor({ From: '', Status: 'Open' })).toBeNull() + expect(sourceAuthor({})).toBeNull() + }) +}) diff --git a/apps/sim/lib/knowledge/search/author.ts b/apps/sim/lib/knowledge/search/author.ts new file mode 100644 index 00000000000..bd6bff80d5e --- /dev/null +++ b/apps/sim/lib/knowledge/search/author.ts @@ -0,0 +1,34 @@ +/** + * The tag names connectors give the person behind a document, in the order + * they are tried. Connectors were never asked to agree on a name, so the + * result's author is derived here rather than in each of them. + */ +const AUTHOR_TAG_NAMES = [ + 'From', + 'Author', + 'Sender', + 'Owner', + 'Organizer', + 'Creator', + 'Reporter', + 'Assignee', +] as const + +/** + * The person a search result shows beside its source: the first author-like + * tag the document carries, reduced to a display name when the connector + * stored an address form such as `Name `. + */ +export function sourceAuthor(metadata: Record): string | null { + for (const name of AUTHOR_TAG_NAMES) { + const value = metadata[name] + if (typeof value !== 'string') continue + const display = value + .replace(/<[^>]*>/g, '') + .trim() + .replace(/^"|"$/g, '') + .trim() + if (display) return display + } + return null +} diff --git a/apps/sim/lib/knowledge/search/snippet.test.ts b/apps/sim/lib/knowledge/search/snippet.test.ts new file mode 100644 index 00000000000..8aeaeaec633 --- /dev/null +++ b/apps/sim/lib/knowledge/search/snippet.test.ts @@ -0,0 +1,59 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + matchSnippet, + SNIPPET_LENGTH, + snippetTerms, + stripLeadingHeaders, +} from '@/lib/knowledge/search/snippet' + +const EMAIL = [ + 'Subject: Invoice #1010 is overdue', + 'From: Support ', + 'To: Someone ', + 'Messages: 1', + '', + `${'Thanks for your patience. '.repeat(12)}The Volvo order shipped on Monday and the tracking number follows. ${'More text here. '.repeat(20)}`, +].join('\n') + +describe('stripLeadingHeaders', () => { + it('drops the header block a connector writes above an email body', () => { + expect(stripLeadingHeaders(EMAIL).startsWith('\nThanks for your patience.')).toBe(true) + }) + + it('leaves a document that does not start with headers alone', () => { + expect(stripLeadingHeaders('Plain prose: with a colon inside.')).toBe( + 'Plain prose: with a colon inside.' + ) + }) +}) + +describe('snippetTerms', () => { + it('keeps distinct terms of three or more characters, longest first', () => { + expect(snippetTerms('the Volvo invoice is volvo')).toEqual(['invoice', 'Volvo', 'volvo', 'the']) + expect(snippetTerms(undefined)).toEqual([]) + }) +}) + +describe('matchSnippet', () => { + it('returns a short document whole, without its headers', () => { + expect(matchSnippet('Subject: Hi\nFrom: A\n\nShort body.', 'body')).toBe('Short body.') + }) + + it('windows around the first query term with ellipses on both sides', () => { + const snippet = matchSnippet(EMAIL, 'volvo') + expect(snippet.startsWith('…')).toBe(true) + expect(snippet.endsWith('…')).toBe(true) + expect(snippet).toContain('The Volvo order shipped') + expect(snippet).not.toContain('Subject:') + expect(snippet.length).toBeLessThanOrEqual(SNIPPET_LENGTH + 2) + }) + + it('falls back to the opening when no term appears in the chunk', () => { + const snippet = matchSnippet(EMAIL, 'unrelated') + expect(snippet.startsWith('Thanks for your patience.')).toBe(true) + expect(snippet.endsWith('…')).toBe(true) + }) +}) diff --git a/apps/sim/lib/knowledge/search/snippet.ts b/apps/sim/lib/knowledge/search/snippet.ts new file mode 100644 index 00000000000..08fb068dec8 --- /dev/null +++ b/apps/sim/lib/knowledge/search/snippet.ts @@ -0,0 +1,69 @@ +/** Characters of a document shown under a search result. */ +export const SNIPPET_LENGTH = 280 +/** Characters kept before the first match, so the hit sits in context rather than at the edge. */ +const LEAD_LENGTH = 90 +/** Query terms shorter than this are too common to anchor a snippet on. */ +const MIN_TERM_LENGTH = 3 +/** `Key: value` lines a connector writes above an email or ticket body. */ +const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/ + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +/** + * The document text without the header block some connectors prefix (the + * `Subject:` / `From:` / `To:` lines of an email): the title already says + * what the subject is, and a snippet spent on the header never shows why the + * document matched. + */ +export function stripLeadingHeaders(content: string): string { + const lines = content.split('\n') + let index = 0 + while (index < lines.length && HEADER_LINE.test(lines[index].trim())) index += 1 + if (index === 0) return content + return lines.slice(index).join('\n') +} + +/** The query's terms worth anchoring on, longest first so the most specific one wins. */ +export function snippetTerms(query: string | undefined): string[] { + return [ + ...new Set( + (query ?? '') + .split(/\s+/) + .map((term) => term.trim()) + .filter((term) => term.length >= MIN_TERM_LENGTH) + ), + ].sort((a, b) => b.length - a.length) +} + +/** + * The passage of a document a search result shows: a window around the first + * query term found, the way a search page shows why a document matched, and + * the document's opening when no term appears in this chunk. Whitespace is + * collapsed and the window is cut on word boundaries with ellipses where the + * text continues. + */ +export function matchSnippet(content: string, query?: string): string { + const flat = stripLeadingHeaders(content).replace(/\s+/g, ' ').trim() + if (flat.length <= SNIPPET_LENGTH) return flat + + let start = 0 + for (const term of snippetTerms(query)) { + const match = new RegExp(`\\b${escapeRegExp(term)}\\b`, 'i').exec(flat) + if (!match) continue + start = Math.max(0, match.index - LEAD_LENGTH) + break + } + if (start > 0) { + const boundary = flat.indexOf(' ', start) + if (boundary !== -1 && boundary - start < LEAD_LENGTH) start = boundary + 1 + } + if (flat.length - start <= SNIPPET_LENGTH) { + return `${start > 0 ? '…' : ''}${flat.slice(start)}` + } + let end = start + SNIPPET_LENGTH + const lastSpace = flat.lastIndexOf(' ', end) + if (lastSpace > start + SNIPPET_LENGTH / 2) end = lastSpace + return `${start > 0 ? '…' : ''}${flat.slice(start, end).trimEnd()}…` +} From 4d887369c7fad35ed547694fe4358bd6462758b3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 10:27:19 -0700 Subject: [PATCH 52/76] refactor(search): style the results and sources as one surface with the composer Result rows take the chat surface's row rhythm with hairlines between them, fade-clipped titles and meta lines, a proper icon button for Copy link, a ghost Summarize matching Answer with Sim, actions revealed on keyboard focus, and a linkless document rendered in the same row with its author, date, and bolded passage. The source and date filters live in the URL beside the query, cleared with it. The sources strip keeps connected chips at full weight, and the member-connector query is gated with an enabled option and cancelled before an optimistic queue write. --- .../knowledge-search-results.tsx | 138 +++++++++++------- .../components/source-card/index.ts | 7 +- .../components/source-card/source-card.tsx | 60 ++++---- .../search-sources/search-sources.tsx | 13 +- .../mode-switcher/mode-switcher.test.tsx | 12 +- .../mode-switcher/mode-switcher.tsx | 15 +- .../app/workspace/[workspaceId]/home/home.tsx | 13 +- .../[workspaceId]/home/search-params.ts | 23 ++- .../workspace/[workspaceId]/search/search.tsx | 2 +- apps/sim/hooks/queries/kb/connectors.ts | 12 +- 10 files changed, 198 insertions(+), 97 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index aaa83c04a4d..69415b70275 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -1,13 +1,26 @@ 'use client' -import { useMemo, useState } from 'react' -import { Button, Chip } from '@sim/emcn' +import { useMemo } from 'react' +import { Button, Chip, OverflowText } from '@sim/emcn' +import { FileText } from '@sim/emcn/icons' +import { formatDate } from '@sim/utils/formatting' +import { useQueryStates } from 'nuqs' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' import { matchSnippet } from '@/lib/knowledge/search/snippet' import { connectorDisplayName } from '@/lib/sim-search/connectors' -import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' +import { + highlightTerms, + SOURCE_ROW_CLASSES, + SOURCE_ROW_MARK_CLASSES, + SourceCard, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import { isIndexing } from '@/app/workspace/[workspaceId]/home/components/search-sources' +import { + resourceUrlKeys, + searchFilterParsers, + UPDATED_WINDOWS, +} from '@/app/workspace/[workspaceId]/home/search-params' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' import { useWorkspaceMemberConnectors, @@ -22,13 +35,8 @@ const MAX_SEARCHED_KNOWLEDGE_BASES = 20 /** Filters appear only once a list is long and mixed enough for them to help. */ const FILTERS_MIN_RESULTS = 10 const DAY_MS = 24 * 60 * 60 * 1000 - -const UPDATED_WINDOWS = [ - { id: 'any', label: 'Any time', days: null }, - { id: '7d', label: 'Past week', days: 7 }, - { id: '30d', label: 'Past month', days: 30 }, -] as const -type UpdatedWindow = (typeof UPDATED_WINDOWS)[number]['id'] +/** Every result without a connector is an upload; the filter names them so. */ +const UPLOAD_SOURCE = 'upload' /** * One card per document, keeping the best-ranked chunk of each: the list is @@ -102,6 +110,41 @@ function handleResultsKeyDown(event: React.KeyboardEvent) { links[next].focus() } +interface UnlinkedResultRowProps { + result: WorkspaceKnowledgeSearchResult + query: string +} + +/** + * A document with nowhere to open, such as an upload: the same row as a + * linked result, with the file mark in place of a brand mark, so the list's + * columns and the matched passage stay aligned whatever the document is. + */ +function UnlinkedResultRow({ result, query }: UnlinkedResultRowProps) { + const meta = [ + result.knowledgeBaseName, + result.author, + result.sourceModifiedAt ? formatDate(new Date(result.sourceModifiedAt)) : null, + ].filter((part): part is string => Boolean(part)) + return ( +
+ + + +
+ + +

+ {highlightTerms(matchSnippet(result.content, query), query)} +

+
+
+ ) +} + interface KnowledgeSearchResultsProps { workspaceId: string query: string @@ -117,7 +160,8 @@ interface KnowledgeSearchResultsProps { * that open the source. A header says how many and that the search ran as * them; while a connected source is still indexing it says so, and the list * grows as documents land. Filters by source and recency appear only once the - * list is long and mixed enough to need them. + * list is long and mixed enough to need them, and live in the URL beside the + * query so a filtered search is a shareable link. */ export function KnowledgeSearchResults({ workspaceId, @@ -152,44 +196,44 @@ export function KnowledgeSearchResults({ */ const memberAccessAvailable = features?.knowledgeMemberAccess === true const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS } = useWorkspaceMemberConnectors( - memberAccessAvailable ? workspaceId : undefined + workspaceId, + { enabled: memberAccessAvailable } ) const indexing = indexingSourceNames(memberConnectors, knowledgeBaseIds) const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) const sourceTypes = useMemo( - () => [...new Set(documents.map((result) => result.connectorType ?? 'upload'))], + () => [...new Set(documents.map((result) => result.connectorType ?? UPLOAD_SOURCE))], [documents] ) - const [sourceFilter, setSourceFilter] = useState(null) - const [updatedFilter, setUpdatedFilter] = useState('any') + const [filters, setFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) const showFilters = documents.length >= FILTERS_MIN_RESULTS && sourceTypes.length > 1 const visible = useMemo(() => { if (!showFilters) return documents - const window = UPDATED_WINDOWS.find((entry) => entry.id === updatedFilter) + const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated) const cutoff = window?.days ? Date.now() - window.days * DAY_MS : null return documents.filter((result) => { - if (sourceFilter && (result.connectorType ?? 'upload') !== sourceFilter) return false + if (filters.source && (result.connectorType ?? UPLOAD_SOURCE) !== filters.source) return false if (cutoff !== null) { const modified = result.sourceModifiedAt ? Date.parse(result.sourceModifiedAt) : Number.NaN if (Number.isNaN(modified) || modified < cutoff) return false } return true }) - }, [documents, showFilters, sourceFilter, updatedFilter]) + }, [documents, showFilters, filters.source, filters.updated]) const failure = basesError ?? error if (failure) { - return

{failure.message}

+ return

{failure.message}

} if (!basesPending && knowledgeBaseIds.length === 0) { return ( -

+

Nothing to search yet. Connect a source above to index what you can open.

) } if (isPending || (isFetching && !results)) { - return

Searching…

+ return

Searching…

} const indexingNote = @@ -198,39 +242,45 @@ export function KnowledgeSearchResults({ : null return ( -
-
- - {documents.length === 1 ? '1 document' : `${documents.length} documents`} · searched as - you - {indexingNote ? ` · ${indexingNote}` : ''} +
+
+ + + {documents.length === 1 ? '1 document' : `${documents.length} documents`} + + {' · searched as you'} + {indexingNote && {indexingNote}} -
{showFilters && ( -
- setSourceFilter(null)}> +
+ setFilters({ source: null })} + > All sources {sourceTypes.map((type) => ( setSourceFilter(sourceFilter === type ? null : type)} + active={filters.source === type} + onClick={() => setFilters({ source: filters.source === type ? null : type })} > - {type === 'upload' ? 'Uploads' : connectorDisplayName(type)} + {type === UPLOAD_SOURCE ? 'Uploads' : connectorDisplayName(type)} ))} - · + {UPDATED_WINDOWS.map((window) => ( setUpdatedFilter(window.id)} + active={filters.updated === window.id} + onClick={() => setFilters({ updated: window.id })} > {window.label} @@ -238,13 +288,13 @@ export function KnowledgeSearchResults({
)} {visible.length === 0 ? ( -

+

{documents.length === 0 ? `No documents you can read match “${query}”.` : 'No documents match these filters.'}

) : ( -
+
{visible.map((result) => { const source = toSource(result, query) return source ? ( @@ -257,17 +307,7 @@ export function KnowledgeSearchResults({ } /> ) : ( -
-

- {result.documentName ?? 'Untitled document'} -

-

- {result.knowledgeBaseName} -

-

- {matchSnippet(result.content, query)} -

-
+ ) })}
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts index ef15ac712fe..2fe0097f08e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts @@ -1 +1,6 @@ -export { highlightTerms, SourceCard } from './source-card' +export { + highlightTerms, + SOURCE_ROW_CLASSES, + SOURCE_ROW_MARK_CLASSES, + SourceCard, +} from './source-card' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index 11b6ad93858..b4b699709a6 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -1,7 +1,7 @@ 'use client' import { type ReactNode, useState } from 'react' -import { Button, cn, Tooltip } from '@sim/emcn' +import { Button, chipIconSlotClass, cn, OverflowText, Tooltip } from '@sim/emcn' import { Check, Link as LinkIcon } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' @@ -26,6 +26,17 @@ const MIN_HIGHLIGHT_TERM_LENGTH = 3 /** How long the copied state shows on the copy-link action. */ const COPIED_FEEDBACK_MS = 1_500 +/** + * The row every source card and its linkless sibling share: the chat surface's + * row rhythm, a hairline between adjacent rows, and the surface fill on hover + * or focus, so a list of results reads like the lists around the composer. + */ +export const SOURCE_ROW_CLASSES = + 'group/source not-prose flex items-start gap-2 border-[var(--border)] px-2 py-2 transition-colors focus-within:bg-[var(--surface-5)] hover-hover:bg-[var(--surface-5)] [&+&]:border-t' + +/** The 16px mark slot, nudged to centre on the title's first line. */ +export const SOURCE_ROW_MARK_CLASSES = cn(chipIconSlotClass, 'mt-[3px]') + function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') } @@ -48,7 +59,7 @@ export function highlightTerms(text: string, query: string | undefined): ReactNo const parts = text.split(pattern) return parts.map((part, index) => index % 2 === 1 ? ( - + {part} ) : ( @@ -79,7 +90,7 @@ function CopyLinkAction({ url }: CopyLinkActionProps) { - {copied ? 'Copied' : 'Copy link'} + {copied ? 'Copied' : 'Copy link'} ) } @@ -117,11 +124,11 @@ interface SourceCardProps { /** * One document a search found, laid out to be scanned: the source's brand - * mark or favicon, the title as a link back to the document, where it lives - * and when it last changed, and the passage that matched with the query terms - * in bold. Actions stay out of the way until the row is hovered or focused. - * The same row serves the composer's search results and the footer of a reply - * that cited its sources with a snippet. + * mark or favicon, the title as a link back to the document, where it lives, + * who it is from, and when it last changed, and the passage that matched with + * the query terms in bold. Actions stay out of the way until the row is + * hovered or its title focused. The same row serves the composer's search + * results and the footer of a reply that cited its sources with a snippet. */ export function SourceCard({ source, query, onSummarize }: SourceCardProps) { const hostname = externalLinkHostname(source.url) @@ -136,8 +143,8 @@ export function SourceCard({ source, query, onSummarize }: SourceCardProps) { ].filter((part): part is string => Boolean(part)) return ( -
- +
+ {ConnectorIcon ? ( ) : hostname ? ( @@ -156,29 +163,24 @@ export function SourceCard({ source, query, onSummarize }: SourceCardProps) { rel='noopener noreferrer' data-source-link='' onClick={(event) => handleExternalLinkClick(event, source.url)} - className={cn( - 'truncate text-[var(--text-primary)] text-sm no-underline hover:underline', - 'underline-offset-2' - )} + className='block min-w-0 text-[var(--text-primary)] text-sm no-underline underline-offset-2 hover:underline' > - {source.title?.trim() || sourceLabel(source)} + -

{meta.join(' · ')}

+ {source.snippet && (

{highlightTerms(source.snippet, query)}

)}
-
+
{onSummarize && ( - )} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx index 64dd5200441..7c54d1e4642 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx @@ -1,7 +1,7 @@ 'use client' import { useMemo } from 'react' -import { Chip } from '@sim/emcn' +import { Chip, chipContentGap, cn } from '@sim/emcn' import { Loader, Plus } from '@sim/emcn/icons' import { canConnectPersonally, @@ -106,8 +106,10 @@ function SourceChip({ } rightIcon={!busy && actionable ? Plus : undefined} @@ -115,7 +117,7 @@ function SourceChip({ busy ? : undefined } > - + {connector.meta.name} {state && {state}} @@ -144,7 +146,8 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { */ const memberAccessAvailable = features?.knowledgeMemberAccess === true const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS } = useWorkspaceMemberConnectors( - memberAccessAvailable ? workspaceId : undefined + workspaceId, + { enabled: memberAccessAvailable } ) const connectionByType = useMemo( () => simSearchConnectionsByType(memberConnectors), @@ -203,7 +206,7 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) { ) })}
- {error &&

{error}

} + {error &&

{error}

} {setupConnector && ( ({ +const { mockCaptureEvent, mockSetSearchQuery, mockSetSearchFilters } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn(), mockSetSearchQuery: vi.fn(), + mockSetSearchFilters: vi.fn(), })) vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), })) -vi.mock('nuqs', () => ({ useQueryState: () => [null, mockSetSearchQuery] })) +vi.mock('nuqs', () => ({ + useQueryState: () => [null, mockSetSearchQuery], + useQueryStates: () => [{}, mockSetSearchFilters], +})) vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) @@ -112,6 +116,10 @@ describe('ModeSwitcher', () => { expect(useMothershipModeStore.getState().mode).toBe('build') expect(mockSetSearchQuery).toHaveBeenCalledWith(null, { history: 'replace', scroll: false }) + expect(mockSetSearchFilters).toHaveBeenCalledWith( + { source: null, updated: null }, + { history: 'replace', scroll: false } + ) }) it('does not report re-selecting the active mode', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx index cb82323125e..c0bfa8ef515 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -11,10 +11,15 @@ import { } from '@sim/emcn' import { Check } from '@sim/emcn/icons' import { useParams } from 'next/navigation' -import { useQueryState } from 'nuqs' +import { useQueryState, useQueryStates } from 'nuqs' import { usePostHog } from 'posthog-js/react' import { captureEvent } from '@/lib/posthog/client' -import { searchQueryParam } from '@/app/workspace/[workspaceId]/home/search-params' +import { + CLEARED_SEARCH_FILTERS, + resourceUrlKeys, + searchFilterParsers, + searchQueryParam, +} from '@/app/workspace/[workspaceId]/home/search-params' import { MOTHERSHIP_MODES, type MothershipMode, @@ -40,12 +45,16 @@ export const ModeSwitcher = memo(function ModeSwitcher() { const setMode = useMothershipModeStore((state) => state.setMode) const [, setSearchQueryParam] = useQueryState(searchQueryParam.key, searchQueryParam.parser) + const [, setSearchFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) /** Leaving Search drops the query from the URL, so a clean URL always means no search is showing. */ const handleSelect = (next: MothershipMode) => { if (next === mode) return setMode(next) - if (next !== 'search') void setSearchQueryParam(null, { history: 'replace', scroll: false }) + if (next !== 'search') { + void setSearchQueryParam(null, { history: 'replace', scroll: false }) + void setSearchFilters(CLEARED_SEARCH_FILTERS, { history: 'replace', scroll: false }) + } captureEvent(posthog, 'chat_mode_changed', { workspace_id: workspaceId, mode: next }) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 4deb957caba..fe8010a467f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -17,7 +17,7 @@ import { PanelLeft } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { useQueryClient } from '@tanstack/react-query' import { useParams, useRouter } from 'next/navigation' -import { useQueryState } from 'nuqs' +import { useQueryState, useQueryStates } from 'nuqs' import { usePostHog } from 'posthog-js/react' import { requestJson } from '@/lib/api/client/request' import { createWorkflowContract } from '@/lib/api/contracts' @@ -50,8 +50,10 @@ import { resolveResourceSelectionUpdate, } from '@/app/workspace/[workspaceId]/home/resource-view-policy' import { + CLEARED_SEARCH_FILTERS, resourceParam, resourceUrlKeys, + searchFilterParsers, searchQueryParam, } from '@/app/workspace/[workspaceId]/home/search-params' import { useFolders } from '@/hooks/queries/folders' @@ -165,9 +167,14 @@ export function Home({ chatId, userName, userId }: HomeProps) { ...resourceUrlKeys, }) const searchQuery = searchQueryValue ?? '' + const [, setSearchFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) + /** A new or cleared query starts from unfiltered results. */ const setSearchQuery = useCallback( - (query: string) => void setSearchQueryParam(query || null), - [setSearchQueryParam] + (query: string) => { + void setSearchQueryParam(query || null) + void setSearchFilters(CLEARED_SEARCH_FILTERS) + }, + [setSearchQueryParam, setSearchFilters] ) /** A link that carries a query opens in Search mode with the query in the box. */ const [initialSearchQuery] = useState(searchQuery) diff --git a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts index cd0a47ffe04..e39e34d1709 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts @@ -1,4 +1,4 @@ -import { parseAsString } from 'nuqs/server' +import { parseAsString, parseAsStringLiteral } from 'nuqs/server' /** * Co-located, typed URL query-param definition for the home/Chat surface. @@ -37,3 +37,24 @@ export const searchQueryParam = { key: 'q', parser: parseAsString, } as const + +/** The recency windows a search can be narrowed to. */ +export const UPDATED_WINDOWS = [ + { id: 'any', label: 'Any time', days: null }, + { id: '7d', label: 'Past week', days: 7 }, + { id: '30d', label: 'Past month', days: 30 }, +] as const +const UPDATED_WINDOW_IDS = UPDATED_WINDOWS.map((window) => window.id) + +/** + * The result filters, beside `q`, so a narrowed search is the same shareable + * link as the search itself. `source` is a connector type or `upload`, absent + * for every source; both are dropped with the query. + */ +export const searchFilterParsers = { + source: parseAsString, + updated: parseAsStringLiteral(UPDATED_WINDOW_IDS).withDefault('any'), +} as const + +/** Every search param at its default: what leaving a search writes. */ +export const CLEARED_SEARCH_FILTERS = { source: null, updated: null } as const diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx index f456582a604..cd90eebf4b8 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/search.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -142,7 +142,7 @@ export function Search() { const setSearchTerm = useDebouncedSearchSetter(setSearchTermParam) const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS, isPending: connectionsPending } = - useWorkspaceMemberConnectors(memberAccessAvailable ? workspaceId : undefined) + useWorkspaceMemberConnectors(workspaceId, { enabled: memberAccessAvailable }) useScrollRestoration(scrollContainerRef, { ready: !memberAccessAvailable || !connectionsPending, }) diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index f24791bd5c7..df57ceb9a3a 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -385,11 +385,14 @@ async function fetchWorkspaceMemberConnectors( } /** Every per-member connector in the workspace and where the viewer stands with each. */ -export function useWorkspaceMemberConnectors(workspaceId?: string) { +export function useWorkspaceMemberConnectors( + workspaceId?: string, + options?: { enabled?: boolean } +) { return useQuery({ queryKey: memberConnectorKeys.list(workspaceId), queryFn: ({ signal }) => fetchWorkspaceMemberConnectors(workspaceId as string, signal), - enabled: Boolean(workspaceId), + enabled: Boolean(workspaceId) && (options?.enabled ?? true), staleTime: WORKSPACE_MEMBER_CONNECTORS_STALE_TIME, placeholderData: keepPreviousData, }) @@ -507,7 +510,10 @@ export function useTriggerSync() { * takes over through `pending` → `syncing` → `active`. */ onMutate: async ({ knowledgeBaseId, connectorId }) => { - await queryClient.cancelQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + await Promise.all([ + queryClient.cancelQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }), + queryClient.cancelQueries({ queryKey: memberConnectorKeys.lists() }), + ]) return optimisticallyQueueSync(queryClient, knowledgeBaseId, connectorId) }, /** From 4b42c2c5c38cded37b703e19992d029921ae9e6e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 10:39:16 -0700 Subject: [PATCH 53/76] fix(search): safe result links, script-aware term matching, and honest header stripping A result links only to an http(s) URL; term matching judges word edges by the surrounding characters instead of ASCII \b and strips quotes from a phrase, and the same matcher bolds the passage; a chunk that is nothing but fields keeps its content; the agent leaves unknown optional citation fields out and keeps the tool's published result count; route errors log the wrapped cause and Postgres code. --- .../knowledge-search-results.tsx | 12 ++-- .../components/source-card/source-card.tsx | 41 +++++-------- .../components/special-tags/index.ts | 1 + .../components/special-tags/special-tags.tsx | 2 +- .../server/knowledge/knowledge-base.test.ts | 2 +- .../tools/server/knowledge/knowledge-base.ts | 4 +- apps/sim/lib/core/utils/with-route-handler.ts | 28 +++++++-- apps/sim/lib/knowledge/search/snippet.test.ts | 42 +++++++++++-- apps/sim/lib/knowledge/search/snippet.ts | 61 +++++++++++++++---- 9 files changed, 139 insertions(+), 54 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 69415b70275..524bc503b3d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -14,7 +14,10 @@ import { SOURCE_ROW_MARK_CLASSES, SourceCard, } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' -import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { + isHttpUrl, + type SourceTagData, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import { isIndexing } from '@/app/workspace/[workspaceId]/home/components/search-sources' import { resourceUrlKeys, @@ -76,11 +79,12 @@ export function indexingSourceNames( /** * A result as the source card renders it: the row's second line names the - * source app, or the knowledge base for an upload. A document without a - * source URL cannot be opened. + * source app, or the knowledge base for an upload. A document without an + * http(s) source URL cannot be opened, and a connector-supplied value of any + * other scheme is never handed to the browser as a link. */ function toSource(result: WorkspaceKnowledgeSearchResult, query: string): SourceTagData | null { - if (!result.sourceUrl) return null + if (!isHttpUrl(result.sourceUrl)) return null return { url: result.sourceUrl, title: result.documentName ?? undefined, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index b4b699709a6..dbbd2f5c5ca 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -7,6 +7,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { formatDate } from '@sim/utils/formatting' import { faviconUrl } from '@/lib/core/utils/favicon' +import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet' import { externalLinkHostname, handleExternalLinkClick, @@ -21,8 +22,6 @@ import { BrandIcon } from '@/blocks/brand-icon' const logger = createLogger('SourceCard') -/** Query terms shorter than this are too common to bold. */ -const MIN_HIGHLIGHT_TERM_LENGTH = 3 /** How long the copied state shows on the copy-link action. */ const COPIED_FEEDBACK_MS = 1_500 @@ -37,35 +36,27 @@ export const SOURCE_ROW_CLASSES = /** The 16px mark slot, nudged to centre on the title's first line. */ export const SOURCE_ROW_MARK_CLASSES = cn(chipIconSlotClass, 'mt-[3px]') -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') -} - /** * The snippet with every query term in bold, so the reader sees why the - * document matched. Terms are matched as whole words, case-insensitively. + * document matched. Terms are matched as whole words in any script, + * case-insensitively, by the same rule the snippet was centred with. */ export function highlightTerms(text: string, query: string | undefined): ReactNode { - const terms = [ - ...new Set( - (query ?? '') - .split(/\s+/) - .map((term) => term.trim()) - .filter((term) => term.length >= MIN_HIGHLIGHT_TERM_LENGTH) - ), - ] - if (terms.length === 0) return text - const pattern = new RegExp(`\\b(${terms.map(escapeRegExp).join('|')})\\b`, 'gi') - const parts = text.split(pattern) - return parts.map((part, index) => - index % 2 === 1 ? ( - - {part} + const matches = findTermMatches(text, queryTerms(query)) + if (matches.length === 0) return text + const parts: ReactNode[] = [] + let cursor = 0 + for (const match of matches) { + if (match.index > cursor) parts.push(text.slice(cursor, match.index)) + parts.push( + + {text.slice(match.index, match.index + match.length)} - ) : ( - part ) - ) + cursor = match.index + match.length + } + if (cursor < text.length) parts.push(text.slice(cursor)) + return parts } function parseUpdatedAt(value: string | undefined): Date | null { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts index 8964244b4eb..609cdc21eca 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts @@ -25,6 +25,7 @@ export { CredentialDisplay, credentialTagHasVisibleCard, formatCredentialSubmissionMessage, + isHttpUrl, PendingTagIndicator, parseCredentialSubmissionMessage, parseCredentialSubmissionProgress, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 307b4f767f0..6add2081368 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -567,7 +567,7 @@ function isMothershipErrorTagData(value: unknown): value is MothershipErrorTagDa * a source. Parsed rather than pattern-matched so a malformed value such as * `https://?` — which a prefix check would accept — never becomes a dead link. */ -function isHttpUrl(value: unknown): value is string { +export function isHttpUrl(value: unknown): value is string { if (typeof value !== 'string' || /\s/.test(value)) return false try { const url = new URL(value) diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts index bf6cadf13c4..8c3757a4cb2 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts @@ -401,7 +401,7 @@ describe('manage_knowledge_base trusted application delegation', () => { workspaceId: 'workspace-paid', knowledgeBaseIds: [KNOWLEDGE_BASE.id], query: '{{KB_QUERY}}', - topK: 10, + topK: 5, resultSecretRegistry: registry, }) expect(mockReadKnowledgeBase).not.toHaveBeenCalled() diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index 14e8c1dc1fb..da47785283e 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -59,14 +59,14 @@ import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-sec const logger = createLogger('KnowledgeBaseServerTool') /** Results a query returns unless the caller asks for a number. */ -const DEFAULT_QUERY_TOP_K = 10 +const DEFAULT_QUERY_TOP_K = 5 /** * How the model cites a knowledge result in its reply. The `` tag is * what the chat renders as a link back to the document, so a result without * a source URL is quoted by name instead. */ const KNOWLEDGE_CITATION_INSTRUCTION = - 'Cite each result you use inline, right after the sentence it supports, as {"url":,"title":,"siteName":,"connectorType":,"snippet":,"updatedAt":,"author":}; omit the tag for a result whose sourceUrl is null and name the document instead.' + 'Cite each result you use inline, right after the sentence it supports, as {"url":,"title":,"siteName":,"connectorType":,"snippet":,"updatedAt":,"author":}; leave out any optional field whose value is null or unknown, and omit the tag for a result whose sourceUrl is null and name the document instead.' /** * Resolves an environment-variable reference passed as a connector API key. diff --git a/apps/sim/lib/core/utils/with-route-handler.ts b/apps/sim/lib/core/utils/with-route-handler.ts index 3e225c3b9d4..70f173986f2 100644 --- a/apps/sim/lib/core/utils/with-route-handler.ts +++ b/apps/sim/lib/core/utils/with-route-handler.ts @@ -1,5 +1,5 @@ import { createLogger, runWithRequestContext } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' +import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' import { getRateLimitHeaders } from '@/lib/api/server/rate-limit-context' @@ -87,6 +87,20 @@ function traceIdFromTraceparent(header: string | null | undefined): string | und return match[1] } +/** + * What a wrapped error hides: a query failure from the database client carries + * the driver's reason and the Postgres code on its cause, and only the outer + * message names the query. + */ +function errorDetail(error: unknown): { cause?: string; code?: string } { + const cause = error instanceof Error && error.cause !== undefined ? error.cause : undefined + const code = getPostgresErrorCode(error) + return { + ...(cause !== undefined ? { cause: getErrorMessage(cause) } : {}), + ...(code ? { code } : {}), + } +} + /** * Wraps a Next.js API route handler with centralized error reporting. * @@ -119,6 +133,7 @@ export function withRouteHandler( } catch (error) { const duration = Date.now() - startTime const message = getErrorMessage(error, 'Unknown error') + const detail = errorDetail(error) if (request.signal.aborted) { logger.info('Client closed request', { duration, status: 499 }) response = options.clientAbortResponse @@ -132,7 +147,12 @@ export function withRouteHandler( if (typedError) { const typedStatus = typedError.statusCode if (typedStatus >= 500) { - logger.error('Unhandled route error', { duration, status: typedStatus, error: message }) + logger.error('Unhandled route error', { + duration, + status: typedStatus, + error: message, + ...detail, + }) } else { logger.warn('Typed route error', { duration, status: typedStatus, error: message }) } @@ -144,13 +164,13 @@ export function withRouteHandler( } if (options.unhandledErrorResponse) { - logger.error('Unhandled route error', { duration, error: message }) + logger.error('Unhandled route error', { duration, error: message, ...detail }) response = options.unhandledErrorResponse({ error, requestId }) applyResponseHeaders(response, request, requestId) return response } - logger.error('Unhandled route error', { duration, error: message }) + logger.error('Unhandled route error', { duration, error: message, ...detail }) response = NextResponse.json({ error: 'Internal server error', requestId }, { status: 500 }) applyResponseHeaders(response, request, requestId) return response diff --git a/apps/sim/lib/knowledge/search/snippet.test.ts b/apps/sim/lib/knowledge/search/snippet.test.ts index 8aeaeaec633..86e91434954 100644 --- a/apps/sim/lib/knowledge/search/snippet.test.ts +++ b/apps/sim/lib/knowledge/search/snippet.test.ts @@ -3,9 +3,10 @@ */ import { describe, expect, it } from 'vitest' import { + findTermMatches, matchSnippet, + queryTerms, SNIPPET_LENGTH, - snippetTerms, stripLeadingHeaders, } from '@/lib/knowledge/search/snippet' @@ -18,6 +19,10 @@ const EMAIL = [ `${'Thanks for your patience. '.repeat(12)}The Volvo order shipped on Monday and the tracking number follows. ${'More text here. '.repeat(20)}`, ].join('\n') +const EVENT = ['Title: Weekly sync', 'Organizer: Ada', 'When: Monday 9am', 'Where: Room 4'].join( + '\n' +) + describe('stripLeadingHeaders', () => { it('drops the header block a connector writes above an email body', () => { expect(stripLeadingHeaders(EMAIL).startsWith('\nThanks for your patience.')).toBe(true) @@ -28,12 +33,35 @@ describe('stripLeadingHeaders', () => { 'Plain prose: with a colon inside.' ) }) + + it('keeps a chunk that is nothing but fields, such as a calendar event', () => { + expect(stripLeadingHeaders(EVENT)).toBe(EVENT) + expect(stripLeadingHeaders(`${EVENT}\n\n`)).toBe(`${EVENT}\n\n`) + }) }) -describe('snippetTerms', () => { +describe('queryTerms', () => { it('keeps distinct terms of three or more characters, longest first', () => { - expect(snippetTerms('the Volvo invoice is volvo')).toEqual(['invoice', 'Volvo', 'volvo', 'the']) - expect(snippetTerms(undefined)).toEqual([]) + expect(queryTerms('the Volvo invoice is volvo')).toEqual(['invoice', 'Volvo', 'volvo', 'the']) + expect(queryTerms(undefined)).toEqual([]) + }) + + it('strips the quotes and punctuation around a term', () => { + expect(queryTerms('"foo bar" (baz),')).toEqual(['foo', 'bar', 'baz']) + }) +}) + +describe('findTermMatches', () => { + it('matches whole words in any script', () => { + expect(findTermMatches('Der Bericht über Zürich.', ['Zürich'])).toEqual([ + { index: 17, length: 6 }, + ]) + expect(findTermMatches('Reports on Zürichsee.', ['Zürich'])).toEqual([]) + expect(findTermMatches('東京の天気', ['天気'])).toEqual([{ index: 3, length: 2 }]) + }) + + it('skips a hit glued to another word character', () => { + expect(findTermMatches('subvolvo volvo_x volvo', ['volvo'])).toEqual([{ index: 17, length: 5 }]) }) }) @@ -51,6 +79,12 @@ describe('matchSnippet', () => { expect(snippet.length).toBeLessThanOrEqual(SNIPPET_LENGTH + 2) }) + it('centres on a quoted phrase and on a non-ASCII term', () => { + expect(matchSnippet(EMAIL, '"Volvo order"')).toContain('The Volvo order shipped') + const german = `${'Einleitung. '.repeat(30)}Die Lieferung nach Zürich ist unterwegs. ${'Mehr. '.repeat(30)}` + expect(matchSnippet(german, 'Zürich')).toContain('nach Zürich') + }) + it('falls back to the opening when no term appears in the chunk', () => { const snippet = matchSnippet(EMAIL, 'unrelated') expect(snippet.startsWith('Thanks for your patience.')).toBe(true) diff --git a/apps/sim/lib/knowledge/search/snippet.ts b/apps/sim/lib/knowledge/search/snippet.ts index 08fb068dec8..fd76cef37f7 100644 --- a/apps/sim/lib/knowledge/search/snippet.ts +++ b/apps/sim/lib/knowledge/search/snippet.ts @@ -6,6 +6,14 @@ const LEAD_LENGTH = 90 const MIN_TERM_LENGTH = 3 /** `Key: value` lines a connector writes above an email or ticket body. */ const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/ +/** + * A character that continues a word, so a term touching one on either side is + * part of a longer word rather than a hit. Scripts written without spaces + * (Han, kana, Hangul, Thai) have no such edges, so their letters never + * disqualify a neighbouring match. + */ +const WORD_CHARACTER = + /(?![\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}\p{sc=Thai}])[\p{L}\p{N}_]/u function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -15,28 +23,60 @@ function escapeRegExp(value: string): string { * The document text without the header block some connectors prefix (the * `Subject:` / `From:` / `To:` lines of an email): the title already says * what the subject is, and a snippet spent on the header never shows why the - * document matched. + * document matched. Only a block the connector closed with a blank line + * counts, and only when a body follows it: a chunk that is nothing but + * `Key: value` fields, such as a calendar event, is the document. */ export function stripLeadingHeaders(content: string): string { const lines = content.split('\n') let index = 0 while (index < lines.length && HEADER_LINE.test(lines[index].trim())) index += 1 - if (index === 0) return content - return lines.slice(index).join('\n') + if (index === 0 || index >= lines.length || lines[index].trim() !== '') return content + const body = lines.slice(index).join('\n') + return body.trim() ? body : content } -/** The query's terms worth anchoring on, longest first so the most specific one wins. */ -export function snippetTerms(query: string | undefined): string[] { +/** + * The query's terms worth matching, longest first so the most specific one + * wins: quotes and other search syntax around a term are not part of it. + */ +export function queryTerms(query: string | undefined): string[] { return [ ...new Set( (query ?? '') .split(/\s+/) - .map((term) => term.trim()) + .map((term) => term.replace(/^["'“”‘’(]+|["'“”‘’),.;:!?]+$/g, '').trim()) .filter((term) => term.length >= MIN_TERM_LENGTH) ), ].sort((a, b) => b.length - a.length) } +export interface TermMatch { + index: number + length: number +} + +/** + * Where the query terms occur in the text as whole words, in order and without + * overlap. Word edges are judged by the characters around a hit rather than + * by `\b`, which knows only ASCII letters, so a term in any script still + * matches; a hit glued to another word character on either side is not a + * word and is skipped. + */ +export function findTermMatches(text: string, terms: readonly string[]): TermMatch[] { + if (terms.length === 0) return [] + const pattern = new RegExp(terms.map(escapeRegExp).join('|'), 'giu') + const matches: TermMatch[] = [] + for (const match of text.matchAll(pattern)) { + const before = text[match.index - 1] + const after = text[match.index + match[0].length] + if (before !== undefined && WORD_CHARACTER.test(before)) continue + if (after !== undefined && WORD_CHARACTER.test(after)) continue + matches.push({ index: match.index, length: match[0].length }) + } + return matches +} + /** * The passage of a document a search result shows: a window around the first * query term found, the way a search page shows why a document matched, and @@ -48,13 +88,8 @@ export function matchSnippet(content: string, query?: string): string { const flat = stripLeadingHeaders(content).replace(/\s+/g, ' ').trim() if (flat.length <= SNIPPET_LENGTH) return flat - let start = 0 - for (const term of snippetTerms(query)) { - const match = new RegExp(`\\b${escapeRegExp(term)}\\b`, 'i').exec(flat) - if (!match) continue - start = Math.max(0, match.index - LEAD_LENGTH) - break - } + const first = findTermMatches(flat, queryTerms(query))[0] + let start = first ? Math.max(0, first.index - LEAD_LENGTH) : 0 if (start > 0) { const boundary = flat.indexOf(' ', start) if (boundary !== -1 && boundary - start < LEAD_LENGTH) start = boundary + 1 From 67f068a48bbb46961a45bc08cdab21260c6aeead Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 10:51:13 -0700 Subject: [PATCH 54/76] fix(search): no cached member rows with access off, no double enrollment, code-point-safe matching Surfaces consume member-connector rows only while the feature is on; the Search page treats an awaited enrollment, including a first connect, as non-actionable; a refused members-mode sync rolls back only the member lists; result actions show on pointers without hover; route error causes go through the redacting describer; the citation template is valid JSON; term edges and snippet windows respect code points. --- .../knowledge-search-results.tsx | 11 +++++--- .../components/source-card/source-card.tsx | 2 +- .../search-sources/search-sources.tsx | 11 +++++--- .../[workspaceId]/search/search.test.tsx | 1 + .../workspace/[workspaceId]/search/search.tsx | 19 +++++++++++--- apps/sim/hooks/queries/kb/connectors.ts | 9 +++++-- .../tools/server/knowledge/knowledge-base.ts | 2 +- apps/sim/lib/core/utils/with-route-handler.ts | 13 +++++----- apps/sim/lib/knowledge/search/snippet.test.ts | 12 +++++++++ apps/sim/lib/knowledge/search/snippet.ts | 26 +++++++++++++++++-- 10 files changed, 82 insertions(+), 24 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 524bc503b3d..a31c3f88377 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -199,10 +199,13 @@ export function KnowledgeSearchResults({ * the viewer will see, and the list is not worth asking for. */ const memberAccessAvailable = features?.knowledgeMemberAccess === true - const { data: memberConnectors = EMPTY_MEMBER_CONNECTORS } = useWorkspaceMemberConnectors( - workspaceId, - { enabled: memberAccessAvailable } - ) + const { data: memberConnectorRows } = useWorkspaceMemberConnectors(workspaceId, { + enabled: memberAccessAvailable, + }) + /** Rows cached before the feature went off are not this surface's to show. */ + const memberConnectors = memberAccessAvailable + ? (memberConnectorRows ?? EMPTY_MEMBER_CONNECTORS) + : EMPTY_MEMBER_CONNECTORS const indexing = indexingSourceNames(memberConnectors, knowledgeBaseIds) const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) const sourceTypes = useMemo( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index dbbd2f5c5ca..4ae460ce909 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -168,7 +168,7 @@ export function SourceCard({ source, query, onSummarize }: SourceCardProps) {

)}
-
+
{onSummarize && ( + + + {label} + + +
+ {sources.map((source) => ( + + ))} +
+
+ ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx index 19ca71fc900..b903e8c2f10 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx @@ -958,6 +958,14 @@ function MessageContentInner({ trailingPendingTag || (isStreamIdle && !trailingStreamActivity && !hasExecutingTool)) + /** The action buttons and, when the reply cited documents, the sources button beside them. */ + const actionsRow = ( +
+ {actions} + {sources.length > 0 && } +
+ ) + return (
@@ -1027,11 +1035,6 @@ function MessageContentInner({ return null } })} - {sources.length > 0 && ( -
- -
- )}
{thinkingExpanded && isLast ? ( // Fixed-height placeholder for the NEXT piece of output: the shimmer @@ -1060,10 +1063,10 @@ function MessageContentInner({ Stopped by user
- {actions &&
{actions}
} + {actions &&
{actionsRow}
} ) : ( - actions &&
{actions}
+ actions &&
{actionsRow}
)}
) diff --git a/apps/sim/lib/copilot/chat/ask-mode.ts b/apps/sim/lib/copilot/chat/ask-mode.ts index e0cc25b88ae..0c54bd90027 100644 --- a/apps/sim/lib/copilot/chat/ask-mode.ts +++ b/apps/sim/lib/copilot/chat/ask-mode.ts @@ -25,6 +25,7 @@ export const ASK_MODE_AGENT_CONTEXT: AskModeAgentContext = { "- Do not use integrations, workflows, tables, files, or the browser. Integration tools are unavailable on this turn. When the question needs live data that is not indexed (today's inbox, a calendar), say that Ask answers from indexed content and suggest Build.", '- Cite every claim with a `` tag exactly as the knowledge tool describes. When nothing relevant is found, say so plainly instead of guessing.', '- Keep the answer short: lead with the answer, then the supporting points.', + '- Suggested follow-ups, when you offer them, are questions the attached sources can answer. Never suggest building, running, or automating anything.', ].join('\n'), } From 108356dd2329cfe2959de1670f305f16340ecf0f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 12:34:48 -0700 Subject: [PATCH 64/76] feat(chat): let the agent use the person's own Credential Group credentials, and keep integrations on Ask turns for questions knowledge cannot answer --- .../executor/utils/credential-token.test.ts | 58 ++++++++++++++++-- apps/sim/executor/utils/credential-token.ts | 30 ++++++++-- .../copilot/auth/application-delegation.ts | 8 ++- apps/sim/lib/copilot/chat/ask-mode.ts | 9 ++- apps/sim/lib/copilot/chat/payload.ts | 6 +- .../copilot/tool-executor/executor.test.ts | 21 ------- .../sim/lib/copilot/tool-executor/executor.ts | 14 ----- apps/sim/lib/copilot/vfs/serializers.ts | 9 ++- apps/sim/lib/copilot/vfs/workspace-vfs.ts | 8 ++- .../application/authorization.test.ts | 48 +++++++++++++-- .../application/authorization.ts | 44 ++++++++++++++ .../workflow-access-policy.test.ts | 16 +++++ .../application/workflow-access-policy.ts | 24 ++++++++ .../copilot-managed-oauth-delegation.test.ts | 39 ++++++++++++ .../copilot-managed-oauth-delegation.ts | 28 +++++++++ .../lib/credentials/application/operations.ts | 2 +- apps/sim/lib/credentials/environment.ts | 60 ++++++++++++++++++- .../credential-visibility.server.ts | 2 +- apps/sim/lib/oauth/token-resolution.ts | 19 +++--- apps/sim/tools/index.ts | 3 + 20 files changed, 371 insertions(+), 77 deletions(-) create mode 100644 apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.test.ts create mode 100644 apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.ts diff --git a/apps/sim/executor/utils/credential-token.test.ts b/apps/sim/executor/utils/credential-token.test.ts index c4a7d64b160..8c5b867fc26 100644 --- a/apps/sim/executor/utils/credential-token.test.ts +++ b/apps/sim/executor/utils/credential-token.test.ts @@ -4,12 +4,19 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutorDelegationOrigin } from '@/executor/types' -const { mockBindExecutorManagedOAuthDelegation, mockResolveCredentialAccessToken } = vi.hoisted( - () => ({ - mockBindExecutorManagedOAuthDelegation: vi.fn(), - mockResolveCredentialAccessToken: vi.fn(), - }) -) +const { + mockBindExecutorManagedOAuthDelegation, + mockCreateCopilotManagedOAuthPrincipal, + mockResolveCredentialAccessToken, +} = vi.hoisted(() => ({ + mockBindExecutorManagedOAuthDelegation: vi.fn(), + mockCreateCopilotManagedOAuthPrincipal: vi.fn(), + mockResolveCredentialAccessToken: vi.fn(), +})) + +vi.mock('@/lib/credentials/application/copilot-managed-oauth-delegation', () => ({ + createCopilotManagedOAuthPrincipal: mockCreateCopilotManagedOAuthPrincipal, +})) vi.mock('@/lib/oauth/token-resolution', () => ({ resolveCredentialAccessToken: mockResolveCredentialAccessToken, @@ -94,6 +101,45 @@ describe('resolveExecutorCredentialToken', () => { expect(mockBindExecutorManagedOAuthDelegation).toHaveBeenCalledWith(ORIGIN, 'managed-1') }) + it('proves a Chat turn through the copilot principal when there is no workflow run', async () => { + mockCreateCopilotManagedOAuthPrincipal.mockReturnValue({ kind: 'delegated' }) + const copilotExecutionContext = { + userId: 'user-1', + workspaceId: 'ws-1', + chatId: 'chat-1', + toolCallId: 'call-1', + copilotToolExecution: true as const, + } + + await resolveExecutorCredentialToken({ + requestId: 'req-1', + credentialId: 'cred-1', + userId: 'user-1', + copilotExecutionContext, + }) + + const input = mockResolveCredentialAccessToken.mock.calls[0][0] + await input.resolveManagedPrincipal('managed-1') + expect(mockCreateCopilotManagedOAuthPrincipal).toHaveBeenCalledWith( + copilotExecutionContext, + 'managed-1' + ) + expect(mockBindExecutorManagedOAuthDelegation).not.toHaveBeenCalled() + }) + + it('leaves managed credentials unproven for a context that is not a trusted Chat call', async () => { + await resolveExecutorCredentialToken({ + requestId: 'req-1', + credentialId: 'cred-1', + userId: 'user-1', + copilotExecutionContext: { userId: 'user-1', workspaceId: 'ws-1' }, + }) + + expect( + mockResolveCredentialAccessToken.mock.calls[0][0].resolveManagedPrincipal + ).toBeUndefined() + }) + it('fails before dispatch when the origin lacks current workflow authority', async () => { await expect( resolveExecutorCredentialToken({ diff --git a/apps/sim/executor/utils/credential-token.ts b/apps/sim/executor/utils/credential-token.ts index 6f02767477d..70fc8a9a69d 100644 --- a/apps/sim/executor/utils/credential-token.ts +++ b/apps/sim/executor/utils/credential-token.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' import { AuthType } from '@/lib/auth/hybrid' +import type { CopilotExecutionContext } from '@/lib/copilot/auth/application-delegation' +import { createCopilotManagedOAuthPrincipal } from '@/lib/credentials/application/copilot-managed-oauth-delegation' import { bindExecutorManagedOAuthDelegation } from '@/lib/credentials/application/managed-oauth-delegation' import { type CredentialTokenPayload, @@ -24,6 +26,11 @@ export interface ResolveExecutorCredentialTokenParams { enforceCredentialAccess?: boolean /** Proves managed-credential delegations in-process when the run carries one. */ executorDelegationOrigin?: ExecutorDelegationOrigin + /** + * The trusted Chat tool call this token is for, when there is no workflow + * run: it proves the signed-in user's own Credential Group credential. + */ + copilotExecutionContext?: CopilotExecutionContext } /** @@ -36,12 +43,28 @@ export interface ResolveExecutorCredentialTokenParams { export async function resolveExecutorCredentialToken( params: ResolveExecutorCredentialTokenParams ): Promise { - const { requestId, credentialId, userId, workflowId, toolId, executorDelegationOrigin } = params + const { + requestId, + credentialId, + userId, + workflowId, + toolId, + executorDelegationOrigin, + copilotExecutionContext, + } = params if (executorDelegationOrigin && !executorDelegationOrigin.currentWorkflow) { throw new Error('Managed credential delegation is missing current workflow authority') } + const resolveManagedPrincipal = executorDelegationOrigin + ? (managedCredentialId: string) => + bindExecutorManagedOAuthDelegation(executorDelegationOrigin, managedCredentialId) + : copilotExecutionContext?.copilotToolExecution + ? async (managedCredentialId: string) => + createCopilotManagedOAuthPrincipal(copilotExecutionContext, managedCredentialId) + : undefined + const result = await resolveCredentialAccessToken({ requestId, credentialId, @@ -55,10 +78,7 @@ export async function resolveExecutorCredentialToken( userId, authType: AuthType.INTERNAL_JWT, }), - resolveManagedPrincipal: executorDelegationOrigin - ? (managedCredentialId: string) => - bindExecutorManagedOAuthDelegation(executorDelegationOrigin, managedCredentialId) - : undefined, + resolveManagedPrincipal, }) if (!result.ok) { diff --git a/apps/sim/lib/copilot/auth/application-delegation.ts b/apps/sim/lib/copilot/auth/application-delegation.ts index 4a779e0a021..e320ab71778 100644 --- a/apps/sim/lib/copilot/auth/application-delegation.ts +++ b/apps/sim/lib/copilot/auth/application-delegation.ts @@ -34,7 +34,7 @@ export class InteractiveCopilotExecutionRequiredError extends Error { export type CopilotResourceScope = Pick< NonNullable, - 'fileId' | 'tableId' + 'fileId' | 'tableId' | 'credentialId' > export interface CopilotDelegationConfiguration { @@ -125,11 +125,17 @@ export function createTrustedCopilotPrincipal( if (options.resourceScope?.tableId !== undefined) { requireNonEmpty(options.resourceScope.tableId, 'a valid table scope') } + if (options.resourceScope?.credentialId !== undefined) { + requireNonEmpty(options.resourceScope.credentialId, 'a valid credential scope') + } const issuedAt = new Date() const resourceScope = Object.freeze({ ...(options.resourceScope?.fileId ? { fileId: options.resourceScope.fileId } : {}), ...(options.resourceScope?.tableId ? { tableId: options.resourceScope.tableId } : {}), + ...(options.resourceScope?.credentialId + ? { credentialId: options.resourceScope.credentialId } + : {}), ...(input.chatId ? { chatId: input.chatId } : {}), ...(input.executionId ? { executionId: input.executionId } : {}), }) diff --git a/apps/sim/lib/copilot/chat/ask-mode.ts b/apps/sim/lib/copilot/chat/ask-mode.ts index 0c54bd90027..d21d3cb6b61 100644 --- a/apps/sim/lib/copilot/chat/ask-mode.ts +++ b/apps/sim/lib/copilot/chat/ask-mode.ts @@ -11,9 +11,8 @@ export const ASK_REQUEST_MODE = 'ask' /** * The instructions an Ask turn carries. Rendered by the agent as an active * skill for the turn, alongside the knowledge bases the composer attached, so - * the model searches them and answers with citations instead of reaching for - * a connected service. The executor refuses integration tools on the turn as - * well; this is what tells the model up front. + * the model searches them first and answers with citations, reaching a + * connected service only when the indexed sources cannot answer. */ export const ASK_MODE_AGENT_CONTEXT: AskModeAgentContext = { type: 'skill', @@ -21,8 +20,8 @@ export const ASK_MODE_AGENT_CONTEXT: AskModeAgentContext = { content: [ 'The person chose Ask: they want an answer drawn from their connected sources, not an action.', '', - '- Answer only from the knowledge bases attached to this message. Search them with the knowledge tool `query` operation, and search again with other phrasings when the first pass returns little. Do not read a base or its metadata first; search.', - "- Do not use integrations, workflows, tables, files, or the browser. Integration tools are unavailable on this turn. When the question needs live data that is not indexed (today's inbox, a calendar), say that Ask answers from indexed content and suggest Build.", + '- Answer from the knowledge bases attached to this message first. Search them with the knowledge tool `query` operation, and search again with other phrasings when the first pass returns little. Do not read a base or its metadata first; search.', + "- Reach for a connected integration only when the indexed sources cannot answer: live or very recent data (today's inbox, a calendar), or an action the person asked for outright. Say which service you used. Never build, run, or schedule anything on an Ask turn.", '- Cite every claim with a `` tag exactly as the knowledge tool describes. When nothing relevant is found, say so plainly instead of guessing.', '- Keep the answer short: lead with the answer, then the supporting points.', '- Suggested follow-ups, when you offer them, are questions the attached sources can answer. Never suggest building, running, or automating anything.', diff --git a/apps/sim/lib/copilot/chat/payload.ts b/apps/sim/lib/copilot/chat/payload.ts index 795cc1f7f36..cec2b962bfe 100644 --- a/apps/sim/lib/copilot/chat/payload.ts +++ b/apps/sim/lib/copilot/chat/payload.ts @@ -412,8 +412,10 @@ export async function buildCopilotRequestPayload( const payloadLogger = logger.withMetadata({ messageId: userMessageId }) // "superagent" is a legacy wire value for Direct Action mode; both modes - // execute connected-service operations through the main-agent gateway. - if (effectiveMode === 'build' || effectiveMode === 'superagent') { + // execute connected-service operations through the main-agent gateway. An + // Ask turn keeps them too: it answers from knowledge first and reaches a + // connected service only when the indexed sources cannot answer. + if (effectiveMode === 'build' || effectiveMode === 'superagent' || effectiveMode === 'ask') { integrationTools = await buildIntegrationToolSchemas( userId, userMessageId, diff --git a/apps/sim/lib/copilot/tool-executor/executor.test.ts b/apps/sim/lib/copilot/tool-executor/executor.test.ts index 317f275618c..47411cefe67 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.test.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.test.ts @@ -151,27 +151,6 @@ describe('copilot tool executor fallback', () => { expect(result).toEqual({ success: true, output: { emails: [] } }) }) - it('refuses integration tools on an Ask turn without dispatching them', async () => { - isKnownTool.mockReturnValue(false) - isSimExecuted.mockReturnValue(false) - - const result = await executeTool( - 'gmail_read', - { maxResults: 10 }, - { - userId: 'user-1', - workflowId: '', - workspaceId: 'ws-1', - chatId: 'chat-1', - requestMode: 'ask', - } - ) - - expect(executeAppTool).not.toHaveBeenCalled() - expect(result.success).toBe(false) - expect(result.error).toContain('Ask mode') - }) - it('forwards trusted authority and cancellation to dynamic custom tools', async () => { isKnownTool.mockReturnValue(false) isSimExecuted.mockReturnValue(false) diff --git a/apps/sim/lib/copilot/tool-executor/executor.ts b/apps/sim/lib/copilot/tool-executor/executor.ts index f8c7f50954b..a39e14880b9 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.ts @@ -1,7 +1,6 @@ import { createLogger } from '@sim/logger' import { type PermissionType, permissionSatisfies } from '@sim/platform-authz/workspace' import { toError } from '@sim/utils/errors' -import { ASK_REQUEST_MODE } from '@/lib/copilot/chat/ask-mode' import { projectToolErrorMessageForCopilot } from '@/lib/copilot/request/tools/resolved-secret-result' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/execution/constants' import { recordSecretUsage } from '@/lib/secrets/usage/record' @@ -34,10 +33,6 @@ export function clearHandlers(): void { handlerRegistry.clear() } -/** An Ask turn answers from the attached knowledge bases; the agent reaches no connected service. */ -const ASK_MODE_INTEGRATION_REFUSAL = - 'Integration tools are not available in Ask mode. Answer from the attached knowledge bases with the knowledge tool (query operation), cite each source, and say so when nothing relevant is found.' - export async function executeTool( toolId: string, params: Record, @@ -78,15 +73,6 @@ export async function executeTool( const normalizedParams = normalizeToolParams(toolId, params, context) - /** - * An Ask turn reaches only Sim-executed server tools. An integration call and - * a headless workflow run are both actions on a connected service or the - * workspace, which an answer drawn from the knowledge bases never takes. - */ - if (context.requestMode === ASK_REQUEST_MODE && !(isKnownTool(toolId) && isSimExecuted(toolId))) { - return { success: false, error: ASK_MODE_INTEGRATION_REFUSAL } - } - const canUseRegisteredHandler = isKnownTool(toolId) && (isSimExecuted(toolId) || usesHeadlessClientFallback) if (!canUseRegisteredHandler) { diff --git a/apps/sim/lib/copilot/vfs/serializers.ts b/apps/sim/lib/copilot/vfs/serializers.ts index 647684e7e78..cf555ba62a1 100644 --- a/apps/sim/lib/copilot/vfs/serializers.ts +++ b/apps/sim/lib/copilot/vfs/serializers.ts @@ -769,8 +769,12 @@ export function serializeCredentials( description?: string | null role?: string | null scope: string | null - /** 'service_account' for a shared app credential; omitted/undefined for a personal OAuth connection. */ - credentialType?: 'oauth' | 'service_account' + /** + * 'service_account' for a shared app credential, 'managed_oauth' for a + * Credential Group credential the person holds through their enrollment; + * omitted/undefined for a personal OAuth connection. + */ + credentialType?: 'oauth' | 'service_account' | 'managed_oauth' createdAt: Date }> ): string { @@ -783,6 +787,7 @@ export function serializeCredentials( role: a.role || undefined, scope: a.scope || undefined, // 'oauth' (personal connection) vs 'service_account' (shared app + // credential) vs 'managed_oauth' (the person's own Credential Group // credential) — they reconnect differently, so the agent must branch on // this. Env-var credentials carry no type. type: a.credentialType, diff --git a/apps/sim/lib/copilot/vfs/workspace-vfs.ts b/apps/sim/lib/copilot/vfs/workspace-vfs.ts index 5e2d841568d..eec6cca7c2a 100644 --- a/apps/sim/lib/copilot/vfs/workspace-vfs.ts +++ b/apps/sim/lib/copilot/vfs/workspace-vfs.ts @@ -120,6 +120,7 @@ import { listCredentialGroups } from '@/lib/credential-groups/service' import { getAccessibleEnvCredentials, getAccessibleOAuthCredentials, + getEnrolledManagedOAuthCredentials, } from '@/lib/credentials/environment' import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' import { BINARY_DOC_TASKS, MAX_DOCUMENT_PREVIEW_CODE_BYTES } from '@/lib/execution/constants' @@ -3174,7 +3175,12 @@ export class WorkspaceVFS { const [envCredentials, oauthCredentials, apiKeyRows, envData, permissionConfig] = await Promise.all([ getAccessibleEnvCredentials(workspaceId, userId, { isWorkspaceAdmin }), - getAccessibleOAuthCredentials(workspaceId, userId, { isWorkspaceAdmin }), + getAccessibleOAuthCredentials(workspaceId, userId, { isWorkspaceAdmin }).then( + async (accessible) => [ + ...accessible, + ...(await getEnrolledManagedOAuthCredentials(workspaceId, userId)), + ] + ), listApiKeys(workspaceId), getPersonalAndWorkspaceEnv(userId, workspaceId), permissionConfigPromise, diff --git a/apps/sim/lib/credential-groups/application/authorization.test.ts b/apps/sim/lib/credential-groups/application/authorization.test.ts index fec8b1a7bbb..cc9a0833452 100644 --- a/apps/sim/lib/credential-groups/application/authorization.test.ts +++ b/apps/sim/lib/credential-groups/application/authorization.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import type { DelegatedPrincipal, WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' import { beforeEach, describe, expect, it, vi } from 'vitest' import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' import { credentialOperations } from '@/lib/credentials/application/operations' @@ -82,10 +82,21 @@ function executorPrincipal(): WorkflowExecutionDelegatedPrincipal { } } -function requireAccess( - principal: WorkflowExecutionDelegatedPrincipal, - accessContext = context -): Promise { +function copilotPrincipal(subjectUserId: string | null = 'user-1'): DelegatedPrincipal { + return { + kind: 'delegated', + serviceId: 'copilot', + ...(subjectUserId ? { subjectUserId } : {}), + workspaceId: 'workspace-1', + delegationId: 'copilot-tool:call-1', + audience: 'sim:managed-oauth-credentials', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: { credentialId: 'credential-1', chatId: 'chat-1' }, + } +} + +function requireAccess(principal: DelegatedPrincipal, accessContext = context): Promise { return requireCredentialGroupCredentialAccess( principal, accessContext, @@ -103,6 +114,33 @@ describe('requireCredentialGroupCredentialAccess', () => { }) }) + it("allows a Chat turn to use only the credential under the signed-in user's own enrollment", async () => { + await expect(requireAccess(copilotPrincipal())).resolves.toBeUndefined() + expect(mocks.loadEnrollmentAccess).toHaveBeenCalledWith('group-1', { + kind: 'sim_user', + userId: 'user-1', + }) + + await expect( + requireAccess(copilotPrincipal(), { ...context, credentialGroupEnrollmentId: 'enrollment-2' }) + ).rejects.toMatchObject({ code: 'forbidden' }) + }) + + it('denies a Chat turn whose user holds no live enrollment, even for an allowlisted workflow', async () => { + mocks.requirePolicy.mockResolvedValue(storedPolicy(['workflow-1'])) + mocks.loadEnrollmentAccess.mockResolvedValue(null) + + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + }) + + it('denies a Chat turn with no Sim user subject before reading anything', async () => { + await expect(requireAccess(copilotPrincipal(null))).rejects.toMatchObject({ + code: 'forbidden', + }) + expect(mocks.requirePolicy).not.toHaveBeenCalled() + expect(mocks.loadEnrollmentAccess).not.toHaveBeenCalled() + }) + it('allows an external actor to use only their own enrollment', async () => { const principal = executorPrincipal() diff --git a/apps/sim/lib/credential-groups/application/authorization.ts b/apps/sim/lib/credential-groups/application/authorization.ts index 13cc444e606..f50ddc066f7 100644 --- a/apps/sim/lib/credential-groups/application/authorization.ts +++ b/apps/sim/lib/credential-groups/application/authorization.ts @@ -12,6 +12,7 @@ import type { import { OrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupWorkflowAccessPolicyCodec, + evaluateCredentialGroupActorCredentialAccess, evaluateCredentialGroupWorkflowAccess, } from '@/lib/credential-groups/application/workflow-access-policy' import type { CredentialGroupCredentialListContext } from '@/lib/credential-groups/credentials' @@ -83,11 +84,54 @@ export function requireCredentialGroupWorkflowActor(principal: Principal): Princ return requireConsistentWorkflowSubject(principal, requireWorkflowExecutionPrincipal(principal)) } +/** + * Authorizes a person using their own Credential Group credential from Chat. + * The copilot delegation names the signed-in user and no workflow, so only the + * actor statement is evaluated: the credential must be the one collected under + * that user's own live enrollment. Nothing the model passes can widen this; + * the acting user is the delegation's subject, not a tool argument. + */ +async function requireCredentialGroupActorCredentialAccess( + principal: Extract, + context: CredentialGroupAuthorizationContext & { credentialGroupEnrollmentId: string }, + resourcePolicy: ResourcePolicyBindingFor<'credential_group'> +): Promise { + const subject = resolvePrincipalSubject(principal) + if (subject?.kind !== 'sim_user' || !subject.userId) { + throw new OrchestrationError('forbidden', 'Credential Group actor access required') + } + const [policy, actorAccess] = await Promise.all([ + requireResourcePolicy({ + workspaceId: context.workspaceId, + resourceType: 'credential_group', + resourceId: context.credentialGroupId, + codec: credentialGroupWorkflowAccessPolicyCodec, + }), + loadCredentialGroupEnrollmentAccessForSubject(context.credentialGroupId, subject), + ]) + if (!actorAccess) { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } + const decision = evaluateCredentialGroupActorCredentialAccess({ + document: policy.document, + credentialGroupId: context.credentialGroupId, + selectedEnrollmentId: context.credentialGroupEnrollmentId, + actorEnrollmentId: actorAccess.enrollmentId, + resourcePolicy, + }) + if (decision.decision !== 'allow') { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } +} + export async function requireCredentialGroupCredentialAccess( principal: Principal, context: CredentialGroupAuthorizationContext & { credentialGroupEnrollmentId: string }, resourcePolicy: ResourcePolicyBindingFor<'credential_group'> ): Promise { + if (principal.kind === 'delegated' && principal.serviceId === 'copilot') { + return requireCredentialGroupActorCredentialAccess(principal, context, resourcePolicy) + } const executionPrincipal = requireWorkflowExecutionPrincipal(principal) const currentWorkflow = requireCurrentWorkflow(principal) const subject = requireConsistentWorkflowSubject(principal, executionPrincipal) diff --git a/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts b/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts index 4c151b17c39..4d2b5b08866 100644 --- a/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts +++ b/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts @@ -8,6 +8,7 @@ import { credentialGroupWorkflowAccessPolicyCodec, decodeCredentialGroupKnowledgeConnectorAccess, decodeCredentialGroupWorkflowAccessPolicy, + evaluateCredentialGroupActorCredentialAccess, evaluateCredentialGroupKnowledgeConnectorAccess, evaluateCredentialGroupWorkflowAccess, requireDefaultCredentialGroupWorkflowAccessPolicy, @@ -434,6 +435,21 @@ describe('Credential Group workflow access policy', () => { ).toThrow('non-default') }) + it("evaluates the actor statement alone when there is no workflow, granting only the actor's own credential", () => { + const document = policy(['workflow-1']) + const evaluate = (selectedEnrollmentId: string) => + evaluateCredentialGroupActorCredentialAccess({ + document, + credentialGroupId: GROUP_ID, + selectedEnrollmentId, + actorEnrollmentId: 'enrollment-1', + resourcePolicy: RESOURCE_POLICY, + }).decision + + expect(evaluate('enrollment-1')).toBe('allow') + expect(evaluate('enrollment-2')).toBe('implicit_deny') + }) + it('evaluates actor ownership and deployed workflow access through registered statements', () => { const document = policy(['workflow-1']) expect( diff --git a/apps/sim/lib/credential-groups/application/workflow-access-policy.ts b/apps/sim/lib/credential-groups/application/workflow-access-policy.ts index 12fe8f8d34e..ff8a340ebac 100644 --- a/apps/sim/lib/credential-groups/application/workflow-access-policy.ts +++ b/apps/sim/lib/credential-groups/application/workflow-access-policy.ts @@ -480,6 +480,30 @@ export function evaluateCredentialGroupWorkflowAccess(input: { }) } +/** + * Decides whether the person acting on their own behalf, outside any workflow + * run, may use a credential: only the actor statement can match, and it grants + * exactly the credential collected under the actor's own enrollment. The + * workflow statements need a current workflow fact and never match here. + */ +export function evaluateCredentialGroupActorCredentialAccess(input: { + document: CredentialGroupWorkflowAccessPolicy + credentialGroupId: string + selectedEnrollmentId: string + actorEnrollmentId: string + resourcePolicy: ResourcePolicyBindingFor<'credential_group'> +}): ResourcePolicyDecision { + const document = parseCanonicalDocument(input.document, input.credentialGroupId) + return evaluateResourcePolicy({ + document, + action: input.resourcePolicy.action, + facts: { + credentialGroupActorEnrollmentId: input.actorEnrollmentId, + credentialGroupCredentialEnrollmentId: input.selectedEnrollmentId, + }, + }) +} + /** * Decides whether a knowledge connector may use a credential collected under * one option. There is no actor and no workflow: the connector is the principal diff --git a/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.test.ts b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.test.ts new file mode 100644 index 00000000000..372d0fbe909 --- /dev/null +++ b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.test.ts @@ -0,0 +1,39 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { createCopilotManagedOAuthPrincipal } from '@/lib/credentials/application/copilot-managed-oauth-delegation' + +const trustedContext = { + userId: 'user-1', + workspaceId: 'workspace-1', + chatId: 'chat-1', + toolCallId: 'call-1', + copilotToolExecution: true as const, +} + +describe('createCopilotManagedOAuthPrincipal', () => { + it('names the signed-in user, the managed-credential audience, and exactly one credential', () => { + const principal = createCopilotManagedOAuthPrincipal(trustedContext, 'credential-1') + + expect(principal).toMatchObject({ + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'copilot-tool:call-1', + audience: 'sim:managed-oauth-credentials', + resourceScope: { credentialId: 'credential-1', chatId: 'chat-1' }, + }) + expect(principal.expiresAt.getTime()).toBeGreaterThan(principal.issuedAt.getTime()) + }) + + it('refuses a context the server did not classify as a Chat tool call', () => { + expect(() => + createCopilotManagedOAuthPrincipal({ ...trustedContext, copilotToolExecution: false }, 'c-1') + ).toThrow('trusted Copilot execution context') + expect(() => createCopilotManagedOAuthPrincipal(undefined, 'c-1')).toThrow( + 'Copilot execution context is required' + ) + }) +}) diff --git a/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.ts b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.ts new file mode 100644 index 00000000000..4df285fca00 --- /dev/null +++ b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.ts @@ -0,0 +1,28 @@ +import type { DelegatedPrincipal } from '@sim/auth/principal' +import { + COPILOT_APPLICATION_DELEGATION_TTL_MS, + type CopilotExecutionContext, + createCopilotApplicationPrincipal, + requireTrustedCopilotExecutionContext, +} from '@/lib/copilot/auth/application-delegation' +import { MANAGED_OAUTH_DELEGATION_AUDIENCE } from '@/lib/credentials/application/authorization' + +/** + * The principal a Chat tool call presents for one managed credential: a copilot + * delegation naming the signed-in user, scoped to that credential, with no + * workflow. The credential-group authorization evaluates its actor statement + * against this subject, so the person can use the credential they collected + * under their own enrollment and nothing else. + */ +export function createCopilotManagedOAuthPrincipal( + context: CopilotExecutionContext | undefined, + credentialId: string +): DelegatedPrincipal { + const trustedContext = requireTrustedCopilotExecutionContext(context) + return createCopilotApplicationPrincipal(trustedContext, { + audience: MANAGED_OAUTH_DELEGATION_AUDIENCE, + ttlMs: COPILOT_APPLICATION_DELEGATION_TTL_MS, + createDelegationId: (trusted) => `copilot-tool:${trusted.toolCallId}`, + resourceScope: { credentialId }, + }) +} diff --git a/apps/sim/lib/credentials/application/operations.ts b/apps/sim/lib/credentials/application/operations.ts index c851eaac563..df3e52834c6 100644 --- a/apps/sim/lib/credentials/application/operations.ts +++ b/apps/sim/lib/credentials/application/operations.ts @@ -177,7 +177,7 @@ export const credentialOperations = { workspaceApiKey: 'deny', capability: 'integrations.manage', principalKinds: ['delegated'], - delegatedServices: ['executor'], + delegatedServices: ['executor', 'copilot'], resourcePolicy: { resourceType: 'credential_group', action: CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION, diff --git a/apps/sim/lib/credentials/environment.ts b/apps/sim/lib/credentials/environment.ts index 9756c4659e7..c1a74677e12 100644 --- a/apps/sim/lib/credentials/environment.ts +++ b/apps/sim/lib/credentials/environment.ts @@ -1,8 +1,11 @@ import { db } from '@sim/db' import { credential, + credentialGroup, + credentialGroupEnrollment, credentialMember, permissions, + user, workspace, workspaceEnvironment, } from '@sim/db/schema' @@ -11,6 +14,7 @@ import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { and, asc, eq, inArray, isNotNull, isNull, notInArray, or, sql } from 'drizzle-orm' import { acquireUserBillingIdentityLock } from '@/lib/billing/organizations/billing-identity-lock' +import { LIVE_ENROLLMENT_STATUSES } from '@/lib/credential-groups/credentials' import type { DbOrTx } from '@/lib/db/types' import { getEffectiveWorkspacePermission, @@ -829,11 +833,63 @@ export interface AccessibleOAuthCredential { providerId: string displayName: string role: 'admin' | 'member' - /** Distinguishes a personal OAuth connection from a shared service account. */ - type: 'oauth' | 'service_account' + /** + * A personal OAuth connection, a shared service account, or a Credential + * Group credential the person collected under their own enrollment. + */ + type: 'oauth' | 'service_account' | 'managed_oauth' updatedAt: Date } +/** + * The Credential Group credentials a verified person holds through their own + * live enrollments in the workspace: active managed OAuth rows whose enrollment + * email is the person's. These are theirs to use as themselves; the policy's + * actor statement is what a use is authorized against, so nothing here widens + * access, it only tells the person (and the agent acting for them) what exists. + */ +export async function getEnrolledManagedOAuthCredentials( + workspaceId: string, + userId: string +): Promise { + const rows = await db + .select({ + id: credential.id, + providerId: credential.providerId, + displayName: credential.displayName, + groupName: credentialGroup.name, + updatedAt: credential.updatedAt, + }) + .from(credential) + .innerJoin( + credentialGroupEnrollment, + eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) + ) + .innerJoin(credentialGroup, eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId)) + .innerJoin(user, eq(sql`lower(btrim(${user.email}))`, credentialGroupEnrollment.email)) + .where( + and( + eq(credential.workspaceId, workspaceId), + eq(credential.type, 'managed_oauth'), + eq(credential.managedOauthStatus, 'active'), + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]), + eq(user.id, userId), + eq(user.emailVerified, true) + ) + ) + + return rows + .filter((row): row is typeof row & { providerId: string } => Boolean(row.providerId)) + .map((row) => ({ + id: row.id, + providerId: row.providerId, + displayName: `${row.displayName} (${row.groupName})`, + role: 'member' as const, + type: 'managed_oauth' as const, + updatedAt: row.updatedAt, + })) +} + export async function getAccessibleOAuthCredentials( workspaceId: string, userId: string, diff --git a/apps/sim/lib/integrations/credential-visibility.server.ts b/apps/sim/lib/integrations/credential-visibility.server.ts index 1837aa67571..929f18cfb7b 100644 --- a/apps/sim/lib/integrations/credential-visibility.server.ts +++ b/apps/sim/lib/integrations/credential-visibility.server.ts @@ -16,7 +16,7 @@ import { isHiddenUnder } from '@/blocks/visibility/context' export interface IntegrationCredentialIdentity { providerId: string - type?: 'oauth' | 'service_account' + type?: 'oauth' | 'service_account' | 'managed_oauth' } interface IntegrationCredentialVisibilityOptions { diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index ba903ef7eb5..dfd1f682b5b 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -1,8 +1,5 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { - resolvePrincipalSubject, - type WorkflowExecutionDelegatedPrincipal, -} from '@sim/auth/principal' +import { type DelegatedPrincipal, resolvePrincipalSubject } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { impersonateEmailSchema, @@ -326,13 +323,13 @@ export interface ResolveCredentialAccessTokenInput */ authenticate: () => AuthResult | Promise /** - * Proves a workflow-execution delegation for one managed credential. The route - * verifies the delegation JWT header; the executor binds its delegation origin - * in-process. Absent, managed credentials are rejected with - * `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. Must throw - * {@link InvalidManagedOAuthDelegationError} on an invalid delegation. + * Proves a delegation for one managed credential: a workflow execution (the + * route verifies the delegation JWT header; the executor binds its delegation + * origin in-process) or a Chat turn acting as the signed-in user. Absent, + * managed credentials are rejected with `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. + * Must throw {@link InvalidManagedOAuthDelegationError} on an invalid delegation. */ - resolveManagedPrincipal?: (credentialId: string) => Promise + resolveManagedPrincipal?: (credentialId: string) => Promise } /** @@ -376,7 +373,7 @@ export async function resolveCredentialAccessToken( } } - let principal: WorkflowExecutionDelegatedPrincipal + let principal: DelegatedPrincipal try { principal = await input.resolveManagedPrincipal(resolved.credentialId) } catch (error) { diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index defef5a4ad8..df21ead22e6 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -1894,6 +1894,9 @@ async function executeToolImplementation( impersonateEmail, enforceCredentialAccess, executorDelegationOrigin: executionContext?.executorDelegationOrigin, + ...(operationContext?.copilotToolExecution + ? { copilotExecutionContext: operationContext } + : {}), }) } else { data = await fetchCredentialTokenFromRoute({ From 961eb78c103836d080dcad349b2d1a86bdec9c90 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 12:39:34 -0700 Subject: [PATCH 65/76] fix(knowledge): treat a reclaimed lease during a member purge as superseded; tidy the chat reply sources after cleanup --- .../message-sources/message-sources.tsx | 9 ++--- .../components/source-card/source-card.tsx | 9 +++-- .../message-content/message-content.tsx | 1 - .../app/workspace/[workspaceId]/home/home.tsx | 18 ++++------ .../connectors/member-observations.test.ts | 35 +++++++++++++++++++ .../connectors/member-observations.ts | 24 +++++++++---- 6 files changed, 67 insertions(+), 29 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx index 3d5cc16a1bb..5b32e024ed3 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx @@ -7,7 +7,7 @@ import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/component /** The action-row button, matching the copy and vote buttons beside it with room for a count. */ const BUTTON_CLASSES = - 'flex h-[26px] items-center gap-1 rounded-[6px] px-1.5 text-[var(--text-icon)] text-caption transition-colors hover-hover:bg-[var(--surface-hover)] focus-visible:outline-none data-[state=open]:bg-[var(--surface-hover)]' + 'flex h-[26px] items-center gap-1 rounded-[6px] px-1.5 text-[var(--text-icon)] text-caption transition-colors hover-hover:bg-[var(--surface-hover)] focus-visible:outline-none data-[state=open]:bg-[var(--surface-active)] data-[state=open]:hover-hover:bg-[var(--surface-active)]' interface MessageSourcesProps { sources: readonly SourceTagData[] @@ -36,12 +36,7 @@ export function MessageSources({ sources }: MessageSourcesProps) { {label} - +
{sources.map((source) => ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx index b4ea7f2f875..4dcf591b77e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -151,7 +151,12 @@ export function SourceCard({ source, query, onSummarize, dense = false }: Source if (dense) { return ( -
+
{mark}
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx index b903e8c2f10..c72f9b89fbb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx @@ -958,7 +958,6 @@ function MessageContentInner({ trailingPendingTag || (isStreamIdle && !trailingStreamActivity && !hasExecutingTool)) - /** The action buttons and, when the reply cited documents, the sources button beside them. */ const actionsRow = (
{actions} diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index b89a1fd7b8e..b6208edaec1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -198,9 +198,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { /** The bases an Ask turn is grounded in; read through a ref so a list refresh never rebuilds the submit handler. */ const { data: knowledgeBases = EMPTY_KNOWLEDGE_BASES } = useKnowledgeBasesQuery(workspaceId) const knowledgeBasesRef = useRef(knowledgeBases) - useEffect(() => { - knowledgeBasesRef.current = knowledgeBases - }, [knowledgeBases]) + knowledgeBasesRef.current = knowledgeBases const hasCheckedLandingStorageRef = useRef(false) const initialViewInputRef = useRef(null) const initialViewUserInputRef = useRef(null) @@ -503,7 +501,6 @@ export function Home({ chatId, userName, userId }: HomeProps) { } prepareResourceViewForAgentTurn() - /** Ask is a turn of the agent grounded in the searched sources. */ const turnContexts = mode === 'ask' ? withSearchedKnowledgeContexts( @@ -525,14 +522,11 @@ export function Home({ chatId, userName, userId }: HomeProps) { const clearSearch = useCallback(() => setSearchQuery(''), [setSearchQuery]) /** Summarize or Answer on a result: hand the question to the agent in Ask mode. */ - const handleSummarize = useCallback( - (prompt: string) => { - useMothershipModeStore.getState().setMode('ask') - setSearchQuery('') - handleSubmit(prompt) - }, - [handleSubmit, setSearchQuery] - ) + const handleSummarize = (prompt: string) => { + useMothershipModeStore.getState().setMode('ask') + setSearchQuery('') + handleSubmit(prompt) + } /** * A chat that already exists never opens in Search: its transcript is a * conversation, and search results never join it. Build and Ask both carry diff --git a/apps/sim/lib/knowledge/connectors/member-observations.test.ts b/apps/sim/lib/knowledge/connectors/member-observations.test.ts index d3471f2563a..5c34c588f33 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.test.ts @@ -5,16 +5,23 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@s import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/knowledge/documents/service', () => ({ + ConnectorSyncDeletionGuardError: class ConnectorSyncDeletionGuardError extends Error {}, hardDeleteDocuments: vi.fn(), })) +import { db } from '@sim/db' import { + applyMemberDocumentLifecycle, rewriteConnectorAcls, staleMemberWindowMs, sweepStaleMemberObservations, } from '@/lib/knowledge/connectors/member-observations' import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' import { SyncLockLostException } from '@/lib/knowledge/connectors/sync-lock' +import { + ConnectorSyncDeletionGuardError, + hardDeleteDocuments, +} from '@/lib/knowledge/documents/service' const NOW = new Date('2026-09-01T12:00:00Z') const STALE_MEMBER = { id: 'm-1', connectorId: 'c-1', syncIntervalMinutes: 60 } @@ -120,3 +127,31 @@ describe('rewriteConnectorAcls', () => { expect(dbChainMockFns.update).not.toHaveBeenCalled() }) }) + +describe('applyMemberDocumentLifecycle', () => { + beforeEach(() => { + resetDbChainMock() + vi.mocked(hardDeleteDocuments).mockReset() + }) + + it('reports a reclaimed lease during a purge batch as the run being superseded', async () => { + dbChainMockFns.returning.mockResolvedValueOnce([]) + queueTableRows(schemaMock.document, []) + queueTableRows(schemaMock.document, [{ id: 'd-1' }]) + vi.mocked(hardDeleteDocuments).mockRejectedValueOnce( + new ConnectorSyncDeletionGuardError('lease reclaimed') + ) + + await expect( + applyMemberDocumentLifecycle({ + connectorId: 'c-1', + knowledgeBaseId: 'kb-1', + runId: 'run-1', + withLease: (fn) => fn(db as never), + failedExternalIds: new Set(), + allowRemoval: true, + lease: { beatIfDue: async () => {} } as never, + }) + ).rejects.toBeInstanceOf(SyncLockLostException) + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts index ac5a5f6cd55..361db08c282 100644 --- a/apps/sim/lib/knowledge/connectors/member-observations.ts +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -30,11 +30,13 @@ import { assertSyncLeaseHeldInTx, connectorIsLive, MEMBER_LOCKABLE_CONNECTOR_STATUSES, + SyncLockLostException, type SyncRunLease, type SyncWriteLease, } from '@/lib/knowledge/connectors/sync-lock' import { type ConnectorSyncDeletionGuard, + ConnectorSyncDeletionGuardError, hardDeleteDocuments, } from '@/lib/knowledge/documents/service' @@ -364,13 +366,21 @@ export async function applyMemberDocumentLifecycle(input: { const purgeIds = purgeCandidates.map((row) => row.id) for (let offset = 0; offset < purgeIds.length; offset += PURGE_CHUNK_SIZE) { await input.lease.beatIfDue() - purged += await hardDeleteDocuments( - purgeIds.slice(offset, offset + PURGE_CHUNK_SIZE), - runId, - connectorId, - knowledgeBaseId, - guard - ) + try { + purged += await hardDeleteDocuments( + purgeIds.slice(offset, offset + PURGE_CHUNK_SIZE), + runId, + connectorId, + knowledgeBaseId, + guard + ) + } catch (error) { + /** The deletion guard refusing the lease is a reclaimed run, not a failed one. */ + if (error instanceof ConnectorSyncDeletionGuardError) { + throw new SyncLockLostException(connectorId) + } + throw error + } } return { tombstoned: tombstoned.length, resurrected: resurrected.length, purged } From f52b3521c9a5f944e767a0969ad5c8d7e1eaf932 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 12:50:01 -0700 Subject: [PATCH 66/76] improvement(home): fold Ask into Search behind an Answer toggle --- .../suggested-actions.test.tsx | 10 --- .../suggested-actions/suggested-actions.tsx | 3 +- .../answer-toggle/answer-toggle.test.tsx | 71 +++++++++++++++++++ .../answer-toggle/answer-toggle.tsx | 44 ++++++++++++ .../components/answer-toggle/index.ts | 1 + .../components/user-input/components/index.ts | 1 + .../mode-switcher/mode-switcher.test.tsx | 7 +- .../mode-switcher/mode-switcher.tsx | 3 +- .../home/components/user-input/user-input.tsx | 2 + .../app/workspace/[workspaceId]/home/home.tsx | 43 ++++++----- apps/sim/lib/copilot/chat/ask-mode.ts | 4 +- apps/sim/lib/posthog/events.ts | 8 ++- apps/sim/stores/mothership-mode/store.ts | 16 +++-- 13 files changed, 167 insertions(+), 46 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx index de6914153ac..b0674f4774b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx @@ -128,14 +128,4 @@ describe('SuggestedActions', () => { expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() expect(rows()).toHaveLength(0) }) - - it('shows the sources in Ask mode, which answers from them', () => { - mount() - - act(() => useMothershipModeStore.getState().setMode('ask')) - - expect(heading()).toBe('Sources') - expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() - expect(rows()).toHaveLength(0) - }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx index d1bf8e20b80..889c988aef7 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx @@ -234,7 +234,6 @@ const INITIAL_ACTIONS: Action[] = [ /** Section heading per composer mode — Search reads as a connect-your-sources list. */ const HEADINGS: Record = { build: 'Suggested actions', - ask: 'Sources', search: 'Sources', } @@ -373,7 +372,7 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { `collapsible-up`/`-down` interpolate height alone, so a margin here would hold its full value through the close and then vanish on unmount, snapping the content below up. */} - {mode !== 'build' && workspaceId ? ( + {mode === 'search' && workspaceId ? (
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx new file mode 100644 index 00000000000..5c5e16e3e0f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx @@ -0,0 +1,71 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCaptureEvent } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn() })) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) +vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) + +import { AnswerToggle } from '@/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle' +import { useMothershipModeStore } from '@/stores/mothership-mode/store' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function button(): HTMLButtonElement | null { + return container?.querySelector('button') ?? null +} + +beforeEach(() => { + mockCaptureEvent.mockClear() + useMothershipModeStore.getState().reset() +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('AnswerToggle', () => { + it('renders nothing outside Search mode', () => { + mount() + expect(button()).toBeNull() + }) + + it('shows an unpressed Answer chip in Search mode and flips the shared flag on click', () => { + useMothershipModeStore.getState().setMode('search') + mount() + + const chip = button() + expect(chip?.textContent).toBe('Answer') + expect(chip?.getAttribute('aria-pressed')).toBe('false') + + act(() => { + chip?.dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) + + expect(useMothershipModeStore.getState().answer).toBe(true) + expect(button()?.getAttribute('aria-pressed')).toBe('true') + expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_answer_toggled', { + workspace_id: 'workspace-1', + enabled: true, + }) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx new file mode 100644 index 00000000000..c7b410ca5bc --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx @@ -0,0 +1,44 @@ +'use client' + +import { memo } from 'react' +import { Chip, Tooltip } from '@sim/emcn' +import { useParams } from 'next/navigation' +import { usePostHog } from 'posthog-js/react' +import { captureEvent } from '@/lib/posthog/client' +import { useMothershipModeStore } from '@/stores/mothership-mode/store' + +/** + * Search mode's Answer toggle: off, a query lists the matching documents; on, + * Sim answers the question from those sources and may use the person's + * connected tools. A label-only round `Chip` in its selected state while on, + * sitting beside the mode switcher in the toolbar's row of round controls. + */ +export const AnswerToggle = memo(function AnswerToggle() { + const { workspaceId } = useParams<{ workspaceId: string }>() + const posthog = usePostHog() + const mode = useMothershipModeStore((state) => state.mode) + const answer = useMothershipModeStore((state) => state.answer) + const setAnswer = useMothershipModeStore((state) => state.setAnswer) + + if (mode !== 'search') return null + + const handleToggle = () => { + setAnswer(!answer) + captureEvent(posthog, 'chat_answer_toggled', { workspace_id: workspaceId, enabled: !answer }) + } + + return ( + + + + Answer + + + + {answer + ? 'Sim answers from your sources and can use your tools' + : 'List matching documents'} + + + ) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts new file mode 100644 index 00000000000..d537887c2cb --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts @@ -0,0 +1 @@ +export { AnswerToggle } from './answer-toggle' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts index 7d8bdca03af..bc937479e16 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts @@ -1,4 +1,5 @@ export { AnimatedPlaceholderEffect } from './animated-placeholder-effect' +export { AnswerToggle } from './answer-toggle' export { AttachedFilesList } from './attached-files-list' export type { ParsedChipLink, PortableKind } from './chip-clipboard-codec' export { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx index 266a65ed205..321a359a2bb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx @@ -78,15 +78,14 @@ describe('ModeSwitcher', () => { expect(button.querySelector('svg')).toBeNull() }) - it('lists every mode and checks the active one', () => { + it('lists both modes and checks the active one', () => { mount() openMenu() const rows = items() - expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Ask', 'Search']) + expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Search']) expect(rows[0].querySelector('svg')).not.toBeNull() expect(rows[1].querySelector('svg')).toBeNull() - expect(rows[2].querySelector('svg')).toBeNull() }) it('switches the shared mode and reports the change', () => { @@ -94,7 +93,7 @@ describe('ModeSwitcher', () => { openMenu() act(() => { - items()[2].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + items()[1].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) }) expect(useMothershipModeStore.getState().mode).toBe('search') diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx index 632aabfc16f..f3e1f380ad4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -28,12 +28,11 @@ import { const MODE_LABELS: Record = { build: 'Build', - ask: 'Ask', search: 'Search', } /** - * The composer's Build / Ask / Search switcher: a label-only `Chip` in its `round` + * The composer's Build / Search switcher: a label-only `Chip` in its `round` * shape — chip chrome throughout (`--text-body` label, `--surface-hover` on * hover, no text-color shift), fully round to sit in the toolbar's row of * round controls — opening a menu that checks the active mode, as diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 9cf844cf597..76d11bc308a 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -21,6 +21,7 @@ import { MOTHERSHIP_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { useChatSurface } from '@/app/workspace/[workspaceId]/home/components/chat-surface-context' import { AnimatedPlaceholderEffect, + AnswerToggle, AttachedFilesList, DropOverlay, MicButton, @@ -719,6 +720,7 @@ const UserInputImpl = forwardRef(function UserI
+ {canSearch && } {canSearch && } {isSttSupported && ( state.mode) + const answerMode = useMothershipModeStore((state) => state.answer) /** The bases an Ask turn is grounded in; read through a ref so a list refresh never rebuilds the submit handler. */ const { data: knowledgeBases = EMPTY_KNOWLEDGE_BASES } = useKnowledgeBasesQuery(workspaceId) const knowledgeBasesRef = useRef(knowledgeBases) @@ -487,11 +488,13 @@ export function Home({ chatId, userName, userId }: HomeProps) { }) /** - * Search mode answers with documents, not a turn of the agent, and only + * Search without Answer lists documents, not a turn of the agent, and only * a query can be answered: attachments alone have nothing to search for. + * With Answer on, the query is a turn of the agent grounded in the sources. */ - const mode = useMothershipModeStore.getState().mode - if (mode === 'search') { + const { mode, answer } = useMothershipModeStore.getState() + const answering = mode === 'search' && answer + if (mode === 'search' && !answer) { if (trimmed) setSearchQuery(trimmed) return } @@ -501,18 +504,17 @@ export function Home({ chatId, userName, userId }: HomeProps) { } prepareResourceViewForAgentTurn() - const turnContexts = - mode === 'ask' - ? withSearchedKnowledgeContexts( - contexts, - searchedKnowledgeBases(knowledgeBasesRef.current, workspaceId) - ) - : contexts + const turnContexts = answering + ? withSearchedKnowledgeContexts( + contexts, + searchedKnowledgeBases(knowledgeBasesRef.current, workspaceId) + ) + : contexts sendMessage( trimmed || 'Analyze the attached file(s).', fileAttachments, turnContexts, - mode === 'ask' ? { requestMode: 'ask' } : undefined + answering ? { requestMode: 'ask' } : undefined ) }, [workspaceId, chatId, prepareResourceViewForAgentTurn, sendMessage, setSearchQuery] @@ -521,20 +523,23 @@ export function Home({ chatId, userName, userId }: HomeProps) { /** An emptied search box returns to the sources; nothing else reads the cleared query. */ const clearSearch = useCallback(() => setSearchQuery(''), [setSearchQuery]) - /** Summarize or Answer on a result: hand the question to the agent in Ask mode. */ + /** Summarize or Answer on a result: turn Answer on and hand the question to the agent. */ const handleSummarize = (prompt: string) => { - useMothershipModeStore.getState().setMode('ask') + const store = useMothershipModeStore.getState() + store.setMode('search') + store.setAnswer(true) setSearchQuery('') handleSubmit(prompt) } /** - * A chat that already exists never opens in Search: its transcript is a - * conversation, and search results never join it. Build and Ask both carry - * over, so a follow-up question stays grounded in the sources. + * A chat that already exists never opens in document-listing Search: its + * transcript is a conversation, and search results never join it. Build and + * Search with Answer both carry over, so a follow-up stays grounded in the + * sources. */ useEffect(() => { const store = useMothershipModeStore.getState() - if (chatId && store.mode === 'search') store.setMode('build') + if (chatId && store.mode === 'search' && !store.answer) store.setMode('build') }, [chatId]) const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 const searchResults = showSearchResults ? ( @@ -767,7 +772,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { draftScopeKey={draftScopeKey} onSubmit={handleSubmit} canSearch - clearOnSubmit={composerMode !== 'search'} + clearOnSubmit={composerMode !== 'search' || answerMode} onCleared={clearSearch} isSending={isSending} onStopGeneration={handleStopGeneration} @@ -796,7 +801,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { isLoading={showChatSkeleton} onSubmit={handleSubmit} canSearch - clearOnSubmit={composerMode !== 'search'} + clearOnSubmit={composerMode !== 'search' || answerMode} onCleared={clearSearch} onStopGeneration={handleStopGeneration} messageQueue={messageQueue} diff --git a/apps/sim/lib/copilot/chat/ask-mode.ts b/apps/sim/lib/copilot/chat/ask-mode.ts index d21d3cb6b61..d55a97f4b1d 100644 --- a/apps/sim/lib/copilot/chat/ask-mode.ts +++ b/apps/sim/lib/copilot/chat/ask-mode.ts @@ -18,10 +18,10 @@ export const ASK_MODE_AGENT_CONTEXT: AskModeAgentContext = { type: 'skill', tag: '@Ask', content: [ - 'The person chose Ask: they want an answer drawn from their connected sources, not an action.', + 'The person asked Search to answer: they want an answer drawn from their connected sources, not an action.', '', '- Answer from the knowledge bases attached to this message first. Search them with the knowledge tool `query` operation, and search again with other phrasings when the first pass returns little. Do not read a base or its metadata first; search.', - "- Reach for a connected integration only when the indexed sources cannot answer: live or very recent data (today's inbox, a calendar), or an action the person asked for outright. Say which service you used. Never build, run, or schedule anything on an Ask turn.", + "- Reach for a connected integration only when the indexed sources cannot answer: live or very recent data (today's inbox, a calendar), or an action the person asked for outright. Say which service you used. Never build, run, or schedule anything on an answer turn.", '- Cite every claim with a `` tag exactly as the knowledge tool describes. When nothing relevant is found, say so plainly instead of guessing.', '- Keep the answer short: lead with the answer, then the supporting points.', '- Suggested follow-ups, when you offer them, are questions the attached sources can answer. Never suggest building, running, or automating anything.', diff --git a/apps/sim/lib/posthog/events.ts b/apps/sim/lib/posthog/events.ts index cbdec60a34f..3a01b26634b 100644 --- a/apps/sim/lib/posthog/events.ts +++ b/apps/sim/lib/posthog/events.ts @@ -617,7 +617,13 @@ export interface PostHogEventMap { /** The chat composer's mode switcher picked a different mode. */ chat_mode_changed: { workspace_id: string - mode: 'build' | 'ask' | 'search' + mode: 'build' | 'search' + } + + /** Search mode's Answer toggle was flipped. */ + chat_answer_toggled: { + workspace_id: string + enabled: boolean } /** diff --git a/apps/sim/stores/mothership-mode/store.ts b/apps/sim/stores/mothership-mode/store.ts index 708c1086420..a3288747b58 100644 --- a/apps/sim/stores/mothership-mode/store.ts +++ b/apps/sim/stores/mothership-mode/store.ts @@ -1,23 +1,26 @@ import { create } from 'zustand' import { devtools } from 'zustand/middleware' -export const MOTHERSHIP_MODES = ['build', 'ask', 'search'] as const +export const MOTHERSHIP_MODES = ['build', 'search'] as const export type MothershipMode = (typeof MOTHERSHIP_MODES)[number] interface MothershipModeState { mode: MothershipMode + /** Search mode's Answer toggle: Sim answers from the sources instead of listing them. */ + answer: boolean setMode: (mode: MothershipMode) => void + setAnswer: (answer: boolean) => void reset: () => void } -const initialState: Pick = { mode: 'build' } +const initialState: Pick = { mode: 'build', answer: false } /** - * The chat composer's mode — Build (default), Ask, or Search — read by the - * input's mode switcher and by the suggested actions beneath the input. Ask is - * a turn of the agent grounded in the searched sources; Search answers with - * documents and no turn at all. + * The chat composer's mode — Build (default) or Search — and Search's Answer + * toggle, read by the input's controls and by the suggested actions beneath + * the input. Search lists the matching documents with no turn at all; with + * Answer on, a query is a turn of the agent grounded in the searched sources. * * A store rather than `Home` state because `Home` remounts per chat * (`key={chatId}`) and the new-chat → `/chat/[chatId]` handoff must carry the @@ -33,6 +36,7 @@ export const useMothershipModeStore = create()( (set) => ({ ...initialState, setMode: (mode) => set({ mode }), + setAnswer: (answer) => set({ answer }), reset: () => set(initialState), }), { name: 'mothership-mode-store' } From 09775cee345841027f5a817700474f4bca1b5b32 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 12:58:51 -0700 Subject: [PATCH 67/76] fix(credentials): prove a Chat managed-credential use only from a real tool call, and list or mint only live group bindings --- .../executor/utils/credential-token.test.ts | 24 ++++++++----- apps/sim/executor/utils/credential-token.ts | 7 +++- .../application/authorization.test.ts | 34 +++++++++++++++++++ .../application/authorization.ts | 22 +++++++++--- apps/sim/lib/credentials/environment.ts | 32 ++++++++++++----- 5 files changed, 96 insertions(+), 23 deletions(-) diff --git a/apps/sim/executor/utils/credential-token.test.ts b/apps/sim/executor/utils/credential-token.test.ts index 8c5b867fc26..d23fe290c26 100644 --- a/apps/sim/executor/utils/credential-token.test.ts +++ b/apps/sim/executor/utils/credential-token.test.ts @@ -128,16 +128,22 @@ describe('resolveExecutorCredentialToken', () => { }) it('leaves managed credentials unproven for a context that is not a trusted Chat call', async () => { - await resolveExecutorCredentialToken({ - requestId: 'req-1', - credentialId: 'cred-1', - userId: 'user-1', - copilotExecutionContext: { userId: 'user-1', workspaceId: 'ws-1' }, - }) + for (const copilotExecutionContext of [ + { userId: 'user-1', workspaceId: 'ws-1' }, + { userId: 'user-1', workspaceId: 'ws-1', copilotToolExecution: true as const }, + ]) { + mockResolveCredentialAccessToken.mockClear() + await resolveExecutorCredentialToken({ + requestId: 'req-1', + credentialId: 'cred-1', + userId: 'user-1', + copilotExecutionContext, + }) - expect( - mockResolveCredentialAccessToken.mock.calls[0][0].resolveManagedPrincipal - ).toBeUndefined() + expect( + mockResolveCredentialAccessToken.mock.calls[0][0].resolveManagedPrincipal + ).toBeUndefined() + } }) it('fails before dispatch when the origin lacks current workflow authority', async () => { diff --git a/apps/sim/executor/utils/credential-token.ts b/apps/sim/executor/utils/credential-token.ts index 70fc8a9a69d..f374f58c79f 100644 --- a/apps/sim/executor/utils/credential-token.ts +++ b/apps/sim/executor/utils/credential-token.ts @@ -57,10 +57,15 @@ export async function resolveExecutorCredentialToken( throw new Error('Managed credential delegation is missing current workflow authority') } + /** + * A Chat proof needs the per-call id the delegation is minted under; a + * context that lacks it is not a Chat tool call and leaves managed + * credentials unproven, so the resolver answers with its own refusal. + */ const resolveManagedPrincipal = executorDelegationOrigin ? (managedCredentialId: string) => bindExecutorManagedOAuthDelegation(executorDelegationOrigin, managedCredentialId) - : copilotExecutionContext?.copilotToolExecution + : copilotExecutionContext?.copilotToolExecution && copilotExecutionContext.toolCallId ? async (managedCredentialId: string) => createCopilotManagedOAuthPrincipal(copilotExecutionContext, managedCredentialId) : undefined diff --git a/apps/sim/lib/credential-groups/application/authorization.test.ts b/apps/sim/lib/credential-groups/application/authorization.test.ts index cc9a0833452..e56191b9add 100644 --- a/apps/sim/lib/credential-groups/application/authorization.test.ts +++ b/apps/sim/lib/credential-groups/application/authorization.test.ts @@ -9,11 +9,23 @@ import { credentialOperations } from '@/lib/credentials/application/operations' const mocks = vi.hoisted(() => ({ loadEnrollmentAccess: vi.fn(), + loadBinding: vi.fn(), requirePolicy: vi.fn(), })) vi.mock('@/lib/credential-groups/credentials', () => ({ loadCredentialGroupEnrollmentAccessForSubject: mocks.loadEnrollmentAccess, + loadManagedCredentialGroupBinding: mocks.loadBinding, + isManagedCredentialGroupBindingLive: (binding: { + managedOauthStatus: string + enrollmentStatus: string + groupStatus: string + optionStatus: string | null + }) => + binding.managedOauthStatus === 'active' && + ['in_progress', 'completed'].includes(binding.enrollmentStatus) && + binding.groupStatus === 'active' && + binding.optionStatus === 'active', })) vi.mock('@/lib/resource-policies/repository', () => ({ @@ -29,10 +41,23 @@ const context = { workspaceId: 'workspace-1', workspaceOrganizationId: null, allowPersonalApiKeys: true, + credentialId: 'credential-1', credentialGroupId: 'group-1', credentialGroupEnrollmentId: 'enrollment-1', } +const liveBinding = { + credentialId: 'credential-1', + workspaceId: 'workspace-1', + providerId: 'google-email', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', +} + function storedPolicy(allowedWorkflowIds: string[] = []) { return { id: 'policy-1', @@ -112,6 +137,15 @@ describe('requireCredentialGroupCredentialAccess', () => { enrollmentId: 'enrollment-1', email: 'person@example.com', }) + mocks.loadBinding.mockResolvedValue(liveBinding) + }) + + it('denies a Chat turn once the credential group or its option is disabled', async () => { + mocks.loadBinding.mockResolvedValue({ ...liveBinding, optionStatus: 'disabled' }) + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + + mocks.loadBinding.mockResolvedValue({ ...liveBinding, groupStatus: 'disabled' }) + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) }) it("allows a Chat turn to use only the credential under the signed-in user's own enrollment", async () => { diff --git a/apps/sim/lib/credential-groups/application/authorization.ts b/apps/sim/lib/credential-groups/application/authorization.ts index f50ddc066f7..ea25d0916cf 100644 --- a/apps/sim/lib/credential-groups/application/authorization.ts +++ b/apps/sim/lib/credential-groups/application/authorization.ts @@ -16,7 +16,11 @@ import { evaluateCredentialGroupWorkflowAccess, } from '@/lib/credential-groups/application/workflow-access-policy' import type { CredentialGroupCredentialListContext } from '@/lib/credential-groups/credentials' -import { loadCredentialGroupEnrollmentAccessForSubject } from '@/lib/credential-groups/credentials' +import { + isManagedCredentialGroupBindingLive, + loadCredentialGroupEnrollmentAccessForSubject, + loadManagedCredentialGroupBinding, +} from '@/lib/credential-groups/credentials' import type { ResourcePolicyBindingFor } from '@/lib/resource-policies/registry' import { requireResourcePolicy } from '@/lib/resource-policies/repository' @@ -93,14 +97,17 @@ export function requireCredentialGroupWorkflowActor(principal: Principal): Princ */ async function requireCredentialGroupActorCredentialAccess( principal: Extract, - context: CredentialGroupAuthorizationContext & { credentialGroupEnrollmentId: string }, + context: CredentialGroupAuthorizationContext & { + credentialId: string + credentialGroupEnrollmentId: string + }, resourcePolicy: ResourcePolicyBindingFor<'credential_group'> ): Promise { const subject = resolvePrincipalSubject(principal) if (subject?.kind !== 'sim_user' || !subject.userId) { throw new OrchestrationError('forbidden', 'Credential Group actor access required') } - const [policy, actorAccess] = await Promise.all([ + const [policy, actorAccess, binding] = await Promise.all([ requireResourcePolicy({ workspaceId: context.workspaceId, resourceType: 'credential_group', @@ -108,8 +115,10 @@ async function requireCredentialGroupActorCredentialAccess( codec: credentialGroupWorkflowAccessPolicyCodec, }), loadCredentialGroupEnrollmentAccessForSubject(context.credentialGroupId, subject), + loadManagedCredentialGroupBinding(context.credentialId), ]) - if (!actorAccess) { + /** A disabled group or option denies here, as it does for every other consumer of a binding. */ + if (!actorAccess || !binding || !isManagedCredentialGroupBindingLive(binding)) { throw new OrchestrationError('forbidden', 'Credential Group credential access denied') } const decision = evaluateCredentialGroupActorCredentialAccess({ @@ -126,7 +135,10 @@ async function requireCredentialGroupActorCredentialAccess( export async function requireCredentialGroupCredentialAccess( principal: Principal, - context: CredentialGroupAuthorizationContext & { credentialGroupEnrollmentId: string }, + context: CredentialGroupAuthorizationContext & { + credentialId: string + credentialGroupEnrollmentId: string + }, resourcePolicy: ResourcePolicyBindingFor<'credential_group'> ): Promise { if (principal.kind === 'delegated' && principal.serviceId === 'copilot') { diff --git a/apps/sim/lib/credentials/environment.ts b/apps/sim/lib/credentials/environment.ts index c1a74677e12..3f6a4815d0e 100644 --- a/apps/sim/lib/credentials/environment.ts +++ b/apps/sim/lib/credentials/environment.ts @@ -14,7 +14,7 @@ import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { and, asc, eq, inArray, isNotNull, isNull, notInArray, or, sql } from 'drizzle-orm' import { acquireUserBillingIdentityLock } from '@/lib/billing/organizations/billing-identity-lock' -import { LIVE_ENROLLMENT_STATUSES } from '@/lib/credential-groups/credentials' +import { isManagedCredentialGroupBindingLive } from '@/lib/credential-groups/credentials' import type { DbOrTx } from '@/lib/db/types' import { getEffectiveWorkspacePermission, @@ -843,10 +843,11 @@ export interface AccessibleOAuthCredential { /** * The Credential Group credentials a verified person holds through their own - * live enrollments in the workspace: active managed OAuth rows whose enrollment - * email is the person's. These are theirs to use as themselves; the policy's - * actor statement is what a use is authorized against, so nothing here widens - * access, it only tells the person (and the agent acting for them) what exists. + * enrollments in the workspace and may use right now: the credential, its + * enrollment, its option, and its group are all live, the same bar every mint + * applies. These are theirs to use as themselves; the policy's actor statement + * is what a use is authorized against, so nothing here widens access, it only + * tells the person (and the agent acting for them) what exists. */ export async function getEnrolledManagedOAuthCredentials( workspaceId: string, @@ -857,7 +858,12 @@ export async function getEnrolledManagedOAuthCredentials( id: credential.id, providerId: credential.providerId, displayName: credential.displayName, + credentialGroupOptionId: credential.credentialGroupOptionId, + managedOauthStatus: credential.managedOauthStatus, + enrollmentStatus: credentialGroupEnrollment.status, groupName: credentialGroup.name, + groupStatus: credentialGroup.status, + groupOptions: credentialGroup.options, updatedAt: credential.updatedAt, }) .from(credential) @@ -871,15 +877,25 @@ export async function getEnrolledManagedOAuthCredentials( and( eq(credential.workspaceId, workspaceId), eq(credential.type, 'managed_oauth'), - eq(credential.managedOauthStatus, 'active'), - inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]), eq(user.id, userId), eq(user.emailVerified, true) ) ) return rows - .filter((row): row is typeof row & { providerId: string } => Boolean(row.providerId)) + .filter( + (row): row is typeof row & { providerId: string } => + Boolean(row.providerId) && + row.managedOauthStatus !== null && + isManagedCredentialGroupBindingLive({ + managedOauthStatus: row.managedOauthStatus, + enrollmentStatus: row.enrollmentStatus, + groupStatus: row.groupStatus, + optionStatus: + row.groupOptions.find((option) => option.id === row.credentialGroupOptionId)?.status ?? + null, + }) + ) .map((row) => ({ id: row.id, providerId: row.providerId, From 47408998f0199f1cd8d2a42052d11510ce0ac654 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:03:41 -0700 Subject: [PATCH 68/76] improvement(home): name both choices in Sources mode with a Search / Assistant toggle --- .../answer-toggle/answer-toggle.test.tsx | 71 ---------------- .../answer-toggle/answer-toggle.tsx | 44 ---------- .../components/answer-toggle/index.ts | 1 - .../components/user-input/components/index.ts | 2 +- .../mode-switcher/mode-switcher.test.tsx | 6 +- .../mode-switcher/mode-switcher.tsx | 6 +- .../components/sources-mode-toggle/index.ts | 1 + .../sources-mode-toggle.test.tsx | 84 +++++++++++++++++++ .../sources-mode-toggle.tsx | 68 +++++++++++++++ .../home/components/user-input/user-input.tsx | 4 +- .../app/workspace/[workspaceId]/home/home.tsx | 32 +++---- apps/sim/lib/copilot/chat/ask-mode.ts | 4 +- apps/sim/lib/posthog/events.ts | 6 +- apps/sim/stores/mothership-mode/store.ts | 22 +++-- 14 files changed, 196 insertions(+), 155 deletions(-) delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx deleted file mode 100644 index 5c5e16e3e0f..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.test.tsx +++ /dev/null @@ -1,71 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { act } from 'react' -import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockCaptureEvent } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn() })) - -vi.mock('next/navigation', () => ({ - useParams: () => ({ workspaceId: 'workspace-1' }), -})) -vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) -vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) - -import { AnswerToggle } from '@/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' - -let root: Root | null = null -let container: HTMLDivElement | null = null - -function mount() { - ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true - container = document.createElement('div') - document.body.appendChild(container) - root = createRoot(container) - act(() => root?.render()) -} - -function button(): HTMLButtonElement | null { - return container?.querySelector('button') ?? null -} - -beforeEach(() => { - mockCaptureEvent.mockClear() - useMothershipModeStore.getState().reset() -}) - -afterEach(() => { - if (root) act(() => root?.unmount()) - container?.remove() - root = null - container = null -}) - -describe('AnswerToggle', () => { - it('renders nothing outside Search mode', () => { - mount() - expect(button()).toBeNull() - }) - - it('shows an unpressed Answer chip in Search mode and flips the shared flag on click', () => { - useMothershipModeStore.getState().setMode('search') - mount() - - const chip = button() - expect(chip?.textContent).toBe('Answer') - expect(chip?.getAttribute('aria-pressed')).toBe('false') - - act(() => { - chip?.dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) - }) - - expect(useMothershipModeStore.getState().answer).toBe(true) - expect(button()?.getAttribute('aria-pressed')).toBe('true') - expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_answer_toggled', { - workspace_id: 'workspace-1', - enabled: true, - }) - }) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx deleted file mode 100644 index c7b410ca5bc..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/answer-toggle.tsx +++ /dev/null @@ -1,44 +0,0 @@ -'use client' - -import { memo } from 'react' -import { Chip, Tooltip } from '@sim/emcn' -import { useParams } from 'next/navigation' -import { usePostHog } from 'posthog-js/react' -import { captureEvent } from '@/lib/posthog/client' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' - -/** - * Search mode's Answer toggle: off, a query lists the matching documents; on, - * Sim answers the question from those sources and may use the person's - * connected tools. A label-only round `Chip` in its selected state while on, - * sitting beside the mode switcher in the toolbar's row of round controls. - */ -export const AnswerToggle = memo(function AnswerToggle() { - const { workspaceId } = useParams<{ workspaceId: string }>() - const posthog = usePostHog() - const mode = useMothershipModeStore((state) => state.mode) - const answer = useMothershipModeStore((state) => state.answer) - const setAnswer = useMothershipModeStore((state) => state.setAnswer) - - if (mode !== 'search') return null - - const handleToggle = () => { - setAnswer(!answer) - captureEvent(posthog, 'chat_answer_toggled', { workspace_id: workspaceId, enabled: !answer }) - } - - return ( - - - - Answer - - - - {answer - ? 'Sim answers from your sources and can use your tools' - : 'List matching documents'} - - - ) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts deleted file mode 100644 index d537887c2cb..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/answer-toggle/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { AnswerToggle } from './answer-toggle' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts index bc937479e16..493b95dfeb3 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts @@ -1,5 +1,4 @@ export { AnimatedPlaceholderEffect } from './animated-placeholder-effect' -export { AnswerToggle } from './answer-toggle' export { AttachedFilesList } from './attached-files-list' export type { ParsedChipLink, PortableKind } from './chip-clipboard-codec' export { @@ -37,3 +36,4 @@ export { PromptEditor, usePromptEditor } from './prompt-editor' export { SendButton } from './send-button' export type { SkillsMenuHandle } from './skills-menu-dropdown/skills-menu-dropdown' export { SkillsMenuDropdown } from './skills-menu-dropdown/skills-menu-dropdown' +export { SourcesModeToggle } from './sources-mode-toggle' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx index 321a359a2bb..377d819bb8d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx @@ -83,7 +83,7 @@ describe('ModeSwitcher', () => { openMenu() const rows = items() - expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Search']) + expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Sources']) expect(rows[0].querySelector('svg')).not.toBeNull() expect(rows[1].querySelector('svg')).toBeNull() }) @@ -97,7 +97,7 @@ describe('ModeSwitcher', () => { }) expect(useMothershipModeStore.getState().mode).toBe('search') - expect(trigger().textContent).toBe('Search') + expect(trigger().textContent).toBe('Sources') expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_mode_changed', { workspace_id: 'workspace-1', mode: 'search', @@ -105,7 +105,7 @@ describe('ModeSwitcher', () => { expect(mockSetSearchQuery).not.toHaveBeenCalled() }) - it('drops the search query from the URL when leaving Search', () => { + it('drops the search query from the URL when leaving Sources', () => { useMothershipModeStore.getState().setMode('search') mount() openMenu() diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx index f3e1f380ad4..f80587fd07e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -28,11 +28,11 @@ import { const MODE_LABELS: Record = { build: 'Build', - search: 'Search', + search: 'Sources', } /** - * The composer's Build / Search switcher: a label-only `Chip` in its `round` + * The composer's Build / Sources switcher: a label-only `Chip` in its `round` * shape — chip chrome throughout (`--text-body` label, `--surface-hover` on * hover, no text-color shift), fully round to sit in the toolbar's row of * round controls — opening a menu that checks the active mode, as @@ -47,7 +47,7 @@ export const ModeSwitcher = memo(function ModeSwitcher() { const [, setSearchQueryParam] = useQueryState(searchQueryParam.key, searchQueryParam.parser) const [, setSearchFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) - /** Leaving Search drops the query from the URL, so a clean URL always means no search is showing. */ + /** Leaving Sources drops the query from the URL, so a clean URL always means no search is showing. */ const handleSelect = (next: MothershipMode) => { if (next === mode) return setMode(next) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts new file mode 100644 index 00000000000..286be05a2ac --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts @@ -0,0 +1 @@ +export { SourcesModeToggle } from './sources-mode-toggle' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx new file mode 100644 index 00000000000..e785adaa350 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx @@ -0,0 +1,84 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCaptureEvent } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn() })) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) +vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) + +import { SourcesModeToggle } from '@/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle' +import { useMothershipModeStore } from '@/stores/mothership-mode/store' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function radios(): HTMLButtonElement[] { + return Array.from(container?.querySelectorAll('[role="radio"]') ?? []) +} + +function click(radio: HTMLButtonElement) { + act(() => { + radio.dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) +} + +beforeEach(() => { + mockCaptureEvent.mockClear() + useMothershipModeStore.getState().reset() +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('SourcesModeToggle', () => { + it('renders nothing outside Sources mode', () => { + mount() + expect(radios()).toHaveLength(0) + }) + + it('names both choices with Search selected by default, and switches to Assistant on click', () => { + useMothershipModeStore.getState().setMode('search') + mount() + + expect(radios().map((radio) => radio.textContent)).toEqual(['Search', 'Assistant']) + expect(radios().map((radio) => radio.getAttribute('aria-checked'))).toEqual(['true', 'false']) + + click(radios()[1]) + + expect(useMothershipModeStore.getState().assistant).toBe(true) + expect(radios().map((radio) => radio.getAttribute('aria-checked'))).toEqual(['false', 'true']) + expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_sources_mode_changed', { + workspace_id: 'workspace-1', + mode: 'assistant', + }) + }) + + it('does not report re-selecting the current choice', () => { + useMothershipModeStore.getState().setMode('search') + mount() + + click(radios()[0]) + + expect(useMothershipModeStore.getState().assistant).toBe(false) + expect(mockCaptureEvent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx new file mode 100644 index 00000000000..d3d04e9757d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx @@ -0,0 +1,68 @@ +'use client' + +import { memo } from 'react' +import { Chip, Tooltip } from '@sim/emcn' +import { useParams } from 'next/navigation' +import { usePostHog } from 'posthog-js/react' +import { captureEvent } from '@/lib/posthog/client' +import { useMothershipModeStore } from '@/stores/mothership-mode/store' + +const OPTIONS = [ + { + assistant: false, + label: 'Search', + hint: 'Enterprise search: list the documents that match, from every source you can read', + }, + { + assistant: true, + label: 'Assistant', + hint: 'Answer in natural language from your sources, citing them, using your tools when needed', + }, +] as const + +/** + * Sources mode's two ways to use the sources, as a pair of round chips that + * act as one radio group: Search lists the matching documents; Assistant + * answers the question in natural language from them. The selected chip is + * the one in its selected state, so both choices are always named and the + * current one is never in doubt. + */ +export const SourcesModeToggle = memo(function SourcesModeToggle() { + const { workspaceId } = useParams<{ workspaceId: string }>() + const posthog = usePostHog() + const mode = useMothershipModeStore((state) => state.mode) + const assistant = useMothershipModeStore((state) => state.assistant) + const setAssistant = useMothershipModeStore((state) => state.setAssistant) + + if (mode !== 'search') return null + + const select = (next: boolean) => { + if (next === assistant) return + setAssistant(next) + captureEvent(posthog, 'chat_sources_mode_changed', { + workspace_id: workspaceId, + mode: next ? 'assistant' : 'search', + }) + } + + return ( +
+ {OPTIONS.map((option) => ( + + + select(option.assistant)} + > + {option.label} + + + {option.hint} + + ))} +
+ ) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 76d11bc308a..37adb0737c5 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -21,7 +21,6 @@ import { MOTHERSHIP_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { useChatSurface } from '@/app/workspace/[workspaceId]/home/components/chat-surface-context' import { AnimatedPlaceholderEffect, - AnswerToggle, AttachedFilesList, DropOverlay, MicButton, @@ -29,6 +28,7 @@ import { ModeSwitcher, PromptEditor, SendButton, + SourcesModeToggle, usePromptEditor, } from '@/app/workspace/[workspaceId]/home/components/user-input/components' import { handleMothershipAddContextEvent } from '@/app/workspace/[workspaceId]/home/components/user-input/mothership-context-event' @@ -720,7 +720,7 @@ const UserInputImpl = forwardRef(function UserI
- {canSearch && } + {canSearch && } {canSearch && } {isSttSupported && ( state.mode) - const answerMode = useMothershipModeStore((state) => state.answer) + const assistantMode = useMothershipModeStore((state) => state.assistant) /** The bases an Ask turn is grounded in; read through a ref so a list refresh never rebuilds the submit handler. */ const { data: knowledgeBases = EMPTY_KNOWLEDGE_BASES } = useKnowledgeBasesQuery(workspaceId) const knowledgeBasesRef = useRef(knowledgeBases) @@ -488,13 +488,13 @@ export function Home({ chatId, userName, userId }: HomeProps) { }) /** - * Search without Answer lists documents, not a turn of the agent, and only - * a query can be answered: attachments alone have nothing to search for. - * With Answer on, the query is a turn of the agent grounded in the sources. + * Sources mode's Search lists documents, not a turn of the agent, and only + * a query can be searched: attachments alone have nothing to search for. + * Its Assistant makes the query a turn of the agent grounded in the sources. */ - const { mode, answer } = useMothershipModeStore.getState() - const answering = mode === 'search' && answer - if (mode === 'search' && !answer) { + const { mode, assistant } = useMothershipModeStore.getState() + const answering = mode === 'search' && assistant + if (mode === 'search' && !assistant) { if (trimmed) setSearchQuery(trimmed) return } @@ -523,23 +523,23 @@ export function Home({ chatId, userName, userId }: HomeProps) { /** An emptied search box returns to the sources; nothing else reads the cleared query. */ const clearSearch = useCallback(() => setSearchQuery(''), [setSearchQuery]) - /** Summarize or Answer on a result: turn Answer on and hand the question to the agent. */ + /** Summarize or Answer on a result: switch to Assistant and hand the question to it. */ const handleSummarize = (prompt: string) => { const store = useMothershipModeStore.getState() store.setMode('search') - store.setAnswer(true) + store.setAssistant(true) setSearchQuery('') handleSubmit(prompt) } /** - * A chat that already exists never opens in document-listing Search: its - * transcript is a conversation, and search results never join it. Build and - * Search with Answer both carry over, so a follow-up stays grounded in the - * sources. + * A chat that already exists never opens in Sources' document-listing + * Search: its transcript is a conversation, and search results never join + * it. Build and the Assistant both carry over, so a follow-up stays grounded + * in the sources. */ useEffect(() => { const store = useMothershipModeStore.getState() - if (chatId && store.mode === 'search' && !store.answer) store.setMode('build') + if (chatId && store.mode === 'search' && !store.assistant) store.setMode('build') }, [chatId]) const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 const searchResults = showSearchResults ? ( @@ -772,7 +772,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { draftScopeKey={draftScopeKey} onSubmit={handleSubmit} canSearch - clearOnSubmit={composerMode !== 'search' || answerMode} + clearOnSubmit={composerMode !== 'search' || assistantMode} onCleared={clearSearch} isSending={isSending} onStopGeneration={handleStopGeneration} @@ -801,7 +801,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { isLoading={showChatSkeleton} onSubmit={handleSubmit} canSearch - clearOnSubmit={composerMode !== 'search' || answerMode} + clearOnSubmit={composerMode !== 'search' || assistantMode} onCleared={clearSearch} onStopGeneration={handleStopGeneration} messageQueue={messageQueue} diff --git a/apps/sim/lib/copilot/chat/ask-mode.ts b/apps/sim/lib/copilot/chat/ask-mode.ts index d55a97f4b1d..02d7b81232b 100644 --- a/apps/sim/lib/copilot/chat/ask-mode.ts +++ b/apps/sim/lib/copilot/chat/ask-mode.ts @@ -18,10 +18,10 @@ export const ASK_MODE_AGENT_CONTEXT: AskModeAgentContext = { type: 'skill', tag: '@Ask', content: [ - 'The person asked Search to answer: they want an answer drawn from their connected sources, not an action.', + 'The person chose the Assistant: they want an answer in natural language drawn from their connected sources, not an action.', '', '- Answer from the knowledge bases attached to this message first. Search them with the knowledge tool `query` operation, and search again with other phrasings when the first pass returns little. Do not read a base or its metadata first; search.', - "- Reach for a connected integration only when the indexed sources cannot answer: live or very recent data (today's inbox, a calendar), or an action the person asked for outright. Say which service you used. Never build, run, or schedule anything on an answer turn.", + "- Reach for a connected integration only when the indexed sources cannot answer: live or very recent data (today's inbox, a calendar), or an action the person asked for outright. Say which service you used. Never build, run, or schedule anything on an Assistant turn.", '- Cite every claim with a `` tag exactly as the knowledge tool describes. When nothing relevant is found, say so plainly instead of guessing.', '- Keep the answer short: lead with the answer, then the supporting points.', '- Suggested follow-ups, when you offer them, are questions the attached sources can answer. Never suggest building, running, or automating anything.', diff --git a/apps/sim/lib/posthog/events.ts b/apps/sim/lib/posthog/events.ts index 3a01b26634b..953c17d467f 100644 --- a/apps/sim/lib/posthog/events.ts +++ b/apps/sim/lib/posthog/events.ts @@ -620,10 +620,10 @@ export interface PostHogEventMap { mode: 'build' | 'search' } - /** Search mode's Answer toggle was flipped. */ - chat_answer_toggled: { + /** Sources mode's Search / Assistant choice changed. */ + chat_sources_mode_changed: { workspace_id: string - enabled: boolean + mode: 'search' | 'assistant' } /** diff --git a/apps/sim/stores/mothership-mode/store.ts b/apps/sim/stores/mothership-mode/store.ts index a3288747b58..c90e70455bd 100644 --- a/apps/sim/stores/mothership-mode/store.ts +++ b/apps/sim/stores/mothership-mode/store.ts @@ -7,20 +7,24 @@ export type MothershipMode = (typeof MOTHERSHIP_MODES)[number] interface MothershipModeState { mode: MothershipMode - /** Search mode's Answer toggle: Sim answers from the sources instead of listing them. */ - answer: boolean + /** Sources mode's Assistant choice: Sim answers from the sources instead of listing them. */ + assistant: boolean setMode: (mode: MothershipMode) => void - setAnswer: (answer: boolean) => void + setAssistant: (assistant: boolean) => void reset: () => void } -const initialState: Pick = { mode: 'build', answer: false } +const initialState: Pick = { + mode: 'build', + assistant: false, +} /** - * The chat composer's mode — Build (default) or Search — and Search's Answer - * toggle, read by the input's controls and by the suggested actions beneath - * the input. Search lists the matching documents with no turn at all; with - * Answer on, a query is a turn of the agent grounded in the searched sources. + * The chat composer's mode — Build (default) or Sources (`search`) — and + * Sources mode's Search / Assistant choice, read by the input's controls and by + * the suggested actions beneath the input. Search lists the matching documents + * with no turn at all; Assistant makes a query a turn of the agent grounded in + * the searched sources. * * A store rather than `Home` state because `Home` remounts per chat * (`key={chatId}`) and the new-chat → `/chat/[chatId]` handoff must carry the @@ -36,7 +40,7 @@ export const useMothershipModeStore = create()( (set) => ({ ...initialState, setMode: (mode) => set({ mode }), - setAnswer: (answer) => set({ answer }), + setAssistant: (assistant) => set({ assistant }), reset: () => set(initialState), }), { name: 'mothership-mode-store' } From d0744a8160db57e47c0a8e03db9e7076814cdd19 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:06:19 -0700 Subject: [PATCH 69/76] improvement(home): make Assistant a peer of Build and Search in the mode switcher --- .../suggested-actions.test.tsx | 10 +++ .../suggested-actions/suggested-actions.tsx | 3 +- .../components/user-input/components/index.ts | 1 - .../mode-switcher/mode-switcher.test.tsx | 9 +- .../mode-switcher/mode-switcher.tsx | 7 +- .../components/sources-mode-toggle/index.ts | 1 - .../sources-mode-toggle.test.tsx | 84 ------------------- .../sources-mode-toggle.tsx | 68 --------------- .../home/components/user-input/user-input.tsx | 2 - .../app/workspace/[workspaceId]/home/home.tsx | 30 +++---- apps/sim/lib/posthog/events.ts | 8 +- apps/sim/stores/mothership-mode/store.ts | 21 ++--- 12 files changed, 42 insertions(+), 202 deletions(-) delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx index b0674f4774b..635fe39bfce 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx @@ -128,4 +128,14 @@ describe('SuggestedActions', () => { expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() expect(rows()).toHaveLength(0) }) + + it('shows the sources in Assistant mode, which answers from them', () => { + mount() + + act(() => useMothershipModeStore.getState().setMode('assistant')) + + expect(heading()).toBe('Sources') + expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() + expect(rows()).toHaveLength(0) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx index 889c988aef7..d2b58ca8666 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx @@ -235,6 +235,7 @@ const INITIAL_ACTIONS: Action[] = [ const HEADINGS: Record = { build: 'Suggested actions', search: 'Sources', + assistant: 'Sources', } interface SuggestedActionsProps { @@ -372,7 +373,7 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { `collapsible-up`/`-down` interpolate height alone, so a margin here would hold its full value through the close and then vanish on unmount, snapping the content below up. */} - {mode === 'search' && workspaceId ? ( + {mode !== 'build' && workspaceId ? (
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts index 493b95dfeb3..7d8bdca03af 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts @@ -36,4 +36,3 @@ export { PromptEditor, usePromptEditor } from './prompt-editor' export { SendButton } from './send-button' export type { SkillsMenuHandle } from './skills-menu-dropdown/skills-menu-dropdown' export { SkillsMenuDropdown } from './skills-menu-dropdown/skills-menu-dropdown' -export { SourcesModeToggle } from './sources-mode-toggle' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx index 377d819bb8d..fafe6bbdb4a 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx @@ -78,14 +78,15 @@ describe('ModeSwitcher', () => { expect(button.querySelector('svg')).toBeNull() }) - it('lists both modes and checks the active one', () => { + it('lists every mode and checks the active one', () => { mount() openMenu() const rows = items() - expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Sources']) + expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Search', 'Assistant']) expect(rows[0].querySelector('svg')).not.toBeNull() expect(rows[1].querySelector('svg')).toBeNull() + expect(rows[2].querySelector('svg')).toBeNull() }) it('switches the shared mode and reports the change', () => { @@ -97,7 +98,7 @@ describe('ModeSwitcher', () => { }) expect(useMothershipModeStore.getState().mode).toBe('search') - expect(trigger().textContent).toBe('Sources') + expect(trigger().textContent).toBe('Search') expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_mode_changed', { workspace_id: 'workspace-1', mode: 'search', @@ -105,7 +106,7 @@ describe('ModeSwitcher', () => { expect(mockSetSearchQuery).not.toHaveBeenCalled() }) - it('drops the search query from the URL when leaving Sources', () => { + it('drops the search query from the URL when leaving Search', () => { useMothershipModeStore.getState().setMode('search') mount() openMenu() diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx index f80587fd07e..0f8388e8298 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -28,11 +28,12 @@ import { const MODE_LABELS: Record = { build: 'Build', - search: 'Sources', + search: 'Search', + assistant: 'Assistant', } /** - * The composer's Build / Sources switcher: a label-only `Chip` in its `round` + * The composer's Build / Search / Assistant switcher: a label-only `Chip` in its `round` * shape — chip chrome throughout (`--text-body` label, `--surface-hover` on * hover, no text-color shift), fully round to sit in the toolbar's row of * round controls — opening a menu that checks the active mode, as @@ -47,7 +48,7 @@ export const ModeSwitcher = memo(function ModeSwitcher() { const [, setSearchQueryParam] = useQueryState(searchQueryParam.key, searchQueryParam.parser) const [, setSearchFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) - /** Leaving Sources drops the query from the URL, so a clean URL always means no search is showing. */ + /** Leaving Search drops the query from the URL, so a clean URL always means no search is showing. */ const handleSelect = (next: MothershipMode) => { if (next === mode) return setMode(next) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts deleted file mode 100644 index 286be05a2ac..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { SourcesModeToggle } from './sources-mode-toggle' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx deleted file mode 100644 index e785adaa350..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.test.tsx +++ /dev/null @@ -1,84 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { act } from 'react' -import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockCaptureEvent } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn() })) - -vi.mock('next/navigation', () => ({ - useParams: () => ({ workspaceId: 'workspace-1' }), -})) -vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) -vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) - -import { SourcesModeToggle } from '@/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' - -let root: Root | null = null -let container: HTMLDivElement | null = null - -function mount() { - ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true - container = document.createElement('div') - document.body.appendChild(container) - root = createRoot(container) - act(() => root?.render()) -} - -function radios(): HTMLButtonElement[] { - return Array.from(container?.querySelectorAll('[role="radio"]') ?? []) -} - -function click(radio: HTMLButtonElement) { - act(() => { - radio.dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) - }) -} - -beforeEach(() => { - mockCaptureEvent.mockClear() - useMothershipModeStore.getState().reset() -}) - -afterEach(() => { - if (root) act(() => root?.unmount()) - container?.remove() - root = null - container = null -}) - -describe('SourcesModeToggle', () => { - it('renders nothing outside Sources mode', () => { - mount() - expect(radios()).toHaveLength(0) - }) - - it('names both choices with Search selected by default, and switches to Assistant on click', () => { - useMothershipModeStore.getState().setMode('search') - mount() - - expect(radios().map((radio) => radio.textContent)).toEqual(['Search', 'Assistant']) - expect(radios().map((radio) => radio.getAttribute('aria-checked'))).toEqual(['true', 'false']) - - click(radios()[1]) - - expect(useMothershipModeStore.getState().assistant).toBe(true) - expect(radios().map((radio) => radio.getAttribute('aria-checked'))).toEqual(['false', 'true']) - expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_sources_mode_changed', { - workspace_id: 'workspace-1', - mode: 'assistant', - }) - }) - - it('does not report re-selecting the current choice', () => { - useMothershipModeStore.getState().setMode('search') - mount() - - click(radios()[0]) - - expect(useMothershipModeStore.getState().assistant).toBe(false) - expect(mockCaptureEvent).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx deleted file mode 100644 index d3d04e9757d..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/sources-mode-toggle/sources-mode-toggle.tsx +++ /dev/null @@ -1,68 +0,0 @@ -'use client' - -import { memo } from 'react' -import { Chip, Tooltip } from '@sim/emcn' -import { useParams } from 'next/navigation' -import { usePostHog } from 'posthog-js/react' -import { captureEvent } from '@/lib/posthog/client' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' - -const OPTIONS = [ - { - assistant: false, - label: 'Search', - hint: 'Enterprise search: list the documents that match, from every source you can read', - }, - { - assistant: true, - label: 'Assistant', - hint: 'Answer in natural language from your sources, citing them, using your tools when needed', - }, -] as const - -/** - * Sources mode's two ways to use the sources, as a pair of round chips that - * act as one radio group: Search lists the matching documents; Assistant - * answers the question in natural language from them. The selected chip is - * the one in its selected state, so both choices are always named and the - * current one is never in doubt. - */ -export const SourcesModeToggle = memo(function SourcesModeToggle() { - const { workspaceId } = useParams<{ workspaceId: string }>() - const posthog = usePostHog() - const mode = useMothershipModeStore((state) => state.mode) - const assistant = useMothershipModeStore((state) => state.assistant) - const setAssistant = useMothershipModeStore((state) => state.setAssistant) - - if (mode !== 'search') return null - - const select = (next: boolean) => { - if (next === assistant) return - setAssistant(next) - captureEvent(posthog, 'chat_sources_mode_changed', { - workspace_id: workspaceId, - mode: next ? 'assistant' : 'search', - }) - } - - return ( -
- {OPTIONS.map((option) => ( - - - select(option.assistant)} - > - {option.label} - - - {option.hint} - - ))} -
- ) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 37adb0737c5..9cf844cf597 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -28,7 +28,6 @@ import { ModeSwitcher, PromptEditor, SendButton, - SourcesModeToggle, usePromptEditor, } from '@/app/workspace/[workspaceId]/home/components/user-input/components' import { handleMothershipAddContextEvent } from '@/app/workspace/[workspaceId]/home/components/user-input/mothership-context-event' @@ -720,7 +719,6 @@ const UserInputImpl = forwardRef(function UserI
- {canSearch && } {canSearch && } {isSttSupported && ( state.mode) - const assistantMode = useMothershipModeStore((state) => state.assistant) /** The bases an Ask turn is grounded in; read through a ref so a list refresh never rebuilds the submit handler. */ const { data: knowledgeBases = EMPTY_KNOWLEDGE_BASES } = useKnowledgeBasesQuery(workspaceId) const knowledgeBasesRef = useRef(knowledgeBases) @@ -488,13 +487,13 @@ export function Home({ chatId, userName, userId }: HomeProps) { }) /** - * Sources mode's Search lists documents, not a turn of the agent, and only - * a query can be searched: attachments alone have nothing to search for. - * Its Assistant makes the query a turn of the agent grounded in the sources. + * Search lists documents, not a turn of the agent, and only a query can + * be searched: attachments alone have nothing to search for. Assistant + * makes the query a turn of the agent grounded in the sources. */ - const { mode, assistant } = useMothershipModeStore.getState() - const answering = mode === 'search' && assistant - if (mode === 'search' && !assistant) { + const mode = useMothershipModeStore.getState().mode + const answering = mode === 'assistant' + if (mode === 'search') { if (trimmed) setSearchQuery(trimmed) return } @@ -525,21 +524,18 @@ export function Home({ chatId, userName, userId }: HomeProps) { /** Summarize or Answer on a result: switch to Assistant and hand the question to it. */ const handleSummarize = (prompt: string) => { - const store = useMothershipModeStore.getState() - store.setMode('search') - store.setAssistant(true) + useMothershipModeStore.getState().setMode('assistant') setSearchQuery('') handleSubmit(prompt) } /** - * A chat that already exists never opens in Sources' document-listing - * Search: its transcript is a conversation, and search results never join - * it. Build and the Assistant both carry over, so a follow-up stays grounded - * in the sources. + * A chat that already exists never opens in Search: its transcript is a + * conversation, and search results never join it. Build and Assistant both + * carry over, so a follow-up stays grounded in the sources. */ useEffect(() => { const store = useMothershipModeStore.getState() - if (chatId && store.mode === 'search' && !store.assistant) store.setMode('build') + if (chatId && store.mode === 'search') store.setMode('build') }, [chatId]) const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 const searchResults = showSearchResults ? ( @@ -772,7 +768,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { draftScopeKey={draftScopeKey} onSubmit={handleSubmit} canSearch - clearOnSubmit={composerMode !== 'search' || assistantMode} + clearOnSubmit={composerMode !== 'search'} onCleared={clearSearch} isSending={isSending} onStopGeneration={handleStopGeneration} @@ -801,7 +797,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { isLoading={showChatSkeleton} onSubmit={handleSubmit} canSearch - clearOnSubmit={composerMode !== 'search' || assistantMode} + clearOnSubmit={composerMode !== 'search'} onCleared={clearSearch} onStopGeneration={handleStopGeneration} messageQueue={messageQueue} diff --git a/apps/sim/lib/posthog/events.ts b/apps/sim/lib/posthog/events.ts index 953c17d467f..982f963fa02 100644 --- a/apps/sim/lib/posthog/events.ts +++ b/apps/sim/lib/posthog/events.ts @@ -617,13 +617,7 @@ export interface PostHogEventMap { /** The chat composer's mode switcher picked a different mode. */ chat_mode_changed: { workspace_id: string - mode: 'build' | 'search' - } - - /** Sources mode's Search / Assistant choice changed. */ - chat_sources_mode_changed: { - workspace_id: string - mode: 'search' | 'assistant' + mode: 'build' | 'search' | 'assistant' } /** diff --git a/apps/sim/stores/mothership-mode/store.ts b/apps/sim/stores/mothership-mode/store.ts index c90e70455bd..b7df5184a6e 100644 --- a/apps/sim/stores/mothership-mode/store.ts +++ b/apps/sim/stores/mothership-mode/store.ts @@ -1,30 +1,24 @@ import { create } from 'zustand' import { devtools } from 'zustand/middleware' -export const MOTHERSHIP_MODES = ['build', 'search'] as const +export const MOTHERSHIP_MODES = ['build', 'search', 'assistant'] as const export type MothershipMode = (typeof MOTHERSHIP_MODES)[number] interface MothershipModeState { mode: MothershipMode - /** Sources mode's Assistant choice: Sim answers from the sources instead of listing them. */ - assistant: boolean setMode: (mode: MothershipMode) => void - setAssistant: (assistant: boolean) => void reset: () => void } -const initialState: Pick = { - mode: 'build', - assistant: false, -} +const initialState: Pick = { mode: 'build' } /** - * The chat composer's mode — Build (default) or Sources (`search`) — and - * Sources mode's Search / Assistant choice, read by the input's controls and by - * the suggested actions beneath the input. Search lists the matching documents - * with no turn at all; Assistant makes a query a turn of the agent grounded in - * the searched sources. + * The chat composer's mode — Build (default), Search, or Assistant — read by + * the input's mode switcher and by the suggested actions beneath the input. + * Build is the agent with everything it can do; Search lists the matching + * documents with no turn at all; Assistant is a turn of the agent grounded in + * the searched sources, answering in natural language. * * A store rather than `Home` state because `Home` remounts per chat * (`key={chatId}`) and the new-chat → `/chat/[chatId]` handoff must carry the @@ -40,7 +34,6 @@ export const useMothershipModeStore = create()( (set) => ({ ...initialState, setMode: (mode) => set({ mode }), - setAssistant: (assistant) => set({ assistant }), reset: () => set(initialState), }), { name: 'mothership-mode-store' } From af2557befeae89720cb9f0ec7a47b93ce580b570 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:14:00 -0700 Subject: [PATCH 70/76] improvement(home): keep the composer mode in the URL and drop the mode store --- .../suggested-actions.test.tsx | 22 ++++-- .../suggested-actions/suggested-actions.tsx | 5 +- .../mode-switcher/mode-switcher.test.tsx | 73 ++++++++++++------- .../mode-switcher/mode-switcher.tsx | 13 ++-- .../app/workspace/[workspaceId]/home/home.tsx | 49 ++++++++----- .../[workspaceId]/home/hooks/chat-url.ts | 9 +++ .../[workspaceId]/home/hooks/index.ts | 1 + .../home/hooks/stream/handle-session-event.ts | 7 +- .../[workspaceId]/home/hooks/use-chat.ts | 3 +- .../home/hooks/use-mothership-mode.ts | 13 ++++ .../[workspaceId]/home/search-params.ts | 18 +++++ apps/sim/stores/mothership-mode/store.ts | 41 ----------- apps/sim/stores/reset-all-stores.ts | 2 - 13 files changed, 147 insertions(+), 109 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/hooks/use-mothership-mode.ts delete mode 100644 apps/sim/stores/mothership-mode/store.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx index 635fe39bfce..a879ee5bc35 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx @@ -5,10 +5,23 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCaptureEvent } = vi.hoisted(() => ({ +const { mockCaptureEvent, modeState } = vi.hoisted(() => ({ mockCaptureEvent: vi.fn(), + /** The URL `mode` param as the nuqs mock serves it; `set` is the live setter once mounted. */ + modeState: { initial: 'build', set: (_next: string) => {} }, })) +vi.mock('nuqs', async () => { + const { useState } = await import('react') + return { + useQueryState: () => { + const [mode, setMode] = useState(modeState.initial) + modeState.set = setMode + return [mode, setMode] + }, + } +}) + vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), })) @@ -74,7 +87,6 @@ vi.mock('@/app/workspace/[workspaceId]/components/connect-oauth-modal', () => ({ })) import { SuggestedActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' let root: Root | null = null let container: HTMLDivElement | null = null @@ -101,7 +113,7 @@ function rows(): HTMLButtonElement[] { beforeEach(() => { onSelectPrompt.mockClear() mockCaptureEvent.mockClear() - useMothershipModeStore.getState().reset() + modeState.initial = 'build' }) afterEach(() => { @@ -122,7 +134,7 @@ describe('SuggestedActions', () => { it('shows every source in Search mode instead of the sampled suggestions', () => { mount() - act(() => useMothershipModeStore.getState().setMode('search')) + act(() => modeState.set('search')) expect(heading()).toBe('Sources') expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() @@ -132,7 +144,7 @@ describe('SuggestedActions', () => { it('shows the sources in Assistant mode, which answers from them', () => { mount() - act(() => useMothershipModeStore.getState().setMode('assistant')) + act(() => modeState.set('assistant')) expect(heading()).toBe('Sources') expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx index d2b58ca8666..76e1ec28c28 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx @@ -21,6 +21,8 @@ import type { OAuthConnectTarget, } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/types' import { weightedSample } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample' +import { useMothershipMode } from '@/app/workspace/[workspaceId]/home/hooks/use-mothership-mode' +import type { MothershipMode } from '@/app/workspace/[workspaceId]/home/search-params' import { BrandIcon } from '@/blocks/brand-icon' import { getAllBlockMeta } from '@/blocks/registry' import type { ModuleTag } from '@/blocks/types' @@ -29,7 +31,6 @@ import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useOAuthConnections } from '@/hooks/queries/oauth/oauth-connections' import { useTablesList } from '@/hooks/queries/tables' import { usePermissionConfig } from '@/hooks/use-permission-config' -import { type MothershipMode, useMothershipModeStore } from '@/stores/mothership-mode/store' /** Lookup integration slug by OAuth service display name (case-insensitive). */ const SLUG_BY_LOWER_NAME: ReadonlyMap = new Map( @@ -245,7 +246,7 @@ interface SuggestedActionsProps { export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { const { workspaceId } = useParams<{ workspaceId: string }>() const posthog = usePostHog() - const mode = useMothershipModeStore((state) => state.mode) + const [mode] = useMothershipMode() const { integrationAvailability } = usePermissionConfig() const { data: credentials = EMPTY_CREDENTIALS } = useWorkspaceCredentials({ diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx index fafe6bbdb4a..92f6aa38b0e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx @@ -5,24 +5,35 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCaptureEvent, mockSetSearchQuery, mockSetSearchFilters } = vi.hoisted(() => ({ - mockCaptureEvent: vi.fn(), - mockSetSearchQuery: vi.fn(), - mockSetSearchFilters: vi.fn(), -})) +const { mockCaptureEvent, mockSetSearchQuery, mockSetSearchFilters, modeState } = vi.hoisted( + () => ({ + mockCaptureEvent: vi.fn(), + mockSetSearchQuery: vi.fn(), + mockSetSearchFilters: vi.fn(), + /** The URL `mode` param as the nuqs mock serves it; `set` is the live setter once mounted. */ + modeState: { initial: 'build', set: (_next: string) => {} }, + }) +) vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), })) -vi.mock('nuqs', () => ({ - useQueryState: () => [null, mockSetSearchQuery], - useQueryStates: () => [{}, mockSetSearchFilters], -})) +vi.mock('nuqs', async () => { + const { useState } = await import('react') + return { + useQueryState: (key: string) => { + const [mode, setMode] = useState(modeState.initial) + if (key !== 'mode') return [null, mockSetSearchQuery] + modeState.set = setMode + return [mode, setMode] + }, + useQueryStates: () => [{}, mockSetSearchFilters], + } +}) vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) import { ModeSwitcher } from '@/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' let root: Root | null = null let container: HTMLDivElement | null = null @@ -52,9 +63,17 @@ function items(): HTMLElement[] { return Array.from(document.querySelectorAll('[role="menuitem"]')) } +function select(index: number) { + act(() => { + items()[index].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) +} + beforeEach(() => { mockCaptureEvent.mockClear() - useMothershipModeStore.getState().reset() + mockSetSearchQuery.mockClear() + mockSetSearchFilters.mockClear() + modeState.initial = 'build' }) afterEach(() => { @@ -89,15 +108,11 @@ describe('ModeSwitcher', () => { expect(rows[2].querySelector('svg')).toBeNull() }) - it('switches the shared mode and reports the change', () => { + it('writes the chosen mode to the URL and reports the change', () => { mount() openMenu() + select(1) - act(() => { - items()[1].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) - }) - - expect(useMothershipModeStore.getState().mode).toBe('search') expect(trigger().textContent).toBe('Search') expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_mode_changed', { workspace_id: 'workspace-1', @@ -106,16 +121,21 @@ describe('ModeSwitcher', () => { expect(mockSetSearchQuery).not.toHaveBeenCalled() }) + it('reads the mode from the URL on mount', () => { + modeState.initial = 'assistant' + mount() + + expect(trigger().textContent).toBe('Assistant') + expect(trigger().getAttribute('aria-label')).toBe('Mode: Assistant') + }) + it('drops the search query from the URL when leaving Search', () => { - useMothershipModeStore.getState().setMode('search') + modeState.initial = 'search' mount() openMenu() + select(0) - act(() => { - items()[0].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) - }) - - expect(useMothershipModeStore.getState().mode).toBe('build') + expect(trigger().textContent).toBe('Build') expect(mockSetSearchQuery).toHaveBeenCalledWith(null, { history: 'replace', scroll: false }) expect(mockSetSearchFilters).toHaveBeenCalledWith( { source: null, updated: null }, @@ -126,12 +146,9 @@ describe('ModeSwitcher', () => { it('does not report re-selecting the active mode', () => { mount() openMenu() + select(0) - act(() => { - items()[0].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) - }) - - expect(useMothershipModeStore.getState().mode).toBe('build') + expect(trigger().textContent).toBe('Build') expect(mockCaptureEvent).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx index 0f8388e8298..e7ec9a3b6a8 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -14,17 +14,15 @@ import { useParams } from 'next/navigation' import { useQueryState, useQueryStates } from 'nuqs' import { usePostHog } from 'posthog-js/react' import { captureEvent } from '@/lib/posthog/client' +import { useMothershipMode } from '@/app/workspace/[workspaceId]/home/hooks/use-mothership-mode' import { CLEARED_SEARCH_FILTERS, + MOTHERSHIP_MODES, + type MothershipMode, resourceUrlKeys, searchFilterParsers, searchQueryParam, } from '@/app/workspace/[workspaceId]/home/search-params' -import { - MOTHERSHIP_MODES, - type MothershipMode, - useMothershipModeStore, -} from '@/stores/mothership-mode/store' const MODE_LABELS: Record = { build: 'Build', @@ -42,8 +40,7 @@ const MODE_LABELS: Record = { export const ModeSwitcher = memo(function ModeSwitcher() { const { workspaceId } = useParams<{ workspaceId: string }>() const posthog = usePostHog() - const mode = useMothershipModeStore((state) => state.mode) - const setMode = useMothershipModeStore((state) => state.setMode) + const [mode, setMode] = useMothershipMode() const [, setSearchQueryParam] = useQueryState(searchQueryParam.key, searchQueryParam.parser) const [, setSearchFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) @@ -51,7 +48,7 @@ export const ModeSwitcher = memo(function ModeSwitcher() { /** Leaving Search drops the query from the URL, so a clean URL always means no search is showing. */ const handleSelect = (next: MothershipMode) => { if (next === mode) return - setMode(next) + void setMode(next) if (next !== 'search') { void setSearchQueryParam(null, { history: 'replace', scroll: false }) void setSearchFilters(CLEARED_SEARCH_FILTERS, { history: 'replace', scroll: false }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 71c754a1018..412766c2570 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -49,6 +49,7 @@ import { persistImportedWorkflow } from '@/lib/workflows/operations/import-expor import { KnowledgeSearchResults } from '@/app/workspace/[workspaceId]/home/components/knowledge-search-results' import { RESOURCE_HEADER_CLASSES } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-tabs/resource-tab-controls' import { SuggestedActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions' +import { useMothershipMode } from '@/app/workspace/[workspaceId]/home/hooks/use-mothership-mode' import { resolveWorkspaceResourceRef } from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' import { resolveResourceEventPresentation, @@ -56,6 +57,7 @@ import { } from '@/app/workspace/[workspaceId]/home/resource-view-policy' import { CLEARED_SEARCH_FILTERS, + type MothershipMode, resourceParam, resourceUrlKeys, searchFilterParsers, @@ -67,7 +69,6 @@ import { useMarkMothershipChatRead } from '@/hooks/queries/mothership-chats' import { useWorkflows } from '@/hooks/queries/workflows' import { getWorkspaceFilesQueryOptions, useWorkspaceFiles } from '@/hooks/queries/workspace-files' import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' import type { ChatContext } from '@/stores/panel' import { ChatSurfaceProvider, @@ -185,16 +186,15 @@ export function Home({ chatId, userName, userId }: HomeProps) { }, [setSearchQueryParam, setSearchFilters] ) + const [composerMode, setComposerMode] = useMothershipMode() /** - * A URL that carries a query opens in Search mode with the query in the box, - * whether it arrived by link or by navigating back to it; the composer - * follows the live query the same way (below), so the box and the results - * never show two different queries. + * A link that carries a query but no mode opens in Search with the query in + * the box; the composer follows the live query the same way (below), so the + * box and the results never show two different queries. */ useEffect(() => { - if (searchQuery) useMothershipModeStore.getState().setMode('search') - }, [searchQuery]) - const composerMode = useMothershipModeStore((state) => state.mode) + if (searchQuery && composerMode === 'build') void setComposerMode('search') + }, [searchQuery, composerMode, setComposerMode]) /** The bases an Ask turn is grounded in; read through a ref so a list refresh never rebuilds the submit handler. */ const { data: knowledgeBases = EMPTY_KNOWLEDGE_BASES } = useKnowledgeBasesQuery(workspaceId) const knowledgeBasesRef = useRef(knowledgeBases) @@ -475,7 +475,12 @@ export function Home({ chatId, userName, userId }: HomeProps) { }, [workspaceId, getCurrentRequestId, stopGeneration]) const handleSubmit = useCallback( - (text: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[]) => { + ( + text: string, + fileAttachments?: FileAttachmentForApi[], + contexts?: ChatContext[], + modeOverride?: MothershipMode + ) => { const trimmed = text.trim() if (!trimmed && !(fileAttachments && fileAttachments.length > 0)) return @@ -491,7 +496,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { * be searched: attachments alone have nothing to search for. Assistant * makes the query a turn of the agent grounded in the sources. */ - const mode = useMothershipModeStore.getState().mode + const mode = modeOverride ?? composerMode const answering = mode === 'assistant' if (mode === 'search') { if (trimmed) setSearchQuery(trimmed) @@ -516,17 +521,28 @@ export function Home({ chatId, userName, userId }: HomeProps) { answering ? { requestMode: 'ask' } : undefined ) }, - [workspaceId, chatId, prepareResourceViewForAgentTurn, sendMessage, setSearchQuery] + [ + workspaceId, + chatId, + composerMode, + prepareResourceViewForAgentTurn, + sendMessage, + setSearchQuery, + ] ) /** An emptied search box returns to the sources; nothing else reads the cleared query. */ const clearSearch = useCallback(() => setSearchQuery(''), [setSearchQuery]) - /** Summarize or Answer on a result: switch to Assistant and hand the question to it. */ + /** + * Summarize or Answer on a result: switch to Assistant and hand the question + * to it. The submit reads the mode from this render, so it is sent as an + * Assistant turn directly rather than waiting for the URL to update. + */ const handleSummarize = (prompt: string) => { - useMothershipModeStore.getState().setMode('assistant') + void setComposerMode('assistant') setSearchQuery('') - handleSubmit(prompt) + handleSubmit(prompt, undefined, undefined, 'assistant') } /** * A chat that already exists never opens in Search: its transcript is a @@ -534,9 +550,8 @@ export function Home({ chatId, userName, userId }: HomeProps) { * carry over, so a follow-up stays grounded in the sources. */ useEffect(() => { - const store = useMothershipModeStore.getState() - if (chatId && store.mode === 'search') store.setMode('build') - }, [chatId]) + if (chatId && composerMode === 'search') void setComposerMode('build') + }, [chatId, composerMode, setComposerMode]) const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 const searchResults = showSearchResults ? ( void - reset: () => void -} - -const initialState: Pick = { mode: 'build' } - -/** - * The chat composer's mode — Build (default), Search, or Assistant — read by - * the input's mode switcher and by the suggested actions beneath the input. - * Build is the agent with everything it can do; Search lists the matching - * documents with no turn at all; Assistant is a turn of the agent grounded in - * the searched sources, answering in natural language. - * - * A store rather than `Home` state because `Home` remounts per chat - * (`key={chatId}`) and the new-chat → `/chat/[chatId]` handoff must carry the - * mode across. Not a URL param: that handoff rewrites the path with - * `history.replaceState`, which would drop a query key, and the mode is a - * composer preference rather than a destination (the same reasoning that keeps - * canvas mode out of the URL). Deliberately not persisted, so the server render - * and the first client render agree — a persisted `search` would hydrate over - * server-rendered Build chrome. - */ -export const useMothershipModeStore = create()( - devtools( - (set) => ({ - ...initialState, - setMode: (mode) => set({ mode }), - reset: () => set(initialState), - }), - { name: 'mothership-mode-store' } - ) -) diff --git a/apps/sim/stores/reset-all-stores.ts b/apps/sim/stores/reset-all-stores.ts index 211d8180377..9665d01a09f 100644 --- a/apps/sim/stores/reset-all-stores.ts +++ b/apps/sim/stores/reset-all-stores.ts @@ -3,7 +3,6 @@ import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { useExecutionStore } from '@/stores/execution' import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store' -import { useMothershipModeStore } from '@/stores/mothership-mode/store' import { useMothershipQueueStore } from '@/stores/mothership-queue/store' import { useOperationQueueStore } from '@/stores/operation-queue/store' import { @@ -56,6 +55,5 @@ export async function resetAllStores(): Promise { clearAllExecutionPointers() useMothershipDraftsStore.setState({ drafts: {} }) useMothershipQueueStore.getState().reset() - useMothershipModeStore.getState().reset() await consolePersistence.persist({ merge: false }) } From cb68ed3b6208a49aa46962594e19c832317a18f8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:17:27 -0700 Subject: [PATCH 71/76] fix(home): let Search be chosen inside a chat now that the mode lives in the URL --- apps/sim/app/workspace/[workspaceId]/home/home.tsx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 412766c2570..712d94e465f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -544,14 +544,6 @@ export function Home({ chatId, userName, userId }: HomeProps) { setSearchQuery('') handleSubmit(prompt, undefined, undefined, 'assistant') } - /** - * A chat that already exists never opens in Search: its transcript is a - * conversation, and search results never join it. Build and Assistant both - * carry over, so a follow-up stays grounded in the sources. - */ - useEffect(() => { - if (chatId && composerMode === 'search') void setComposerMode('build') - }, [chatId, composerMode, setComposerMode]) const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 const searchResults = showSearchResults ? ( Date: Wed, 2 Sep 2026 13:31:52 -0700 Subject: [PATCH 72/76] fix(chat): scope the managed-credential listing to the group's workspace and carry only chat params on the handoff --- .../[workspaceId]/home/hooks/chat-url.test.ts | 28 +++++++++++++++++++ .../[workspaceId]/home/hooks/chat-url.ts | 23 +++++++++++---- apps/sim/lib/credentials/environment.ts | 1 + 3 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.test.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.test.ts new file mode 100644 index 00000000000..c2a8e485d3f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.test.ts @@ -0,0 +1,28 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it } from 'vitest' +import { chatUrl } from '@/app/workspace/[workspaceId]/home/hooks/chat-url' + +function withSearch(search: string) { + window.history.replaceState(null, '', `/workspace/ws-1/home${search}`) +} + +describe('chatUrl', () => { + it('carries the mode and the open resource onto the chat path', () => { + withSearch('?mode=assistant&resource=res-1') + expect(chatUrl('ws-1', 'chat-1')).toBe( + '/workspace/ws-1/chat/chat-1?mode=assistant&resource=res-1' + ) + }) + + it('leaves a search query and its filters behind', () => { + withSearch('?q=volvo&source=gmail&updated=7d&mode=assistant') + expect(chatUrl('ws-1', 'chat-1')).toBe('/workspace/ws-1/chat/chat-1?mode=assistant') + }) + + it('produces a clean path when nothing belongs on the chat', () => { + withSearch('?q=volvo') + expect(chatUrl('ws-1', 'chat-1')).toBe('/workspace/ws-1/chat/chat-1') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts index 34df305d989..2046927bc57 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts @@ -1,9 +1,22 @@ +import { modeParam, resourceParam } from '@/app/workspace/[workspaceId]/home/search-params' + +/** The composer's URL state that belongs on a chat page: the mode and the open resource. */ +const CHAT_URL_PARAMS = [modeParam.key, resourceParam.key] as const + /** - * The URL a new chat is handed off to once the server names it. The current - * query string rides along so the composer's URL-backed state, the mode above - * all, survives the path swap: the first Assistant message must not bounce the - * person back to Build. + * The URL a new chat is handed off to once the server names it. Only the + * params that belong on a chat ride along, so the mode survives the path swap + * (the first Assistant message must not bounce the person back to Build) while + * a search's `q` and filters, which never join a transcript, are left behind + * whatever the URL held at that instant. */ export function chatUrl(workspaceId: string, chatId: string): string { - return `/workspace/${workspaceId}/chat/${chatId}${window.location.search}` + const current = new URLSearchParams(window.location.search) + const carried = new URLSearchParams() + for (const key of CHAT_URL_PARAMS) { + const value = current.get(key) + if (value) carried.set(key, value) + } + const search = carried.toString() + return `/workspace/${workspaceId}/chat/${chatId}${search ? `?${search}` : ''}` } diff --git a/apps/sim/lib/credentials/environment.ts b/apps/sim/lib/credentials/environment.ts index 3f6a4815d0e..569f5663b22 100644 --- a/apps/sim/lib/credentials/environment.ts +++ b/apps/sim/lib/credentials/environment.ts @@ -876,6 +876,7 @@ export async function getEnrolledManagedOAuthCredentials( .where( and( eq(credential.workspaceId, workspaceId), + eq(credentialGroup.workspaceId, workspaceId), eq(credential.type, 'managed_oauth'), eq(user.id, userId), eq(user.emailVerified, true) From 6da5af5c385454106c8ac51812a7f2591f80e8f0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:35:57 -0700 Subject: [PATCH 73/76] fix(credentials): refuse a managed credential whose group or option is disabled on every mint, workflow runs included --- .../application/authorization.test.ts | 12 +++++++ .../application/authorization.ts | 33 +++++++++++++------ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/apps/sim/lib/credential-groups/application/authorization.test.ts b/apps/sim/lib/credential-groups/application/authorization.test.ts index e56191b9add..5522e1402b9 100644 --- a/apps/sim/lib/credential-groups/application/authorization.test.ts +++ b/apps/sim/lib/credential-groups/application/authorization.test.ts @@ -148,6 +148,18 @@ describe('requireCredentialGroupCredentialAccess', () => { await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) }) + it('denies a workflow run the same way once the group or option is disabled', async () => { + mocks.loadBinding.mockResolvedValue({ ...liveBinding, optionStatus: 'disabled' }) + await expect(requireAccess(executorPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.requirePolicy).not.toHaveBeenCalled() + }) + + it('denies a Chat turn for a credential with no OAuth binding, which a workflow may still hold', async () => { + mocks.loadBinding.mockResolvedValue(null) + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + await expect(requireAccess(executorPrincipal())).resolves.toBeUndefined() + }) + it("allows a Chat turn to use only the credential under the signed-in user's own enrollment", async () => { await expect(requireAccess(copilotPrincipal())).resolves.toBeUndefined() expect(mocks.loadEnrollmentAccess).toHaveBeenCalledWith('group-1', { diff --git a/apps/sim/lib/credential-groups/application/authorization.ts b/apps/sim/lib/credential-groups/application/authorization.ts index ea25d0916cf..10a4bda02a7 100644 --- a/apps/sim/lib/credential-groups/application/authorization.ts +++ b/apps/sim/lib/credential-groups/application/authorization.ts @@ -15,7 +15,10 @@ import { evaluateCredentialGroupActorCredentialAccess, evaluateCredentialGroupWorkflowAccess, } from '@/lib/credential-groups/application/workflow-access-policy' -import type { CredentialGroupCredentialListContext } from '@/lib/credential-groups/credentials' +import type { + CredentialGroupCredentialListContext, + ManagedCredentialGroupBinding, +} from '@/lib/credential-groups/credentials' import { isManagedCredentialGroupBindingLive, loadCredentialGroupEnrollmentAccessForSubject, @@ -97,17 +100,19 @@ export function requireCredentialGroupWorkflowActor(principal: Principal): Princ */ async function requireCredentialGroupActorCredentialAccess( principal: Extract, - context: CredentialGroupAuthorizationContext & { - credentialId: string - credentialGroupEnrollmentId: string - }, + context: CredentialGroupAuthorizationContext & { credentialGroupEnrollmentId: string }, + binding: ManagedCredentialGroupBinding | null, resourcePolicy: ResourcePolicyBindingFor<'credential_group'> ): Promise { const subject = resolvePrincipalSubject(principal) if (subject?.kind !== 'sim_user' || !subject.userId) { throw new OrchestrationError('forbidden', 'Credential Group actor access required') } - const [policy, actorAccess, binding] = await Promise.all([ + /** Chat mints OAuth credentials only; a credential with no OAuth binding is not its to use. */ + if (!binding) { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } + const [policy, actorAccess] = await Promise.all([ requireResourcePolicy({ workspaceId: context.workspaceId, resourceType: 'credential_group', @@ -115,10 +120,8 @@ async function requireCredentialGroupActorCredentialAccess( codec: credentialGroupWorkflowAccessPolicyCodec, }), loadCredentialGroupEnrollmentAccessForSubject(context.credentialGroupId, subject), - loadManagedCredentialGroupBinding(context.credentialId), ]) - /** A disabled group or option denies here, as it does for every other consumer of a binding. */ - if (!actorAccess || !binding || !isManagedCredentialGroupBindingLive(binding)) { + if (!actorAccess) { throw new OrchestrationError('forbidden', 'Credential Group credential access denied') } const decision = evaluateCredentialGroupActorCredentialAccess({ @@ -141,8 +144,18 @@ export async function requireCredentialGroupCredentialAccess( }, resourcePolicy: ResourcePolicyBindingFor<'credential_group'> ): Promise { + /** + * A managed OAuth credential is usable only while its credential, enrollment, + * option, and group are all live, whoever is using it: an admin disabling the + * group or option denies the next mint from a workflow and from Chat alike. A + * managed MCP credential has no OAuth binding row and keeps its own checks. + */ + const binding = await loadManagedCredentialGroupBinding(context.credentialId) + if (binding && !isManagedCredentialGroupBindingLive(binding)) { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } if (principal.kind === 'delegated' && principal.serviceId === 'copilot') { - return requireCredentialGroupActorCredentialAccess(principal, context, resourcePolicy) + return requireCredentialGroupActorCredentialAccess(principal, context, binding, resourcePolicy) } const executionPrincipal = requireWorkflowExecutionPrincipal(principal) const currentWorkflow = requireCurrentWorkflow(principal) From dfeaf87ff684b4577d7a1f6dda4e6aa2718a38bb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:42:28 -0700 Subject: [PATCH 74/76] fix(home): clear the composer when a result is handed to the Assistant, follow the search query in a chat, restore a queued message's mode, and seed the base list --- .../knowledge-search-results.tsx | 2 +- .../mothership-chat/mothership-chat.tsx | 20 ++++++-- .../home/components/user-input/user-input.tsx | 49 +++++++++++-------- .../app/workspace/[workspaceId]/home/home.tsx | 35 +++++++++++-- .../workspace/[workspaceId]/home/prefetch.ts | 28 ++++++++++- .../app/workspace/[workspaceId]/home/types.ts | 4 +- 6 files changed, 105 insertions(+), 33 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 966fca17ff9..5c9f0a1b469 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -231,7 +231,7 @@ export function KnowledgeSearchResults({ if (!basesPending && knowledgeBaseIds.length === 0) { return (

- Nothing to search yet. Connect a source above to index what you can open. + Nothing to search yet. Clear the query and connect a source to index what you can open.

) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx index e1bb764b241..977ef369b7f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx @@ -3,6 +3,7 @@ import { memo, type ReactNode, + type RefObject, useCallback, useDeferredValue, useEffect, @@ -63,6 +64,12 @@ interface MothershipChatProps { isSending: boolean /** The composer's Search-mode results, shown above the input. */ searchResults?: ReactNode + /** The live search query; the composer shows it so the box and the results never disagree. */ + searchQuery?: string + /** The composer, for a caller that hands a question to the agent from outside the box. */ + userInputRef?: RefObject + /** Puts the composer in the mode a queued message was written in, when one is loaded for editing. */ + onRestoreQueuedMode?: (requestMode: QueuedMessage['requestMode']) => void isReconnecting?: boolean isLoading?: boolean onSubmit: ( @@ -319,6 +326,9 @@ export function MothershipChat({ messages: messagesProp, isSending, searchResults, + searchQuery, + userInputRef: userInputRefProp, + onRestoreQueuedMode, isReconnecting = false, isLoading = false, onSubmit, @@ -663,7 +673,8 @@ export function MothershipChat({ item.index !== lastIndex && item.start < (instance.scrollElement?.scrollTop ?? 0) const scrolledChatRef = useRef(UNSCROLLED) - const userInputRef = useRef(null) + const ownUserInputRef = useRef(null) + const userInputRef = userInputRefProp ?? ownUserInputRef const messageQueueRef = useRef(messageQueue) useEffect(() => { messageQueueRef.current = messageQueue @@ -686,9 +697,11 @@ export function MothershipChat({ const handleEditQueued = useCallback( (id: string) => { const msg = onEditQueuedMessage(id) - if (msg) userInputRef.current?.loadQueuedMessage(msg) + if (!msg) return + onRestoreQueuedMode?.(msg.requestMode) + userInputRef.current?.loadQueuedMessage(msg) }, - [onEditQueuedMessage] + [onEditQueuedMessage, onRestoreQueuedMode, userInputRef] ) const handleEditQueuedTail = useCallback(() => { @@ -831,6 +844,7 @@ export function MothershipChat({ void + /** Empties the composer and its draft, as a send does; for a question handed to the agent from outside the box. */ + clear: () => void } /** @@ -434,6 +436,7 @@ const UserInputImpl = forwardRef(function UserI currentEditor.setContexts(msg.contexts ?? []) currentEditor.focusAtEnd() }, + clear: clearComposer, populatePrompt: (text: string) => { // `text` is a curated prompt, so opt its bare integration names into // `@`-mention form before chipification (the auto-mention pipeline only @@ -551,6 +554,24 @@ const UserInputImpl = forwardRef(function UserI textareaRef.current?.focus() } + /** Empties the text, chips, attachments, transcript, and the saved draft in one step. */ + const clearComposer = useCallback(() => { + editorRef.current.clear() + sttPrefixRef.current = '' + if (draftSaveTimerRef.current !== null) { + window.clearTimeout(draftSaveTimerRef.current) + draftSaveTimerRef.current = null + } + pendingDraftRef.current = null + if (draftScopeKeyRef.current) { + useMothershipDraftsStore.getState().clearDraft(draftScopeKeyRef.current) + } + /** The chips are gone with the text, and clearing is not a removal to report. */ + prevSelectedContextsRef.current = [] + resetTranscript() + filesRef.current.clearAttachedFiles() + }, [resetTranscript]) + const handleSubmit = useCallback(() => { const currentFiles = filesRef.current const currentEditor = editorRef.current @@ -575,27 +596,13 @@ const UserInputImpl = forwardRef(function UserI fileAttachmentsForApi.length > 0 ? fileAttachmentsForApi : undefined, activeContexts.length > 0 ? activeContexts : undefined ) - if (clearOnSubmitRef.current) { - currentEditor.clear() - sttPrefixRef.current = '' - if (draftSaveTimerRef.current !== null) { - window.clearTimeout(draftSaveTimerRef.current) - draftSaveTimerRef.current = null - } - pendingDraftRef.current = null - if (draftScopeKeyRef.current) { - useMothershipDraftsStore.getState().clearDraft(draftScopeKeyRef.current) - } - /** - * The chips are gone with the text, and clearing is not a removal to - * report. A composer that keeps its text (Search mode) keeps its chips - * too, so the diff base stays in step with what is still selected. - */ - prevSelectedContextsRef.current = [] - } - resetTranscript() - currentFiles.clearAttachedFiles() - }, [onSubmit, resetTranscript]) + /** + * A composer that keeps its text (Search mode) keeps its attachments and + * chips too: the search took the query alone, and the person may hand the + * rest to the agent next. + */ + if (clearOnSubmitRef.current) clearComposer() + }, [onSubmit, clearComposer]) /** * Enter policy for the editor: mirror canSubmit's uploading guard (Enter diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 712d94e465f..121af751033 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -89,6 +89,7 @@ import type { FileAttachmentForApi, MothershipResource, MothershipResourceType, + QueuedMessage, WorkspaceResourceRef, } from './types' @@ -193,7 +194,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { * box and the results never show two different queries. */ useEffect(() => { - if (searchQuery && composerMode === 'build') void setComposerMode('search') + if (searchQuery.trim() && composerMode === 'build') void setComposerMode('search') }, [searchQuery, composerMode, setComposerMode]) /** The bases an Ask turn is grounded in; read through a ref so a list refresh never rebuilds the submit handler. */ const { data: knowledgeBases = EMPTY_KNOWLEDGE_BASES } = useKnowledgeBasesQuery(workspaceId) @@ -202,6 +203,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { const hasCheckedLandingStorageRef = useRef(false) const initialViewInputRef = useRef(null) const initialViewUserInputRef = useRef(null) + const chatViewUserInputRef = useRef(null) const [isInputEntering, setIsInputEntering] = useState(false) @@ -499,6 +501,8 @@ export function Home({ chatId, userName, userId }: HomeProps) { const mode = modeOverride ?? composerMode const answering = mode === 'assistant' if (mode === 'search') { + /** A search sends nothing, so an edit in progress is released rather than left waiting. */ + if (editingQueuedId) cancelQueueEdit() if (trimmed) setSearchQuery(trimmed) return } @@ -525,23 +529,43 @@ export function Home({ chatId, userName, userId }: HomeProps) { workspaceId, chatId, composerMode, + editingQueuedId, + cancelQueueEdit, prepareResourceViewForAgentTurn, sendMessage, setSearchQuery, ] ) - /** An emptied search box returns to the sources; nothing else reads the cleared query. */ - const clearSearch = useCallback(() => setSearchQuery(''), [setSearchQuery]) + /** + * A queued message re-enters the composer in the mode it was written in: an + * Assistant question edits as an Assistant question, and never as a Search, + * which submits nothing and would leave the edit stranded. + */ + const restoreQueuedMode = useCallback( + (requestMode: QueuedMessage['requestMode']) => { + void setComposerMode(requestMode === 'ask' ? 'assistant' : 'build') + }, + [setComposerMode] + ) + + /** An emptied search box returns to the sources; a send in any other mode has no search to clear. */ + const clearSearch = useCallback(() => { + if (searchQueryValue !== null) setSearchQuery('') + }, [searchQueryValue, setSearchQuery]) /** * Summarize or Answer on a result: switch to Assistant and hand the question * to it. The submit reads the mode from this render, so it is sent as an - * Assistant turn directly rather than waiting for the URL to update. + * Assistant turn directly rather than waiting for the URL to update, and the + * box is emptied as a send empties it, so the query does not linger as a + * draft under the answer. */ const handleSummarize = (prompt: string) => { void setComposerMode('assistant') setSearchQuery('') + initialViewUserInputRef.current?.clear() + chatViewUserInputRef.current?.clear() handleSubmit(prompt, undefined, undefined, 'assistant') } const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 @@ -800,6 +824,9 @@ export function Home({ chatId, userName, userId }: HomeProps) { messages={messages} isSending={isSending} searchResults={searchResults} + searchQuery={searchQuery} + userInputRef={chatViewUserInputRef} + onRestoreQueuedMode={restoreQueuedMode} isReconnecting={isReconnecting} isLoading={showChatSkeleton} onSubmit={handleSubmit} diff --git a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts index b47b000b0f1..bb634b7fd9b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts @@ -1,13 +1,21 @@ import type { QueryClient } from '@tanstack/react-query' +import { listKnowledgeBasesContract } from '@/lib/api/contracts/knowledge' +import { internalSessionAuth } from '@/lib/api/server/routes' +import { internalKnowledgePresenters } from '@/lib/knowledge/api/internal-route' +import { listInternalKnowledgeBases } from '@/lib/knowledge/application/knowledge-bases' import { getWorkspaceHostContextForViewer } from '@/lib/workspaces/host-context' import { seedWorkspaceFiles } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files' +import { KNOWLEDGE_BASE_LIST_STALE_TIME, knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' /** * Prefetches what the Home surface needs on top of the workspace layout's own prefetch. * * Home reads the workspace file list on mount (resource tabs, mentions, the resource picker), so * the list is seeded by the routes that render Home rather than by the layout: seeding it in the - * layout would pay for it on every workspace route, including the ones that never read it. + * layout would pay for it on every workspace route, including the ones that never read it. The + * knowledge-base list is seeded the same way, under the client hook's key and stale time: an + * Assistant turn attaches the searched bases at submit, and a first question typed before the + * list arrived would otherwise go out with nothing to search. * * The seed carries no authorization of its own, so the viewer is proved first. This reuses the * layout's `cache`d host-context lookup rather than re-deriving the permission, so it costs no @@ -23,5 +31,21 @@ export async function prefetchHomeSurface( const hostContext = await getWorkspaceHostContextForViewer(workspaceId, userId) if (!hostContext) return - await seedWorkspaceFiles(queryClient, workspaceId) + await Promise.all([ + seedWorkspaceFiles(queryClient, workspaceId), + queryClient.prefetchQuery({ + queryKey: knowledgeKeys.list(workspaceId, 'active'), + queryFn: async () => { + const principal = await internalSessionAuth.authenticate() + const result = await listInternalKnowledgeBases.execute({ + principal, + input: { workspaceId, scope: 'active' }, + }) + return listKnowledgeBasesContract.response.schema.parse( + internalKnowledgePresenters.list(result) + ).data + }, + staleTime: KNOWLEDGE_BASE_LIST_STALE_TIME, + }), + ]) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/types.ts b/apps/sim/app/workspace/[workspaceId]/home/types.ts index 8b106ba4ee5..778f6f5ba68 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/types.ts @@ -25,8 +25,8 @@ export interface FileAttachmentForApi { /** * A request mode a send asks the agent for beyond the default. `ask` is an - * answer drawn from the attached knowledge bases with the knowledge tool - * alone: the server attaches no integration tools to the turn. + * Assistant turn: an answer drawn from the attached knowledge bases first, + * with a connected integration reached only when those cannot answer. */ export type ChatRequestMode = 'ask' From ed3d7ab3c8f9e0c51fbf509f98f15d801fd0d999 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:48:27 -0700 Subject: [PATCH 75/76] fix(knowledge): budget the pending ACL rewrite per run, and converge a racing first connect on the oldest base and connector --- .../knowledge/application/sim-search.test.ts | 37 +++++++++- .../lib/knowledge/application/sim-search.ts | 73 +++++++++++++------ .../connectors/member-sync-engine.ts | 23 +++++- 3 files changed, 106 insertions(+), 27 deletions(-) diff --git a/apps/sim/lib/knowledge/application/sim-search.test.ts b/apps/sim/lib/knowledge/application/sim-search.test.ts index e1d9e7450a4..8ec011f4310 100644 --- a/apps/sim/lib/knowledge/application/sim-search.test.ts +++ b/apps/sim/lib/knowledge/application/sim-search.test.ts @@ -11,7 +11,9 @@ const mocks = vi.hoisted(() => ({ resolvePermission: vi.fn(), isMemberAccessAvailable: vi.fn(), createKnowledgeBase: vi.fn(), + deleteKnowledgeBase: vi.fn(), createConnector: vi.fn(), + deleteConnector: vi.fn(), enroll: vi.fn(), getUserPermissionConfig: vi.fn(), recordAudit: vi.fn(), @@ -53,10 +55,12 @@ vi.mock('@/lib/knowledge/access/availability', async () => { vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ createKnowledgeBase: { execute: mocks.createKnowledgeBase }, + deleteKnowledgeBaseOperation: { execute: mocks.deleteKnowledgeBase }, })) vi.mock('@/lib/knowledge/application/connectors', () => ({ createKnowledgeConnector: { execute: mocks.createConnector }, + deleteKnowledgeConnector: { execute: mocks.deleteConnector }, })) vi.mock('@/lib/knowledge/application/connector-access', () => ({ @@ -186,8 +190,12 @@ describe('connectSimSearchConnector', () => { it('lets an admin create the base and the connector with the setup fields, then enrolls them', async () => { mocks.resolvePermission.mockResolvedValue('admin') - queueConnectorLookups(null, null) queueTableRows(knowledgeBase, []) + queueTableRows(knowledgeBase, [{ id: 'kb-new' }]) + queueConnectorLookups(null, null, { + knowledgeBaseId: 'kb-new', + connectorId: 'connector-new', + } as typeof existingConnector) const result = await connectSimSearchConnector.execute({ principal, @@ -198,6 +206,9 @@ describe('connectSimSearchConnector', () => { }, }) + expect(mocks.deleteKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.deleteConnector).not.toHaveBeenCalled() + expect(mocks.createKnowledgeBase).toHaveBeenCalledWith( expect.objectContaining({ principal, @@ -254,4 +265,28 @@ describe('connectSimSearchConnector', () => { expect(mocks.createConnector).not.toHaveBeenCalled() expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) }) + + it('converges on the row another instance created first and deletes its own', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueTableRows(knowledgeBase, []) + queueTableRows(knowledgeBase, [{ id: existingConnector.knowledgeBaseId }]) + queueConnectorLookups(null, null, existingConnector) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + + expect(mocks.deleteKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ knowledgeBaseId: 'kb-new' }), + }) + ) + expect(mocks.deleteConnector).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ connectorId: 'connector-new', deleteDocuments: true }), + }) + ) + expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) + }) }) diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts index eea2f3dc116..5123723e4d1 100644 --- a/apps/sim/lib/knowledge/application/sim-search.ts +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -11,12 +11,18 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' -import { createKnowledgeConnector } from '@/lib/knowledge/application/connectors' +import { + createKnowledgeConnector, + deleteKnowledgeConnector, +} from '@/lib/knowledge/application/connectors' import { type KnowledgeWorkspaceContext, resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' -import { createKnowledgeBase } from '@/lib/knowledge/application/knowledge-bases' +import { + createKnowledgeBase, + deleteKnowledgeBaseOperation, +} from '@/lib/knowledge/application/knowledge-bases' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { canConnectPersonally, @@ -154,27 +160,36 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ /** * Concurrent first connects in this process share one creation per * workspace base and per source, and each re-checks before creating. - * Two instances can still race in the same instant; both then converge - * on the oldest row, which every lookup here orders by, and the stray - * one is inert. + * Two instances can still race in the same instant: every lookup here + * orders by the oldest row, so after creating, each re-reads and the one + * that finds an older row than its own deletes what it just made and + * converges on the older one. Nothing stray outlives the request. */ - const knowledgeBaseId = await coalesceLocally( - `sim-search:base:${workspaceId}`, - async () => - (await findSimSearchKnowledgeBase(workspaceId))?.id ?? - ( - await createKnowledgeBase.execute({ - principal, - input: { - workspaceId, - name: SIM_SEARCH_KNOWLEDGE_BASE_NAME, - description: SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION, - source: 'ui', - }, - request, - }) - ).knowledgeBase.id - ) + const knowledgeBaseId = await coalesceLocally(`sim-search:base:${workspaceId}`, async () => { + const existing = await findSimSearchKnowledgeBase(workspaceId) + if (existing) return existing.id + const created = ( + await createKnowledgeBase.execute({ + principal, + input: { + workspaceId, + name: SIM_SEARCH_KNOWLEDGE_BASE_NAME, + description: SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION, + source: 'ui', + }, + request, + }) + ).knowledgeBase.id + const oldest = (await findSimSearchKnowledgeBase(workspaceId))?.id ?? created + if (oldest !== created) { + await deleteKnowledgeBaseOperation.execute({ + principal, + input: { knowledgeBaseId: created, assertedWorkspaceId: workspaceId, source: 'ui' }, + request, + }) + } + return oldest + }) target = await coalesceLocally( `sim-search:connect:${workspaceId}:${input.connectorType}`, async () => { @@ -193,6 +208,20 @@ export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ }, request, }) + const oldest = await findSimSearchConnector(workspaceId, input.connectorType) + if (oldest && oldest.connectorId !== created.connector.id) { + await deleteKnowledgeConnector.execute({ + principal, + input: { + connectorId: created.connector.id, + assertedWorkspaceId: workspaceId, + deleteDocuments: true, + source: 'ui', + }, + request, + }) + return oldest + } return { knowledgeBaseId, connectorId: created.connector.id } } ) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index c1c94cfd8e7..afda33d905f 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -465,13 +465,17 @@ async function insertMemberSyncLog(runId: string, connectorId: string, startedAt /** * Finishes an ACL rewrite a mode switch left behind before this run lists * anything: every document of the connector is hidden until an observation - * makes it visible again. + * makes it visible again. Bounded by the run's own budget like every other + * step, so a large corpus is hidden across as many runs as it takes rather + * than one run that never reaches a member; returns whether it finished. */ -async function finishPendingAccessRewrite(run: MemberSyncRun): Promise { - await rewriteConnectorAcls(run.connectorId, EMPTY_ACL, { +async function finishPendingAccessRewrite(run: MemberSyncRun): Promise { + const finished = await rewriteConnectorAcls(run.connectorId, EMPTY_ACL, { + deadlineAt: run.deadlineAt, beforeBatch: run.lease.beatIfDue, lease: run.lease, }) + if (!finished) return false await db .update(knowledgeConnector) .set({ accessRewritePending: false, updatedAt: new Date() }) @@ -481,6 +485,7 @@ async function finishPendingAccessRewrite(run: MemberSyncRun): Promise { stillHoldsMemberSyncLock(run.connectorId, run.runId) ) ) + return true } interface MembershipReconciliation { @@ -1376,7 +1381,17 @@ export async function executeMemberSync( } const sourceConfig = connector.sourceConfig as Record - if (connector.accessRewritePending) await finishPendingAccessRewrite(run) + if (connector.accessRewritePending && !(await finishPendingAccessRewrite(run))) { + /** The rewrite is not done, so nothing is listed yet; the next run picks it up at once. */ + result.membersRemaining = true + const landed = await completeMemberSync(run, connector.syncIntervalMinutes) + if (!landed) return skipped(result, 'sync_superseded') + logger.info('Member sync spent its budget hiding documents after a mode switch', { + connectorId, + runId, + }) + return result + } const affectedDocumentIds = new Set() /** From 02bf4c751dc3088b966bcb7ea1c84c6f8ea942c4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 13:50:54 -0700 Subject: [PATCH 76/76] fix(home): ground an Assistant send from the query cache instead of growing the page graph with a prefetch --- .../app/workspace/[workspaceId]/home/home.tsx | 30 +++++++++++-------- .../workspace/[workspaceId]/home/prefetch.ts | 28 ++--------------- 2 files changed, 20 insertions(+), 38 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 121af751033..eb7b651a274 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -21,7 +21,6 @@ import { useQueryState, useQueryStates } from 'nuqs' import { usePostHog } from 'posthog-js/react' import { requestJson } from '@/lib/api/client/request' import { createWorkflowContract } from '@/lib/api/contracts' -import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge/base' import { LandingPromptStorage, type LandingWorkflowSeed, @@ -64,8 +63,9 @@ import { searchQueryParam, } from '@/app/workspace/[workspaceId]/home/search-params' import { useFolders } from '@/hooks/queries/folders' -import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' +import { fetchKnowledgeBases } from '@/hooks/queries/kb/knowledge' import { useMarkMothershipChatRead } from '@/hooks/queries/mothership-chats' +import { KNOWLEDGE_BASE_LIST_STALE_TIME, knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { useWorkflows } from '@/hooks/queries/workflows' import { getWorkspaceFilesQueryOptions, useWorkspaceFiles } from '@/hooks/queries/workspace-files' import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' @@ -95,9 +95,6 @@ import type { const logger = createLogger('Home') -/** Stable empty list, so a missing base list never rebuilds what reads it. */ -const EMPTY_KNOWLEDGE_BASES: KnowledgeBaseData[] = [] - /** * The resource preview panel pulls in the file-viewer stack (rich-markdown * editor, CSV/PDF viewers). It only renders once a chat has messages, so it is @@ -196,10 +193,6 @@ export function Home({ chatId, userName, userId }: HomeProps) { useEffect(() => { if (searchQuery.trim() && composerMode === 'build') void setComposerMode('search') }, [searchQuery, composerMode, setComposerMode]) - /** The bases an Ask turn is grounded in; read through a ref so a list refresh never rebuilds the submit handler. */ - const { data: knowledgeBases = EMPTY_KNOWLEDGE_BASES } = useKnowledgeBasesQuery(workspaceId) - const knowledgeBasesRef = useRef(knowledgeBases) - knowledgeBasesRef.current = knowledgeBases const hasCheckedLandingStorageRef = useRef(false) const initialViewInputRef = useRef(null) const initialViewUserInputRef = useRef(null) @@ -477,7 +470,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { }, [workspaceId, getCurrentRequestId, stopGeneration]) const handleSubmit = useCallback( - ( + async ( text: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[], @@ -512,10 +505,22 @@ export function Home({ chatId, userName, userId }: HomeProps) { } prepareResourceViewForAgentTurn() + /** + * An Assistant turn is grounded in the searched bases, read from the + * query cache the Search panel shares: instant once loaded, and awaited + * the one time a question is typed before the list has arrived. + */ const turnContexts = answering ? withSearchedKnowledgeContexts( contexts, - searchedKnowledgeBases(knowledgeBasesRef.current, workspaceId) + searchedKnowledgeBases( + await queryClient.ensureQueryData({ + queryKey: knowledgeKeys.list(workspaceId, 'active'), + queryFn: ({ signal }) => fetchKnowledgeBases(workspaceId, 'active', signal), + staleTime: KNOWLEDGE_BASE_LIST_STALE_TIME, + }), + workspaceId + ) ) : contexts sendMessage( @@ -532,6 +537,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { editingQueuedId, cancelQueueEdit, prepareResourceViewForAgentTurn, + queryClient, sendMessage, setSearchQuery, ] @@ -566,7 +572,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { setSearchQuery('') initialViewUserInputRef.current?.clear() chatViewUserInputRef.current?.clear() - handleSubmit(prompt, undefined, undefined, 'assistant') + void handleSubmit(prompt, undefined, undefined, 'assistant') } const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 const searchResults = showSearchResults ? ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts index bb634b7fd9b..b47b000b0f1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts @@ -1,21 +1,13 @@ import type { QueryClient } from '@tanstack/react-query' -import { listKnowledgeBasesContract } from '@/lib/api/contracts/knowledge' -import { internalSessionAuth } from '@/lib/api/server/routes' -import { internalKnowledgePresenters } from '@/lib/knowledge/api/internal-route' -import { listInternalKnowledgeBases } from '@/lib/knowledge/application/knowledge-bases' import { getWorkspaceHostContextForViewer } from '@/lib/workspaces/host-context' import { seedWorkspaceFiles } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files' -import { KNOWLEDGE_BASE_LIST_STALE_TIME, knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' /** * Prefetches what the Home surface needs on top of the workspace layout's own prefetch. * * Home reads the workspace file list on mount (resource tabs, mentions, the resource picker), so * the list is seeded by the routes that render Home rather than by the layout: seeding it in the - * layout would pay for it on every workspace route, including the ones that never read it. The - * knowledge-base list is seeded the same way, under the client hook's key and stale time: an - * Assistant turn attaches the searched bases at submit, and a first question typed before the - * list arrived would otherwise go out with nothing to search. + * layout would pay for it on every workspace route, including the ones that never read it. * * The seed carries no authorization of its own, so the viewer is proved first. This reuses the * layout's `cache`d host-context lookup rather than re-deriving the permission, so it costs no @@ -31,21 +23,5 @@ export async function prefetchHomeSurface( const hostContext = await getWorkspaceHostContextForViewer(workspaceId, userId) if (!hostContext) return - await Promise.all([ - seedWorkspaceFiles(queryClient, workspaceId), - queryClient.prefetchQuery({ - queryKey: knowledgeKeys.list(workspaceId, 'active'), - queryFn: async () => { - const principal = await internalSessionAuth.authenticate() - const result = await listInternalKnowledgeBases.execute({ - principal, - input: { workspaceId, scope: 'active' }, - }) - return listKnowledgeBasesContract.response.schema.parse( - internalKnowledgePresenters.list(result) - ).data - }, - staleTime: KNOWLEDGE_BASE_LIST_STALE_TIME, - }), - ]) + await seedWorkspaceFiles(queryClient, workspaceId) }