diff --git a/internal-packages/clickhouse/schema/042_reduce_task_events_v2_storage_overhead.sql b/internal-packages/clickhouse/schema/042_reduce_task_events_v2_storage_overhead.sql new file mode 100644 index 00000000000..c8cbeb24825 --- /dev/null +++ b/internal-packages/clickhouse/schema/042_reduce_task_events_v2_storage_overhead.sql @@ -0,0 +1,21 @@ +-- +goose Up + +-- Full-text search is served outside the source event table. Keeping these +-- indexes here adds work to every event insert and merge without serving reads. +ALTER TABLE trigger_dev.task_events_v2 + DROP INDEX IF EXISTS idx_attributes_text_search; + +ALTER TABLE trigger_dev.task_events_v2 + DROP INDEX IF EXISTS idx_message_text_search; + +-- attributes remains an insert input for attributes_text, but is no longer +-- stored. Writers must include attributes in an explicit insert column list +-- because implicit INSERT column lists exclude EPHEMERAL columns. +ALTER TABLE trigger_dev.task_events_v2 + MODIFY COLUMN attributes JSON EPHEMERAL; + +-- +goose Down + +-- Restoring the stored JSON column safely requires inspecting the live schema +-- and coordinating the writer rollback. Use a new forward migration instead. +SELECT throwIf(1, 'This migration cannot be rolled back automatically'); diff --git a/internal-packages/clickhouse/src/taskEvents.test.ts b/internal-packages/clickhouse/src/taskEvents.test.ts index 8a2599bf90d..deb32ac1697 100644 --- a/internal-packages/clickhouse/src/taskEvents.test.ts +++ b/internal-packages/clickhouse/src/taskEvents.test.ts @@ -62,6 +62,49 @@ describe("task events v2", () => { has_inserted_at: 1, }, ]); + + const readColumnKinds = ch.reader.query({ + name: "read-task-event-attribute-column-kinds", + query: `SELECT name, default_kind, default_expression + FROM system.columns + WHERE database = 'trigger_dev' + AND table = 'task_events_v2' + AND name IN ('attributes', 'attributes_text') + ORDER BY name`, + schema: z.object({ + name: z.string(), + default_kind: z.string(), + default_expression: z.string(), + }), + }); + const [columnError, columns] = await readColumnKinds({}); + expect(columnError).toBeNull(); + expect(columns).toEqual([ + { + name: "attributes", + default_kind: "EPHEMERAL", + default_expression: "defaultValueOfTypeName('JSON')", + }, + { + name: "attributes_text", + default_kind: "MATERIALIZED", + default_expression: "toJSONString(attributes)", + }, + ]); + + const readRemovedIndexes = ch.reader.query({ + name: "read-removed-task-event-text-indexes", + query: `SELECT name + FROM system.data_skipping_indices + WHERE database = 'trigger_dev' + AND table = 'task_events_v2' + AND name IN ('idx_attributes_text_search', 'idx_message_text_search') + ORDER BY name`, + schema: z.object({ name: z.string() }), + }); + const [indexError, indexes] = await readRemovedIndexes({}); + expect(indexError).toBeNull(); + expect(indexes).toEqual([]); } ); });