feat: PR-6 add data processing for collation, export, and presets

src/collate.ts

@@ -0,0 +1,144 @@
+import type { Embed, Message } from "@/discord/schemas.ts";
+
+export type FlattenedEmbed = {
+  channelId: string;
+  embed: Embed;
+  messageAuthor: { id: string; username: string; bot?: boolean };
+  messageId: string;
+  messageTimestamp: string;
+};
+
+export type ExtractedFieldRow = {
+  channelId: string;
+  embedDescription: string | null;
+  embedTitle: string | null;
+  fields: Record<string, string>;
+  messageId: string;
+  messageTimestamp: string;
+};
+
+export type CollatedData = {
+  byAuthor: Record<string, { count: number; username: string; bot?: boolean }>;
+  byChannel: Record<string, number>;
+  embeds: FlattenedEmbed[];
+  embedsByAuthor: Record<string, number>;
+  embedsByDomain: Record<string, number>;
+  embedsByProvider: Record<string, number>;
+  embedsByType: Record<string, number>;
+  extractedFields: ExtractedFieldRow[];
+  messages: Message[];
+  totalEmbeds: number;
+  totalMessages: number;
+};
+
+const extractDomain = (url: string): string | null => {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return null;
+  }
+};
+
+const increment = (record: Record<string, number>, key: string): void => {
+  record[key] = (record[key] ?? 0) + 1;
+};
+
+const processEmbed = (embed: Embed, data: CollatedData): void => {
+  if (embed.type) {
+    increment(data.embedsByType, embed.type);
+  }
+  if (embed.provider?.name) {
+    increment(data.embedsByProvider, embed.provider.name);
+  }
+  if (embed.author?.name) {
+    increment(data.embedsByAuthor, embed.author.name);
+  }
+
+  if (embed.url) {
+    const domain = extractDomain(embed.url);
+    if (domain) {
+      increment(data.embedsByDomain, domain);
+    }
+  }
+};
+
+const extractFields = (
+  embed: Embed,
+  msg: Message,
+  data: CollatedData
+): void => {
+  if (!embed.fields || embed.fields.length === 0) {
+    return;
+  }
+
+  const fields: Record<string, string> = {};
+  for (const field of embed.fields) {
+    fields[field.name] = field.value;
+  }
+
+  data.extractedFields.push({
+    messageId: msg.id,
+    channelId: msg.channel_id,
+    messageTimestamp: msg.timestamp,
+    embedTitle: embed.title ?? null,
+    embedDescription: embed.description ?? null,
+    fields,
+  });
+};
+
+const processMessage = (msg: Message, data: CollatedData): void => {
+  increment(data.byChannel, msg.channel_id);
+
+  const authorEntry = data.byAuthor[msg.author.id];
+  if (authorEntry) {
+    authorEntry.count++;
+  } else {
+    data.byAuthor[msg.author.id] = {
+      count: 1,
+      username: msg.author.username,
+      bot: msg.author.bot,
+    };
+  }
+
+  const embeds = msg.embeds ?? [];
+  data.totalEmbeds += embeds.length;
+
+  for (const embed of embeds) {
+    data.embeds.push({
+      messageId: msg.id,
+      channelId: msg.channel_id,
+      messageTimestamp: msg.timestamp,
+      messageAuthor: {
+        id: msg.author.id,
+        username: msg.author.username,
+        bot: msg.author.bot,
+      },
+      embed,
+    });
+
+    processEmbed(embed, data);
+    extractFields(embed, msg, data);
+  }
+};
+
+export const collateResults = (messages: Message[]): CollatedData => {
+  const data: CollatedData = {
+    totalMessages: messages.length,
+    byChannel: {},
+    byAuthor: {},
+    totalEmbeds: 0,
+    embedsByType: {},
+    embedsByProvider: {},
+    embedsByDomain: {},
+    embedsByAuthor: {},
+    messages,
+    embeds: [],
+    extractedFields: [],
+  };
+
+  for (const msg of messages) {
+    processMessage(msg, data);
+  }
+
+  return data;
+};

src/errors.ts

@@ -32,3 +32,8 @@ export class ExportError extends TaggedError("ExportError")<{
   message: string;
   cause: unknown;
 }>() {}
+
+export class PresetError extends TaggedError("PresetError")<{
+  message: string;
+  cause: unknown;
+}>() {}

src/export.ts

@@ -0,0 +1,195 @@
+import { Result } from "better-result";
+import type { CollatedData } from "@/collate.ts";
+import { ExportError } from "@/errors.ts";
+
+const escapeCsvField = (value: string): string => {
+  if (
+    value.includes(",") ||
+    value.includes('"') ||
+    value.includes("\n") ||
+    value.includes("\r")
+  ) {
+    return `"${value.replace(/"/g, '""')}"`;
+  }
+  return value;
+};
+
+const toCsvRow = (fields: string[]): string =>
+  fields.map(escapeCsvField).join(",");
+
+export const exportJson = async (
+  data: CollatedData,
+  filePath: string
+): Promise<Result<void, ExportError>> => {
+  return await Result.tryPromise({
+    try: async () => {
+      const { messages: _messages, ...summary } = data;
+      const output = {
+        ...summary,
+        messages: data.messages.map((msg) => ({
+          id: msg.id,
+          channel_id: msg.channel_id,
+          author: msg.author,
+          content: msg.content,
+          timestamp: msg.timestamp,
+          embeds: msg.embeds,
+          attachments: msg.attachments,
+        })),
+      };
+      await Bun.write(filePath, JSON.stringify(output, null, 2));
+    },
+    catch: (cause) =>
+      new ExportError({
+        message: `Failed to export JSON: ${cause instanceof Error ? cause.message : String(cause)}`,
+        cause,
+      }),
+  });
+};
+
+export const exportMessagesCsv = async (
+  data: CollatedData,
+  filePath: string
+): Promise<Result<void, ExportError>> => {
+  return await Result.tryPromise({
+    try: async () => {
+      const headers = [
+        "message_id",
+        "channel_id",
+        "timestamp",
+        "author_id",
+        "author_username",
+        "author_is_bot",
+        "content",
+        "embed_count",
+        "attachment_count",
+      ];
+
+      const rows = [toCsvRow(headers)];
+
+      for (const msg of data.messages) {
+        rows.push(
+          toCsvRow([
+            msg.id,
+            msg.channel_id,
+            msg.timestamp,
+            msg.author.id,
+            msg.author.username,
+            String(msg.author.bot ?? false),
+            msg.content,
+            String(msg.embeds?.length ?? 0),
+            String(msg.attachments?.length ?? 0),
+          ])
+        );
+      }
+
+      await Bun.write(filePath, rows.join("\n"));
+    },
+    catch: (cause) =>
+      new ExportError({
+        message: `Failed to export messages CSV: ${cause instanceof Error ? cause.message : String(cause)}`,
+        cause,
+      }),
+  });
+};
+
+export const exportEmbedsCsv = async (
+  data: CollatedData,
+  filePath: string
+): Promise<Result<void, ExportError>> => {
+  return await Result.tryPromise({
+    try: async () => {
+      const headers = [
+        "message_id",
+        "channel_id",
+        "timestamp",
+        "author_id",
+        "author_username",
+        "embed_type",
+        "embed_title",
+        "embed_description",
+        "embed_url",
+        "embed_provider",
+        "embed_author",
+        "field_count",
+      ];
+
+      const rows = [toCsvRow(headers)];
+
+      for (const entry of data.embeds) {
+        rows.push(
+          toCsvRow([
+            entry.messageId,
+            entry.channelId,
+            entry.messageTimestamp,
+            entry.messageAuthor.id,
+            entry.messageAuthor.username,
+            entry.embed.type ?? "",
+            entry.embed.title ?? "",
+            entry.embed.description ?? "",
+            entry.embed.url ?? "",
+            entry.embed.provider?.name ?? "",
+            entry.embed.author?.name ?? "",
+            String(entry.embed.fields?.length ?? 0),
+          ])
+        );
+      }
+
+      await Bun.write(filePath, rows.join("\n"));
+    },
+    catch: (cause) =>
+      new ExportError({
+        message: `Failed to export embeds CSV: ${cause instanceof Error ? cause.message : String(cause)}`,
+        cause,
+      }),
+  });
+};
+
+export const exportFieldsCsv = async (
+  data: CollatedData,
+  filePath: string
+): Promise<Result<void, ExportError>> => {
+  return await Result.tryPromise({
+    try: async () => {
+      // Collect all unique field names across all extracted fields
+      const fieldNames = new Set<string>();
+      for (const row of data.extractedFields) {
+        for (const name of Object.keys(row.fields)) {
+          fieldNames.add(name);
+        }
+      }
+
+      const sortedFieldNames = [...fieldNames].sort();
+
+      const headers = [
+        "message_id",
+        "channel_id",
+        "timestamp",
+        "embed_title",
+        "embed_description",
+        ...sortedFieldNames,
+      ];
+
+      const rows = [toCsvRow(headers)];
+
+      for (const row of data.extractedFields) {
+        rows.push(
+          toCsvRow([
+            row.messageId,
+            row.channelId,
+            row.messageTimestamp,
+            row.embedTitle ?? "",
+            row.embedDescription ?? "",
+            ...sortedFieldNames.map((name) => row.fields[name] ?? ""),
+          ])
+        );
+      }
+
+      await Bun.write(filePath, rows.join("\n"));
+    },
+    catch: (cause) =>
+      new ExportError({
+        message: `Failed to export fields CSV: ${cause instanceof Error ? cause.message : String(cause)}`,
+        cause,
+      }),
+  });
+};

src/paths.ts

@@ -4,6 +4,7 @@ import { join } from "node:path";
 
 export const APP_DIR = join(homedir(), ".discord-search");
 export const SETTINGS_FILE = join(APP_DIR, "settings.json");
+export const PRESETS_FILE = join(APP_DIR, ".discord-search-presets.json");
 export const OUTPUT_DIR = join(APP_DIR, "output");
 
 const chmodSafe = async (path: string, mode: number): Promise<void> => {