store crawled research results in a folder, log research topic/follow-up questions+learnings in output file

.gitignore

@@ -2,6 +2,7 @@
 
 # Output files
 output.md
+downloaded-urls
 
 # Dependencies
 node_modules

package.json

@@ -15,7 +15,7 @@
   "devDependencies": {
     "@ianvs/prettier-plugin-sort-imports": "^4.4.1",
     "@types/lodash-es": "^4.17.12",
-    "@types/node": "^22.13.0",
+    "@types/node": "^22.13.4",
     "prettier": "^3.4.2",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3"
@@ -27,6 +27,7 @@
     "js-tiktoken": "^1.0.17",
     "lodash-es": "^4.17.21",
     "p-limit": "^6.2.0",
+    "sanitize-filename": "^1.6.3",
     "zod": "^3.24.1"
   },
   "engines": {

src/deep-research.ts

@@ -3,6 +3,7 @@ import { generateObject } from 'ai';
 import { compact } from 'lodash-es';
 import pLimit from 'p-limit';
 import { z } from 'zod';
+import * as fs from 'fs/promises';
 
 import { o3MiniModel, trimPrompt } from './ai/providers';
 import { systemPrompt } from './prompt';
@@ -86,6 +87,12 @@ async function generateSerpQueries({
   return res.object.queries.slice(0, numQueries);
 }
 
+import sanitize from 'sanitize-filename';
+import path from 'path';
+export function urlToFilepath(url: string): string {
+  return path.join('downloaded-urls', `${sanitize(url, { replacement: '-' })}.md`);
+}
+
 async function processSerpResult({
   query,
   result,
@@ -97,6 +104,25 @@ async function processSerpResult({
   numLearnings?: number;
   numFollowUpQuestions?: number;
 }) {
+  // Create downloaded-urls directory if it doesn't exist
+  await fs.mkdir('downloaded-urls', { recursive: true });
+
+  // Save each document
+  for (const doc of result.data) {
+    if (doc.markdown && doc.url) {
+      const content = [
+        doc.title ? `Title: ${doc.title}` : '',
+        doc.description ? `Description: ${doc.description}` : '',
+        `URL: ${doc.url}`,
+        `Accessed at: ${Date()}`,
+        '',
+        doc.markdown
+      ].filter(Boolean).join('\n');
+
+      await fs.writeFile(urlToFilepath(doc.url), content, 'utf-8');
+    }
+  }
+
   const contents = compact(result.data.map(item => item.markdown)).map(
     content => trimPrompt(content, 25_000),
   );
@@ -156,7 +182,7 @@ export async function writeFinalReport({
   });
 
   // Append the visited URLs section to the report
-  const urlsSection = `\n\n## Sources\n\n${visitedUrls.map(url => `- ${url}`).join('\n')}`;
+  const urlsSection = `\n\n## Sources\n\n${visitedUrls.map(url => `- ${url}, saved at ${urlToFilepath(url)}`).join('\n')}`;
   return res.object.reportMarkdown + urlsSection;
 }
 

src/run.ts

@@ -30,7 +30,7 @@ function askQuestion(query: string): Promise<string> {
 async function run() {
   // Get initial query
   const initialQuery = await askQuestion('What would you like to research? ');
-
+  const outputFile = 'output.md';
   // Get breath and depth parameters
   const breadth =
     parseInt(
@@ -45,7 +45,10 @@ async function run() {
       10,
     ) || 2;
 
-  log(`Creating research plan...`);
+  const inittext = `Research starting at ${new Date().toISOString()} with breadth ${breadth} and depth ${depth}.`;
+
+  await fs.writeFile(outputFile, inittext, 'utf-8');
+  log(`${inittext}\nCreating research plan...`);
 
   // Generate follow-up questions
   const followUpQuestions = await generateFeedback({
@@ -69,6 +72,7 @@ Initial Query: ${initialQuery}
 Follow-up Questions and Answers:
 ${followUpQuestions.map((q: string, i: number) => `Q: ${q}\nA: ${answers[i]}`).join('\n')}
 `;
+  await fs.appendFile(outputFile, combinedQuery, 'utf-8');
 
   log('\nResearching your topic...');
 
@@ -83,23 +87,23 @@ ${followUpQuestions.map((q: string, i: number) => `Q: ${q}\nA: ${answers[i]}`).j
     },
   });
 
-  log(`\n\nLearnings:\n\n${learnings.join('\n')}`);
-  log(
-    `\n\nVisited URLs (${visitedUrls.length}):\n\n${visitedUrls.join('\n')}`,
-  );
+  const URLsandLearnings = `\n\nLearnings:\n\n${learnings.join('\n')}\n\nVisited URLs (${visitedUrls.length}):\n\n${visitedUrls.join('\n')}\n`;
+  await fs.appendFile(outputFile, URLsandLearnings, 'utf-8');
+
+  log(URLsandLearnings);
   log('Writing final report...');
 
-  const report = await writeFinalReport({
+  const report = `\n\nFinal Report:\n\n${await writeFinalReport({
     prompt: combinedQuery,
     learnings,
     visitedUrls,
-  });
+  })}` 
 
   // Save report to file
-  await fs.writeFile('output.md', report, 'utf-8');
+  await fs.appendFile(outputFile, report, 'utf-8');
 
-  console.log(`\n\nFinal Report:\n\n${report}`);
-  console.log('\nReport has been saved to output.md');
+  console.log(report);
+  console.log(`\nReport has been saved to ${outputFile}`);
   rl.close();
 }