integration: spider search & fetch for faster retrieval of sources

app/api/getAnswer/route.ts

@@ -5,6 +5,7 @@ import {
   TogetherAIStreamPayload,
 } from "@/utils/TogetherAIStream";
 import Together from "together-ai";
+import { Spider } from "@spider-cloud/spider-client";
 
 const together = new Together({
   apiKey: process.env["TOGETHER_API_KEY"],
@@ -23,22 +24,35 @@ export async function POST(request: Request) {
   let finalResults = await Promise.all(
     sources.map(async (result: any) => {
       try {
-        // Fetch the source URL, or abort if it's been 3 seconds
-        const response = await fetchWithTimeout(result.url);
-        const html = await response.text();
-        const virtualConsole = new jsdom.VirtualConsole();
-        const dom = new JSDOM(html, { virtualConsole });
-
-        const doc = dom.window.document;
-        const parsed = new Readability(doc).parse();
-        let parsedContent = parsed
-          ? cleanedText(parsed.textContent)
-          : "Nothing found";
-
-        return {
-          ...result,
-          fullContent: parsedContent,
-        };
+        if (sources[0].source === "spider") {
+          console.log("fetching with spider", result.url);
+          const spider = new Spider({
+            apiKey: process.env.SPIDER_API_KEY,
+          });
+          const spiderResult = await spider.scrapeUrl(result.url, { return_format: "markdown", metadata: false, readability: true });
+          return {
+            ...result,
+            fullContent: spiderResult[0].content.toString(),
+          };
+        } else {
+          console.log("fetching", result.url)
+          // Fetch the source URL, or abort if it's been 3 seconds
+          const response = await fetchWithTimeout(result.url);
+          const html = await response.text();
+          const virtualConsole = new jsdom.VirtualConsole();
+          const dom = new JSDOM(html, { virtualConsole });
+
+          const doc = dom.window.document;
+          const parsed = new Readability(doc).parse();
+          let parsedContent = parsed
+            ? cleanedText(parsed.textContent)
+            : "Nothing found";
+
+          return {
+            ...result,
+            fullContent: parsedContent,
+          };
+        }
       } catch (e) {
         console.log(`error parsing ${result.name}, error: ${e}`);
         return {

app/api/getSources/route.ts

@@ -1,9 +1,12 @@
 import { NextResponse } from "next/server";
 import { z } from "zod";
+import { Spider } from "@spider-cloud/spider-client";
 
 let excludedSites = ["youtube.com"];
 
-let searchEngine: "bing" | "serper" = "serper";
+type SearchEngine = "bing" | "serper" | "spider";
+
+let searchEngine: SearchEngine = "serper";
 
 export async function POST(request: Request) {
   let { question } = await request.json();
@@ -77,6 +80,22 @@ export async function POST(request: Request) {
       url: result.link,
     }));
 
+    return NextResponse.json(results);
+  } else if (searchEngine === "spider") {
+    const SPIDER_API_KEY = process.env["SPIDER_API_KEY"];
+    if (!SPIDER_API_KEY) {
+      throw new Error("SPIDER_API_KEY is required");
+    }
+
+    const app = new Spider({ apiKey: SPIDER_API_KEY });
+    const response = await app.search(question, { fetch_page_content: false, limit: 5 });
+
+    let results = response.content.map((result: { title: string, url: string }) => ({
+      name: result.title,
+      url: result.url,
+      source: "spider",
+    }));
+
     return NextResponse.json(results);
   }
 }

package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@mozilla/readability": "^0.5.0",
+    "@spider-cloud/spider-client": "^0.0.27",
     "eventsource-parser": "^1.1.2",
     "jsdom": "^24.1.0",
     "next": "14.2.3",