diff --git a/app/api/getAnswer/route.ts b/app/api/getAnswer/route.ts index 3315094..d43a4a8 100644 --- a/app/api/getAnswer/route.ts +++ b/app/api/getAnswer/route.ts @@ -5,6 +5,7 @@ import { TogetherAIStreamPayload, } from "@/utils/TogetherAIStream"; import Together from "together-ai"; +import { Spider } from "@spider-cloud/spider-client"; const together = new Together({ apiKey: process.env["TOGETHER_API_KEY"], @@ -23,22 +24,35 @@ export async function POST(request: Request) { let finalResults = await Promise.all( sources.map(async (result: any) => { try { - // Fetch the source URL, or abort if it's been 3 seconds - const response = await fetchWithTimeout(result.url); - const html = await response.text(); - const virtualConsole = new jsdom.VirtualConsole(); - const dom = new JSDOM(html, { virtualConsole }); - - const doc = dom.window.document; - const parsed = new Readability(doc).parse(); - let parsedContent = parsed - ? cleanedText(parsed.textContent) - : "Nothing found"; - - return { - ...result, - fullContent: parsedContent, - }; + if (sources[0].source === "spider") { + console.log("fetching with spider", result.url); + const spider = new Spider({ + apiKey: process.env.SPIDER_API_KEY, + }); + const spiderResult = await spider.scrapeUrl(result.url, { return_format: "markdown", metadata: false, readability: true }); + return { + ...result, + fullContent: spiderResult[0].content.toString(), + }; + } else { + console.log("fetching", result.url) + // Fetch the source URL, or abort if it's been 3 seconds + const response = await fetchWithTimeout(result.url); + const html = await response.text(); + const virtualConsole = new jsdom.VirtualConsole(); + const dom = new JSDOM(html, { virtualConsole }); + + const doc = dom.window.document; + const parsed = new Readability(doc).parse(); + let parsedContent = parsed + ? cleanedText(parsed.textContent) + : "Nothing found"; + + return { + ...result, + fullContent: parsedContent, + }; + } } catch (e) { console.log(`error parsing ${result.name}, error: ${e}`); return { diff --git a/app/api/getSources/route.ts b/app/api/getSources/route.ts index a476ce6..62739b2 100644 --- a/app/api/getSources/route.ts +++ b/app/api/getSources/route.ts @@ -1,9 +1,12 @@ import { NextResponse } from "next/server"; import { z } from "zod"; +import { Spider } from "@spider-cloud/spider-client"; let excludedSites = ["youtube.com"]; -let searchEngine: "bing" | "serper" = "serper"; +type SearchEngine = "bing" | "serper" | "spider"; + +let searchEngine: SearchEngine = "serper"; export async function POST(request: Request) { let { question } = await request.json(); @@ -77,6 +80,22 @@ export async function POST(request: Request) { url: result.link, })); + return NextResponse.json(results); + } else if (searchEngine === "spider") { + const SPIDER_API_KEY = process.env["SPIDER_API_KEY"]; + if (!SPIDER_API_KEY) { + throw new Error("SPIDER_API_KEY is required"); + } + + const app = new Spider({ apiKey: SPIDER_API_KEY }); + const response = await app.search(question, { fetch_page_content: false, limit: 5 }); + + let results = response.content.map((result: { title: string, url: string }) => ({ + name: result.title, + url: result.url, + source: "spider", + })); + return NextResponse.json(results); } } diff --git a/package-lock.json b/package-lock.json index c7eaa71..eeb36ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@mozilla/readability": "^0.5.0", + "@spider-cloud/spider-client": "^0.0.27", "eventsource-parser": "^1.1.2", "jsdom": "^24.1.0", "next": "14.2.3", @@ -447,6 +448,11 @@ "integrity": "sha512-qC/xYId4NMebE6w/V33Fh9gWxLgURiNYgVNObbJl2LZv0GUUItCcCqC5axQSwRaAgaxl2mELq1rMzlswaQ0Zxg==", "dev": true }, + "node_modules/@spider-cloud/spider-client": { + "version": "0.0.27", + "resolved": "https://registry.npmjs.org/@spider-cloud/spider-client/-/spider-client-0.0.27.tgz", + "integrity": "sha512-pNOwe4cC8uhTHE+ToUkgYLsl0+9jo0PgBYELZkk5n4ld/fKjNMR297ZtAB/ylxeZqXIrvYb0LtI/dmlMFrSdDA==" + }, "node_modules/@swc/counter": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", diff --git a/package.json b/package.json index b099179..e3ca442 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@mozilla/readability": "^0.5.0", + "@spider-cloud/spider-client": "^0.0.27", "eventsource-parser": "^1.1.2", "jsdom": "^24.1.0", "next": "14.2.3",