之前自定义上游与 Exa 都写死 500 字,换上游就等于悄悄丢内容。现在: - 默认原样返回整段 snippet(只压平空白) - 传 maxSnippetLength: N 才截断到 N 字并补省略号;0 / 负数 / 非数字一律视为不传 - 三路上游(自定义 /search、Parallel、Exa)统一按同一参数处理 - 上游报错时的响应片段仍固定截 200 字(那是诊断信息,不参与该参数)
245 lines
9.4 KiB
JavaScript
245 lines
9.4 KiB
JavaScript
/**
|
||
* 零依赖搜索核心:三路上游,统一归一化成 `{ url, title?, snippet?, publishedAt? }[]`。
|
||
*
|
||
* - 自定义上游(`X-Web-Search-Base-Url` + `Authorization`):任意 `POST {base}/search`,
|
||
* 请求体 `{ query, max_results }`、响应 `{ results: [{ title, url, snippet }] }`。
|
||
* - Parallel(免费匿名 MCP 端点,默认后端)。
|
||
* - Exa(免费 MCP 端点,Parallel 挂掉时的兜底)。
|
||
*
|
||
* 只用 Node 内置 fetch,不依赖任何 npm 包。免费侧两个后端都是公开的 MCP 端点,
|
||
* 本模块以 JSON-RPC `tools/call` 的方式直接调用它们。
|
||
*
|
||
* 上游实现参考:@sheep233_/dsh-web-search-free(github.com/sheep-programmer/dsh-web-search-free)。
|
||
*/
|
||
|
||
/** Parallel Web Search 免费 MCP 端点(匿名可用,OpenCode 默认后端之一)。 */
|
||
export const PARALLEL_MCP_URL = 'https://search.parallel.ai/mcp';
|
||
/** Exa 免费 MCP 端点(匿名可用;EXA_API_KEY 可提升限额)。 */
|
||
export const EXA_MCP_URL = 'https://mcp.exa.ai/mcp';
|
||
|
||
const USER_AGENT = 'web_search/1.1.0';
|
||
/** 单次上游搜索请求的超时,避免远端挂起拖死 MCP 请求。 */
|
||
const REQUEST_TIMEOUT_MS = Number(process.env.SEARCH_TIMEOUT_MS ?? 30_000);
|
||
/**
|
||
* snippet 一律截断:搜索结果会进历史,之后每一轮都可能被整条重放。
|
||
* 有些上游(如自建 v1/search)的 snippet 是整段正文,不截等于白烧 token。
|
||
*/
|
||
/**
|
||
* snippet 一律压平空白;是否截断由调用方传 maxSnippetLength 决定,缺省不截断。
|
||
* 为什么默认不截:上限该由调用方按自己的上下文预算定,写死在服务里会让「换上游 = 悄悄丢内容」。
|
||
*/
|
||
export function capSnippet(text, max) {
|
||
const s = String(text == null ? '' : text).replace(/\s+/g, ' ').trim();
|
||
if (!Number.isFinite(max) || max <= 0 || s.length <= max) return s;
|
||
return s.slice(0, max) + '…';
|
||
}
|
||
|
||
/**
|
||
* 向免费 MCP 端点发一次 `tools/call`,返回工具结果里的 text 载荷。
|
||
* 响应可能是裸 JSON,也可能是 SSE(`data: {...}`)分帧,两种都兼容。
|
||
*/
|
||
async function callMcpTool(endpoint, toolName, toolArgs, signal) {
|
||
const response = await fetch(endpoint, {
|
||
method: 'POST',
|
||
redirect: 'error',
|
||
headers: {
|
||
'content-type': 'application/json',
|
||
'accept': 'application/json, text/event-stream',
|
||
'user-agent': USER_AGENT,
|
||
},
|
||
body: JSON.stringify({
|
||
jsonrpc: '2.0',
|
||
id: 1,
|
||
method: 'tools/call',
|
||
params: { name: toolName, arguments: toolArgs },
|
||
}),
|
||
...(signal !== undefined ? { signal } : {}),
|
||
});
|
||
if (!response.ok) {
|
||
throw new Error(`${toolName} API error (HTTP ${response.status})`);
|
||
}
|
||
const text = extractToolText(await response.text());
|
||
if (text === undefined) {
|
||
throw new Error(`${toolName} returned no tool result`);
|
||
}
|
||
return text;
|
||
}
|
||
|
||
/** 从裸 JSON 或 SSE 文本中提取 MCP `tools/call` 的 result.content[].text。 */
|
||
export function extractToolText(body) {
|
||
const parsePayload = (raw) => {
|
||
try {
|
||
const data = JSON.parse(raw);
|
||
const item = data.result?.content?.find((c) => c.type === 'text' && typeof c.text === 'string');
|
||
return item?.text || undefined;
|
||
} catch {
|
||
return undefined; // 不是 JSON 载荷,交给 SSE 行解析
|
||
}
|
||
};
|
||
|
||
const trimmed = body.trim();
|
||
if (trimmed.length > 0) {
|
||
const direct = parsePayload(trimmed);
|
||
if (direct !== undefined) return direct;
|
||
}
|
||
for (const line of body.split('\n')) {
|
||
if (!line.startsWith('data: ')) continue;
|
||
const data = parsePayload(line.slice(6));
|
||
if (data !== undefined) return data;
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/** Parallel 响应解析:text 是内嵌 JSON `{ results: [...] }`。 */
|
||
export function mapParallelResponse(text, maxSnippetLength) {
|
||
let payload;
|
||
try {
|
||
payload = JSON.parse(text);
|
||
} catch (error) {
|
||
throw new Error(`Parallel search returned an unprocessable payload: ${String(error)}`);
|
||
}
|
||
const sources = [];
|
||
for (const item of payload.results ?? []) {
|
||
if (!item.url) continue;
|
||
const snippet = item.excerpts?.length > 0 ? capSnippet(item.excerpts.join(' '), maxSnippetLength) : '';
|
||
sources.push({
|
||
url: item.url,
|
||
...(item.title ? { title: item.title } : {}),
|
||
...(snippet ? { snippet } : {}),
|
||
...(item.publish_date ? { publishedAt: item.publish_date } : {}),
|
||
});
|
||
}
|
||
return sources;
|
||
}
|
||
|
||
/**
|
||
* Exa 响应解析:text 是纯文本块,每条结果以 `Title: ` 开头:
|
||
* ```
|
||
* Title: <标题>
|
||
* URL: <链接>
|
||
* Published: <日期 | N/A>
|
||
* Author: <作者 | N/A>
|
||
* Highlights:
|
||
* <正文摘要>
|
||
* ```
|
||
*/
|
||
export function parseExaText(text, maxSnippetLength) {
|
||
const blocks = text.split(/(?=^Title: )/m).filter((block) => block.trim().length > 0);
|
||
const sources = [];
|
||
for (const block of blocks) {
|
||
const url = block.match(/^URL: (\S+)$/m)?.[1];
|
||
if (url === undefined) continue;
|
||
const title = block.match(/^Title: (.+)$/m)?.[1]?.trim();
|
||
const published = block.match(/^Published: (.+)$/m)?.[1]?.trim();
|
||
const highlights = block.split(/^Highlights:\s*$/m)[1]?.trim();
|
||
const snippet = capSnippet(highlights, maxSnippetLength);
|
||
sources.push({
|
||
url,
|
||
...(title ? { title } : {}),
|
||
...(snippet ? { snippet } : {}),
|
||
...(published && published !== 'N/A' ? { publishedAt: published } : {}),
|
||
});
|
||
}
|
||
return sources;
|
||
}
|
||
|
||
/**
|
||
* 自定义上游:`POST {base}/search`,body `{ query, max_results }`。
|
||
* 配置了地址就以它为准——失败直接抛错,不静默退回免费后端(否则 key 过期/base 写错
|
||
* 只会表现为「结果质量突然变差」,永远没人发现)。
|
||
* @param {{ auth: string, base: string }} up 由 mcp.js 从请求 header 解析(容器不存密钥)
|
||
* @param {number} [maxSnippetLength] 调用方要求的最大 snippet 长度;缺省不截断
|
||
*/
|
||
export async function customSearch(query, maxResults, up, maxSnippetLength) {
|
||
const url = up.base.replace(/\/+$/, '') + '/search';
|
||
const headers = {
|
||
'content-type': 'application/json',
|
||
'accept': 'application/json',
|
||
'user-agent': USER_AGENT,
|
||
};
|
||
// 免鉴权的自建服务可省;有值才带,避免给上游塞一个空 Authorization 反而被拒
|
||
if (up.auth) headers.authorization = up.auth;
|
||
let response;
|
||
try {
|
||
response = await fetch(url, {
|
||
method: 'POST',
|
||
// 不跟随跳转:Authorization 不能被顺手带到别的地址去
|
||
redirect: 'error',
|
||
headers,
|
||
body: JSON.stringify({ query, max_results: maxResults }),
|
||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
||
});
|
||
} catch (error) {
|
||
throw new Error(`搜索上游请求失败(${url}):${String(error && error.message || error)}`);
|
||
}
|
||
const raw = await response.text();
|
||
if (!response.ok) {
|
||
// 带上游响应片段:401 和 500 的处置完全不同,只说「请求失败」没法排查
|
||
const detail = capSnippet(raw, 200);
|
||
throw new Error(`搜索上游 HTTP ${response.status}${detail ? ':' + detail : ''}`);
|
||
}
|
||
let payload;
|
||
try {
|
||
payload = JSON.parse(raw);
|
||
} catch (error) {
|
||
throw new Error(`搜索上游返回了无法解析的响应:${String(error)}`);
|
||
}
|
||
const sources = [];
|
||
for (const item of payload.results ?? []) {
|
||
if (!item || !item.url) continue;
|
||
const snippet = capSnippet(item.snippet, maxSnippetLength);
|
||
const publishedAt = item.published_at ?? item.publishedAt ?? item.publish_date;
|
||
sources.push({
|
||
url: String(item.url),
|
||
...(item.title ? { title: String(item.title) } : {}),
|
||
...(snippet ? { snippet } : {}),
|
||
...(publishedAt ? { publishedAt: String(publishedAt) } : {}),
|
||
});
|
||
}
|
||
return maxResults !== undefined ? sources.slice(0, maxResults) : sources;
|
||
}
|
||
|
||
/** Parallel 搜索(默认免费后端)。 */
|
||
export async function parallelSearch(query, maxResults, maxSnippetLength) {
|
||
const text = await callMcpTool(PARALLEL_MCP_URL, 'web_search', {
|
||
objective: query,
|
||
search_queries: [query],
|
||
}, AbortSignal.timeout(REQUEST_TIMEOUT_MS));
|
||
const sources = mapParallelResponse(text, maxSnippetLength);
|
||
return maxResults !== undefined ? sources.slice(0, maxResults) : sources;
|
||
}
|
||
|
||
/**
|
||
* 免费侧入口:Parallel 优先,失败自动退 Exa。
|
||
* 降级放在服务端(而不是让模型显式传 provider)——模型看不到后端,也就不会选错。
|
||
* @param {string} [exaApiKey] 可选,提升 Exa 匿名限额
|
||
* @param {number} [maxSnippetLength] 缺省不截断
|
||
*/
|
||
export async function freeSearch(query, maxResults, exaApiKey, maxSnippetLength) {
|
||
try {
|
||
return await parallelSearch(query, maxResults, maxSnippetLength);
|
||
} catch (first) {
|
||
try {
|
||
return await exaSearch(query, maxResults, exaApiKey, maxSnippetLength);
|
||
} catch (second) {
|
||
throw new Error(`免费搜索后端均不可用(parallel: ${String(first && first.message || first)};` +
|
||
`exa: ${String(second && second.message || second)})`);
|
||
}
|
||
}
|
||
}
|
||
|
||
/** Exa 搜索(备用免费后端;apiKey 可选,用于提升匿名限额)。 */
|
||
export async function exaSearch(query, maxResults, apiKey, maxSnippetLength) {
|
||
const endpoint = apiKey
|
||
? `${EXA_MCP_URL}?exaApiKey=${encodeURIComponent(apiKey)}`
|
||
: EXA_MCP_URL;
|
||
const text = await callMcpTool(endpoint, 'web_search_exa', {
|
||
query,
|
||
type: 'auto',
|
||
numResults: maxResults ?? 8,
|
||
livecrawl: 'fallback',
|
||
}, AbortSignal.timeout(REQUEST_TIMEOUT_MS));
|
||
const sources = parseExaText(text, maxSnippetLength);
|
||
return maxResults !== undefined ? sources.slice(0, maxResults) : sources;
|
||
}
|