|
|
@@ -0,0 +1,139 @@
|
|
|
+// 网页抓取:两个引擎共用(经 MCP Gateway 提供)。
|
|
|
+// 在连接层校验目标 IP(undici 自定义 lookup),防止 DNS 重绑定绕过内网拦截;每一跳重定向都重新校验。
|
|
|
+import crypto from 'node:crypto'
|
|
|
+import dns from 'node:dns'
|
|
|
+import fs from 'node:fs'
|
|
|
+import path from 'node:path'
|
|
|
+import TurndownService from 'turndown'
|
|
|
+import { Agent, fetch as undiciFetch } from 'undici'
|
|
|
+import { isPrivateAddress } from '../lib/net'
|
|
|
+
|
|
|
+const MAX_BYTES = 3 * 1048576
|
|
|
+const MAX_TEXT = 40_000
|
|
|
+const MAX_REDIRECTS = 5
|
|
|
+const TIMEOUT_MS = 20_000
|
|
|
+
|
|
|
+export class FetchBlockedError extends Error {}
|
|
|
+
|
|
|
+/** 解析域名后拒绝私网地址的连接器 */
|
|
|
+function guardedAgent(blockPrivate: boolean) {
|
|
|
+ return new Agent({
|
|
|
+ connect: {
|
|
|
+ lookup: (hostname, options, callback) => {
|
|
|
+ dns.lookup(hostname, { ...options, all: true }, (err, addresses) => {
|
|
|
+ if (err) return callback(err, '', 4)
|
|
|
+ const list = addresses as dns.LookupAddress[]
|
|
|
+ const bad = blockPrivate ? list.find(a => isPrivateAddress(a.address)) : undefined
|
|
|
+ if (bad) return callback(new FetchBlockedError(`不允许访问内网地址:${hostname}(${bad.address})`), '', 4)
|
|
|
+ const first = list[0]
|
|
|
+ if (!first) return callback(new Error(`无法解析:${hostname}`), '', 4)
|
|
|
+ // undici 传入 all: true 时期望返回数组
|
|
|
+ if ((options as { all?: boolean }).all) (callback as unknown as (e: null, a: dns.LookupAddress[]) => void)(null, list)
|
|
|
+ else callback(null, first.address, first.family)
|
|
|
+ })
|
|
|
+ },
|
|
|
+ },
|
|
|
+ })
|
|
|
+}
|
|
|
+
|
|
|
+export interface FetchResult {
|
|
|
+ url: string
|
|
|
+ status: number
|
|
|
+ contentType: string
|
|
|
+ /** 返回给模型的文本(可能被截断) */
|
|
|
+ text: string
|
|
|
+ truncated: boolean
|
|
|
+ /** 完整内容或二进制文件保存在工作区中的相对路径 */
|
|
|
+ savedPath: string | null
|
|
|
+}
|
|
|
+
|
|
|
+export async function webFetch(opts: { url: string; workspaceDir: string; blockPrivate: boolean; signal?: AbortSignal }): Promise<FetchResult> {
|
|
|
+ let url = parseUrl(opts.url)
|
|
|
+ const agent = guardedAgent(opts.blockPrivate)
|
|
|
+ const timeout = AbortSignal.timeout(TIMEOUT_MS)
|
|
|
+ const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout
|
|
|
+ try {
|
|
|
+ for (let hop = 0; ; hop++) {
|
|
|
+ if (opts.blockPrivate && isPrivateAddress(url.hostname.replace(/^\[|\]$/g, ''))) {
|
|
|
+ throw new FetchBlockedError(`不允许访问内网地址:${url.hostname}`)
|
|
|
+ }
|
|
|
+ const res = await undiciFetch(url, {
|
|
|
+ dispatcher: agent, redirect: 'manual', signal,
|
|
|
+ headers: { 'user-agent': 'Mozilla/5.0 (compatible; AgentServer/0.1; +web_fetch)', accept: 'text/html,application/xhtml+xml,application/json,text/plain,*/*;q=0.5' },
|
|
|
+ })
|
|
|
+ if (res.status >= 300 && res.status < 400 && res.headers.get('location')) {
|
|
|
+ if (hop >= MAX_REDIRECTS) throw new Error('重定向次数过多')
|
|
|
+ url = parseUrl(new URL(res.headers.get('location')!, url).toString())
|
|
|
+ await res.body?.cancel()
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ const contentType = (res.headers.get('content-type') ?? 'application/octet-stream').split(';')[0]!.trim().toLowerCase()
|
|
|
+ const buf = await readLimited(res.body, MAX_BYTES)
|
|
|
+ return toResult(url.toString(), res.status, contentType, buf, opts.workspaceDir)
|
|
|
+ }
|
|
|
+ } catch (err) {
|
|
|
+ // undici 把 lookup 中的错误包在 cause 里
|
|
|
+ const cause = (err as { cause?: unknown }).cause
|
|
|
+ if (cause instanceof FetchBlockedError) throw cause
|
|
|
+ throw err
|
|
|
+ } finally {
|
|
|
+ await agent.close().catch(() => {})
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function parseUrl(raw: string): URL {
|
|
|
+ let u: URL
|
|
|
+ try {
|
|
|
+ u = new URL(raw)
|
|
|
+ } catch {
|
|
|
+ throw new Error(`无效的 URL:${raw}`)
|
|
|
+ }
|
|
|
+ if (!['http:', 'https:'].includes(u.protocol)) throw new Error(`只支持 http/https:${u.protocol}`)
|
|
|
+ return u
|
|
|
+}
|
|
|
+
|
|
|
+async function readLimited(body: ReadableStream<Uint8Array> | null, max: number): Promise<Buffer> {
|
|
|
+ if (!body) return Buffer.alloc(0)
|
|
|
+ const chunks: Buffer[] = []
|
|
|
+ let size = 0
|
|
|
+ for await (const chunk of body) {
|
|
|
+ size += chunk.byteLength
|
|
|
+ if (size > max) throw new Error(`内容超过 ${Math.round(max / 1048576)} MB 上限`)
|
|
|
+ chunks.push(Buffer.from(chunk))
|
|
|
+ }
|
|
|
+ return Buffer.concat(chunks)
|
|
|
+}
|
|
|
+
|
|
|
+function toResult(url: string, status: number, contentType: string, buf: Buffer, workspaceDir: string): FetchResult {
|
|
|
+ const dir = path.join(workspaceDir, '.agent', 'fetch')
|
|
|
+ const base = `${new Date().toISOString().replace(/[:.]/g, '-')}-${crypto.createHash('sha1').update(url).digest('hex').slice(0, 8)}`
|
|
|
+ const save = (ext: string, data: Buffer | string) => {
|
|
|
+ fs.mkdirSync(dir, { recursive: true })
|
|
|
+ const file = path.join(dir, `${base}.${ext}`)
|
|
|
+ fs.writeFileSync(file, data)
|
|
|
+ return path.relative(workspaceDir, file)
|
|
|
+ }
|
|
|
+
|
|
|
+ let text: string
|
|
|
+ if (contentType === 'text/html' || contentType === 'application/xhtml+xml') {
|
|
|
+ text = htmlToMarkdown(buf.toString('utf8'))
|
|
|
+ } else if (contentType.startsWith('text/') || contentType === 'application/json' || contentType.endsWith('+json') || contentType === 'application/xml') {
|
|
|
+ text = buf.toString('utf8')
|
|
|
+ } else {
|
|
|
+ // 二进制(PDF、图片、表格等)保存到工作区,由 Agent 用相应工具处理
|
|
|
+ const ext = { 'application/pdf': 'pdf', 'image/png': 'png', 'image/jpeg': 'jpg', 'text/csv': 'csv' }[contentType]
|
|
|
+ ?? (/spreadsheetml/.test(contentType) ? 'xlsx' : /wordprocessingml/.test(contentType) ? 'docx' : 'bin')
|
|
|
+ const saved = save(ext, buf)
|
|
|
+ return { url, status, contentType, text: `已下载 ${contentType}(${buf.length} 字节)到 ${saved}`, truncated: false, savedPath: saved }
|
|
|
+ }
|
|
|
+ if (text.length <= MAX_TEXT) return { url, status, contentType, text, truncated: false, savedPath: null }
|
|
|
+ const saved = save('md', text)
|
|
|
+ return { url, status, contentType, text: text.slice(0, MAX_TEXT), truncated: true, savedPath: saved }
|
|
|
+}
|
|
|
+
|
|
|
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' })
|
|
|
+turndown.remove(['script', 'style', 'noscript', 'iframe', 'svg', 'form'])
|
|
|
+
|
|
|
+export function htmlToMarkdown(html: string): string {
|
|
|
+ return turndown.turndown(html).replace(/\n{3,}/g, '\n\n').trim()
|
|
|
+}
|