Firecrawl基于AI的链接内容提取工具

{"success": true,"data": {"content": "Raw Content","metadata": {"title": "Mendable","description": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide","robots": "follow, index","ogTitle": "Mendable","ogDescription": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide","ogUrl": "https://mendable.ai/","ogImage": "https://mendable.ai/mendable_new_og1.png","ogLocaleAlternate": [],"ogSiteName": "Mendable","sourceURL": "https://mendable.ai/"},"llm_extraction": {"company_mission": "Train a secure AI on your technical resources that answers customer and employee questions so your team doesn't have to","supports_sso": true,"is_open_source": false,"is_in_yc": true}}}

pip install firecrawl-py

from firecrawl import FirecrawlApp
app = FirecrawlApp(api_key="YOUR_API_KEY")
crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}})for result in crawl_result:print(result['markdown'])

url = 'https://example.com'scraped_data = app.scrape_url(url)

from pydantic import BaseModel, Fieldfrom typing import List
class ArticleSchema(BaseModel):title: strpoints: intby: strcommentsURL: str
class TopArticlesSchema(BaseModel):top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories")
data = app.scrape_url('https://news.ycombinator.com', {'extractorOptions': {'extractionSchema': TopArticlesSchema.schema_json(),'mode': 'llm-extraction'},'pageOptions':{'onlyMainContent': True}})print(data["llm_extraction"])

query = 'What is Mendable?'search_result = app.search(query)

npm install @mendable/firecrawl-js

try {const url = 'https://example.com';const scrapedData = await app.scrapeUrl(url);console.log(scrapedData);} catch (error) {console.error('Error occurred while scraping:',error.message);}

const crawlUrl = 'https://example.com';const params = {crawlerOptions: {excludes: ['blog/'],includes: [], // 留空表示所有页面limit: 1000,},pageOptions: {onlyMainContent: true}};const waitUntilDone = true;const timeout = 5;const crawlResult = await app.crawlUrl(crawlUrl,params,waitUntilDone,timeout);

const status = await app.checkCrawlStatus(jobId);console.log(status);

import FirecrawlApp from "@mendable/firecrawl-js";import { z } from "zod";
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY",});
// 定义架构以提取内容const schema = z.object({top: z.array(z.object({title: z.string(),points: z.number(),by: z.string(),commentsURL: z.string(),})).length(5).describe("Top 5 stories on Hacker News"),});
const scrapeResult = await app.scrapeUrl("https://news.ycombinator.com", {extractorOptions: { extractionSchema: schema },});
console.log(scrapeResult.data["llm_extraction"]);

const query = 'what is mendable?';const searchResults = await app.search(query, {pageOptions: {fetchPageContent: true // 获取每个搜索结果的页面内容}});