mirror of
https://github.com/mendableai/firecrawl.git
synced 2024-11-16 19:58:08 +08:00
60 lines
1.5 KiB
YAML
60 lines
1.5 KiB
YAML
config:
|
|
target: "https://staging-firecrawl-scraper-js.fly.dev/v0"
|
|
phases:
|
|
- duration: 60
|
|
arrivalRate: 10
|
|
defaults:
|
|
headers:
|
|
Authorization: "Bearer fc-f3f77b6d5e084d2b970f595089cefa58"
|
|
scenarios:
|
|
- name: Scrape a URL
|
|
flow:
|
|
- post:
|
|
url: "/scrape"
|
|
json:
|
|
url: "https://www.scrapethissite.com"
|
|
pageOptions:
|
|
onlyMainContent: true
|
|
capture:
|
|
- json: "$.data.markdown"
|
|
as: markdown_content
|
|
|
|
# - name: Crawl a URL
|
|
# flow:
|
|
# - post:
|
|
# url: "/crawl"
|
|
# json:
|
|
# url: "https://mendable.ai"
|
|
# crawlerOptions:
|
|
# includes: []
|
|
# excludes: ["blog/*"]
|
|
# limit: 100
|
|
# pageOptions:
|
|
# onlyMainContent: true
|
|
# capture:
|
|
# - json: "$.jobId"
|
|
# as: job_id
|
|
# - get:
|
|
# url: "/crawl/status/{{ job_id }}"
|
|
# capture:
|
|
# - json: "$.status"
|
|
# as: crawl_status
|
|
# until:
|
|
# - condition: "equals"
|
|
# value: "completed"
|
|
# variable: "crawl_status"
|
|
# retry:
|
|
# count: 20
|
|
# wait: 10
|
|
|
|
# - name: Search for a query
|
|
# flow:
|
|
# - post:
|
|
# url: "/search"
|
|
# json:
|
|
# query: "firecrawl"
|
|
# pageOptions:
|
|
# fetchPageContent: true
|
|
# capture:
|
|
# - json: "$.data[0].markdown"
|
|
# as: search_markdown_content |