mirror of
https://github.com/mendableai/firecrawl.git
synced 2024-11-16 19:58:08 +08:00
67 lines
1.6 KiB
YAML
67 lines
1.6 KiB
YAML
config:
|
|
target: "https://staging-firecrawl-scraper-js.fly.dev/v0"
|
|
http:
|
|
timeout: 30
|
|
phases:
|
|
- duration: 60
|
|
arrivalRate: 1 # Initial load
|
|
- duration: 120
|
|
arrivalRate: 2 # Increased load
|
|
- duration: 180
|
|
arrivalRate: 3 # Peak load
|
|
- duration: 60
|
|
arrivalRate: 1 # Cool down
|
|
defaults:
|
|
headers:
|
|
Authorization: "Bearer YOUR_API_KEY"
|
|
scenarios:
|
|
# - name: Scrape a URL
|
|
# flow:
|
|
# - post:
|
|
# url: "/scrape"
|
|
# json:
|
|
# url: "https://www.scrapethissite.com"
|
|
# pageOptions:
|
|
# onlyMainContent: true
|
|
# capture:
|
|
# - json: "$.data.markdown"
|
|
# as: markdown_content
|
|
|
|
- name: Crawl a URL
|
|
flow:
|
|
- post:
|
|
url: "/crawl"
|
|
json:
|
|
url: "https://rsseau.fr"
|
|
crawlerOptions:
|
|
limit: 100
|
|
pageOptions:
|
|
onlyMainContent: true
|
|
capture:
|
|
- json: "$.jobId"
|
|
as: job_id
|
|
- think: 10
|
|
- get:
|
|
url: "/crawl/status/{{ job_id }}"
|
|
capture:
|
|
- json: "$.status"
|
|
as: crawl_status
|
|
until:
|
|
- condition: "equals"
|
|
value: "completed"
|
|
variable: "crawl_status"
|
|
retry:
|
|
count: 20
|
|
wait: 10
|
|
|
|
# - name: Search for a query
|
|
# flow:
|
|
# - post:
|
|
# url: "/search"
|
|
# json:
|
|
# query: "firecrawl"
|
|
# pageOptions:
|
|
# fetchPageContent: true
|
|
# capture:
|
|
# - json: "$.data[0].markdown"
|
|
# as: search_markdown_content |