Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Feat] Implemented retry attempts to handle 502 errors

Browse Source
This commit is contained in:
rafaelsideguide 2024-04-25 10:51:01 -03:00
parent abf6918642
commit d3ab2ea926
1 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -1,5 +1,6 @@
import os
import requests
import time
class FirecrawlApp:
def __init__(self, api_key=None):
@ -62,11 +63,23 @@ class FirecrawlApp:
'Authorization': f'Bearer {self.api_key}'
}
def _post_request(self, url, data, headers):
return requests.post(url, headers=headers, json=data)
def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5):
for attempt in range(retries):
response = requests.post(url, headers=headers, json=data)
if response.status_code == 502:
time.sleep(backoff_factor * (2 ** attempt))
else:
return response
return response
def _get_request(self, url, headers):
return requests.get(url, headers=headers)
def _get_request(self, url, headers, retries=3, backoff_factor=0.5):
for attempt in range(retries):
response = requests.get(url, headers=headers)
if response.status_code == 502:
time.sleep(backoff_factor * (2 ** attempt))
else:
return response
return response
def _monitor_job_status(self, job_id, headers, timeout):
import time