Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update builder to handle uploading sourcemaps

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-21 19:08:03 +02:00
parent 86942728e3
commit 920702cdde
4 changed files with 34 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/Dockerfile
View File

@ -12,8 +12,11 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-l
FROM base AS build
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
RUN apt-get update -qq && apt-get install -y ca-certificates && update-ca-certificates
RUN pnpm install
RUN pnpm run build
RUN --mount=type=secret,id=SENTRY_AUTH_TOKEN \
SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)" \
bash -c "if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi"
# Install packages needed for deployment

6
apps/api/package.json
View File

@ -5,11 +5,12 @@
"main": "index.js",
"scripts": {
"start": "nodemon --exec ts-node src/index.ts",
"start:production": "tsc && pnpm sentry:sourcemaps && node dist/src/index.js",
"start:production": "tsc && node dist/src/index.js",
"format": "prettier --write \"src/**/*.(js|ts)\"",
"flyio": "node dist/src/index.js",
"start:dev": "nodemon --exec ts-node src/index.ts",
"build": "tsc && pnpm sentry:sourcemaps",
"build:nosentry": "tsc",
"test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",
"test:local-no-auth": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_withAuth/*'",
"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'",
@ -19,7 +20,7 @@
"mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
"mongo-docker-console": "docker exec -it mongodb mongosh",
"run-example": "npx ts-node src/example.ts",
"deploy:fly": "flyctl deploy",
"deploy:fly": "flyctl deploy --build-secret SENTRY_AUTH_TOKEN=$(dotenv -p SENTRY_AUTH_TOKEN)",
"deploy:fly:staging": "fly deploy -c fly.staging.toml",
"sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist"
},
@ -73,6 +74,7 @@
"cron-parser": "^4.9.0",
"date-fns": "^3.6.0",
"dotenv": "^16.3.1",
"dotenv-cli": "^7.4.2",
"express-rate-limit": "^7.3.1",
"form-data": "^4.0.0",
"glob": "^10.4.2",

20
apps/api/pnpm-lock.yaml
View File

@ -86,6 +86,9 @@ importers:
dotenv:
specifier: ^16.3.1
version: 16.4.5
dotenv-cli:
specifier: ^7.4.2
version: 7.4.2
express-rate-limit:
specifier: ^7.3.1
version: 7.3.1(express@4.19.2)
@ -2367,6 +2370,14 @@ packages:
domutils@3.1.0:
resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==}
dotenv-cli@7.4.2:
resolution: {integrity: sha512-SbUj8l61zIbzyhIbg0FwPJq6+wjbzdn9oEtozQpZ6kW2ihCcapKVZj49oCT3oPM+mgQm+itgvUQcG5szxVrZTA==}
hasBin: true
dotenv-expand@10.0.0:
resolution: {integrity: sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==}
engines: {node: '>=12'}
dotenv@16.4.5:
resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}
engines: {node: '>=12'}
@ -7396,6 +7407,15 @@ snapshots:
domelementtype: 2.3.0
domhandler: 5.0.3
dotenv-cli@7.4.2:
dependencies:
cross-spawn: 7.0.3
dotenv: 16.4.5
dotenv-expand: 10.0.0
minimist: 1.2.8
dotenv-expand@10.0.0: {}
dotenv@16.4.5: {}
duck@0.1.12:

9
apps/api/src/services/queue-worker.ts
View File

@ -1,5 +1,5 @@
import "dotenv/config";
import "./services/sentry"
import "./sentry"
import * as Sentry from "@sentry/node";
import { CustomError } from "../lib/custom-error";
import {
@ -105,8 +105,11 @@ const workerFun = async (queueName: string, processJobInternal: (token: string,
const job = await worker.getNextJob(token);
if (job) {
Sentry.startSpan({
name: "Job " + job.id,
parentSpan: null,
name: "Scrape job",
op: "bullmq.job",
attributes: {
job: job.id,
},
}, async () => {
await processJobInternal(token, job);
});