diff --git a/apps/api/src/controllers/liveness.ts b/apps/api/src/controllers/liveness.ts new file mode 100644 index 00000000..8ff1a96f --- /dev/null +++ b/apps/api/src/controllers/liveness.ts @@ -0,0 +1,6 @@ +import { Request, Response } from "express"; + +export async function livenessController(req: Request, res: Response) { + //TODO: add checks if the application is live and healthy like checking the redis connection + res.status(200).json({ status: "ok" }); +} diff --git a/apps/api/src/controllers/readiness.ts b/apps/api/src/controllers/readiness.ts new file mode 100644 index 00000000..cdb1f02c --- /dev/null +++ b/apps/api/src/controllers/readiness.ts @@ -0,0 +1,6 @@ +import { Request, Response } from "express"; + +export async function readinessController(req: Request, res: Response) { + // TODO: add checks when the application is ready to serve traffic + res.status(200).json({ status: "ok" }); +} diff --git a/apps/api/src/routes/v0.ts b/apps/api/src/routes/v0.ts index a9a3a9bf..9c68d9bb 100644 --- a/apps/api/src/routes/v0.ts +++ b/apps/api/src/routes/v0.ts @@ -7,6 +7,8 @@ import { crawlJobStatusPreviewController } from "../../src/controllers/status"; import { searchController } from "../../src/controllers/search"; import { crawlCancelController } from "../../src/controllers/crawl-cancel"; import { keyAuthController } from "../../src/controllers/keyAuth"; +import { livenessController } from "../controllers/liveness"; +import { readinessController } from "../controllers/readiness"; export const v0Router = express.Router(); @@ -23,3 +25,6 @@ v0Router.get("/v0/keyAuth", keyAuthController); // Search routes v0Router.post("/v0/search", searchController); +// Health/Probe routes +v0Router.get("/v0/health/liveness", livenessController); +v0Router.get("/v0/health/readiness", readinessController); diff --git a/apps/playwright-service/main.py b/apps/playwright-service/main.py index bd6b14e3..c9099d3b 100644 --- a/apps/playwright-service/main.py +++ b/apps/playwright-service/main.py @@ -5,7 +5,7 @@ the HTML content of a specified URL. It supports optional proxy settings and med from os import environ -from fastapi import FastAPI +from fastapi import FastAPI, Response from fastapi.responses import JSONResponse from playwright.async_api import Browser, async_playwright from pydantic import BaseModel @@ -39,14 +39,28 @@ async def shutdown_event(): """Event handler for application shutdown to close the browser.""" await browser.close() +@app.get("/health/liveness") +def liveness_probe(): + """Endpoint for liveness probe.""" + return JSONResponse(content={"status": "ok"}, status_code=200) + + +@app.get("/health/readiness") +async def readiness_probe(): + """Endpoint for readiness probe. Checks if the browser instance is ready.""" + if browser: + return JSONResponse(content={"status": "ok"}, status_code=200) + return JSONResponse(content={"status": "Service Unavailable"}, status_code=503) + + @app.post("/html") async def root(body: UrlModel): """ Endpoint to fetch and return HTML content of a given URL. - + Args: body (UrlModel): The URL model containing the target URL, wait time, and timeout. - + Returns: JSONResponse: The HTML content of the page. """ diff --git a/examples/kubernetes/cluster-install/README.md b/examples/kubernetes/cluster-install/README.md index f874d829..736ae038 100644 --- a/examples/kubernetes/cluster-install/README.md +++ b/examples/kubernetes/cluster-install/README.md @@ -4,12 +4,12 @@ 2. Build Docker images, and host it in your Docker Registry (replace the target registry with your own) 1. API (which is also used as a worker image) 1. ```bash - docker build -t ghcr.io/winkk-dev/firecrawl:latest ../../apps/api + docker build --no-cache -t ghcr.io/winkk-dev/firecrawl:latest ../../../apps/api docker push ghcr.io/winkk-dev/firecrawl:latest ``` 2. Playwright 1. ```bash - docker build -t ghcr.io/winkk-dev/firecrawl-playwright:latest ../../apps/playwright-service + docker build --no-cache -t ghcr.io/winkk-dev/firecrawl-playwright:latest ../../../apps/playwright-service docker push ghcr.io/winkk-dev/firecrawl-playwright:latest ``` 3. Replace the image in [worker.yaml](worker.yaml), [api.yaml](api.yaml) and [playwright-service.yaml](playwright-service.yaml) diff --git a/examples/kubernetes/cluster-install/api.yaml b/examples/kubernetes/cluster-install/api.yaml index cdc69c3d..54ecfbf6 100644 --- a/examples/kubernetes/cluster-install/api.yaml +++ b/examples/kubernetes/cluster-install/api.yaml @@ -15,16 +15,35 @@ spec: imagePullSecrets: - name: docker-registry-secret containers: - - name: api - image: ghcr.io/winkk-dev/firecrawl:latest - args: [ "pnpm", "run", "start:production" ] - ports: - - containerPort: 3002 - envFrom: - - configMapRef: - name: firecrawl-config - - secretRef: - name: firecrawl-secret + - name: api + image: ghcr.io/winkk-dev/firecrawl:latest + imagePullPolicy: Always + args: [ "pnpm", "run", "start:production" ] + ports: + - containerPort: 3002 + envFrom: + - configMapRef: + name: firecrawl-config + #- secretRef: + # name: firecrawl-secret + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 --- apiVersion: v1 kind: Service diff --git a/examples/kubernetes/cluster-install/configmap.yaml b/examples/kubernetes/cluster-install/configmap.yaml index b415d562..b56cfbcd 100644 --- a/examples/kubernetes/cluster-install/configmap.yaml +++ b/examples/kubernetes/cluster-install/configmap.yaml @@ -7,8 +7,7 @@ data: PORT: "3002" HOST: "0.0.0.0" REDIS_URL: "redis://redis:6379" - PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000" + REDIS_RATE_LIMIT_URL: "redis://redis:6379" + PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/html" USE_DB_AUTHENTICATION: "false" - SUPABASE_ANON_TOKEN: "" - SUPABASE_URL: "" - SUPABASE_SERVICE_TOKEN: "" + HDX_NODE_BETA_MODE: "1" diff --git a/examples/kubernetes/cluster-install/playwright-service.yaml b/examples/kubernetes/cluster-install/playwright-service.yaml index ce794253..43cf15f0 100644 --- a/examples/kubernetes/cluster-install/playwright-service.yaml +++ b/examples/kubernetes/cluster-install/playwright-service.yaml @@ -1,3 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: playwright-service-config +data: + PORT: "3000" +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -15,13 +22,32 @@ spec: imagePullSecrets: - name: docker-registry-secret containers: - - name: playwright-service - image: ghcr.io/winkk-dev/firecrawl-playwright:latest - ports: - - containerPort: 3000 - envFrom: - - configMapRef: - name: firecrawl-config + - name: playwright-service + image: ghcr.io/winkk-dev/firecrawl-playwright:latest + imagePullPolicy: Always + ports: + - containerPort: 3000 + envFrom: + - configMapRef: + name: playwright-service-config + livenessProbe: + httpGet: + path: /health/liveness + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health/readiness + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 --- apiVersion: v1 kind: Service diff --git a/examples/kubernetes/cluster-install/worker.yaml b/examples/kubernetes/cluster-install/worker.yaml index 2b3b2e79..8e992cf1 100644 --- a/examples/kubernetes/cluster-install/worker.yaml +++ b/examples/kubernetes/cluster-install/worker.yaml @@ -15,10 +15,12 @@ spec: imagePullSecrets: - name: docker-registry-secret containers: - - name: worker - image: ghcr.io/winkk-dev/firecrawl:latest - envFrom: - - configMapRef: - name: firecrawl-config - - secretRef: - name: firecrawl-secret + - name: worker + image: ghcr.io/winkk-dev/firecrawl:latest + imagePullPolicy: Always + args: [ "pnpm", "run", "workers" ] + envFrom: + - configMapRef: + name: firecrawl-config + #- secretRef: + # name: firecrawl-secret