Update Kubernetes YAMLs and add worker service

Refactored container configurations in worker, api, and playwright-service YAMLs to streamline syntax and add missing fields. Added a service definition for the worker component and included a new environment variable in the configmap for rate-limiting. These changes enhance configuration clarity and ensure proper resource definitions.

examples/kubernetes/cluster-install/api.yaml

@@ -15,37 +15,35 @@ spec:
       imagePullSecrets:
         - name: docker-registry-secret
       containers:
-      - name: api
-        image: ghcr.io/winkk-dev/firecrawl:latest
-        imagePullPolicy: Always
-        args:  [ "pnpm", "run", "start:production" ]
-        ports:
-        - containerPort: 3002
-        envFrom:
-        - configMapRef:
-            name: firecrawl-config
-        #- secretRef:
-        #    name: firecrawl-secret
-      - livenessProbe:
-          httpGet:
-            path: /v0/health/liveness
-            port: 3002
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-        name: api-container
-      - readinessProbe:
-          httpGet:
-            path: /v0/health/readiness
-            port: 3002
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-        name: api-container
+        - name: api
+          image: ghcr.io/winkk-dev/firecrawl:latest
+          imagePullPolicy: Always
+          args: [ "pnpm", "run", "start:production" ]
+          ports:
+            - containerPort: 3002
+          envFrom:
+            - configMapRef:
+                name: firecrawl-config
+          #- secretRef:
+          #    name: firecrawl-secret
+          livenessProbe:
+            httpGet:
+              path: /v0/health/liveness
+              port: 3002
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /v0/health/readiness
+              port: 3002
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
 ---
 apiVersion: v1
 kind: Service

examples/kubernetes/cluster-install/configmap.yaml

@@ -7,6 +7,7 @@ data:
   PORT: "3002"
   HOST: "0.0.0.0"
   REDIS_URL: "redis://redis:6379"
+  REDIS_RATE_LIMIT_URL: "redis://redis:6379"
   PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/html"
   USE_DB_AUTHENTICATION: "false"
   HDX_NODE_BETA_MODE: "1"

examples/kubernetes/cluster-install/playwright-service.yaml

@@ -22,34 +22,32 @@ spec:
       imagePullSecrets:
         - name: docker-registry-secret
       containers:
-      - name: playwright-service
-        image: ghcr.io/winkk-dev/firecrawl-playwright:latest
-        imagePullPolicy: Always
-        ports:
-        - containerPort: 3000
-        envFrom:
-        - configMapRef:
-            name: playwright-service-config
-      - livenessProbe:
-          httpGet:
-            path: /health/liveness
-            port: 3000
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-        name: playwright-service-container
-      - readinessProbe:
-          httpGet:
-            path: /health/readiness
-            port: 3000
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-        name: playwright-service-container
+        - name: playwright-service
+          image: ghcr.io/winkk-dev/firecrawl-playwright:latest
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 3000
+          envFrom:
+            - configMapRef:
+                name: playwright-service-config
+          livenessProbe:
+            httpGet:
+              path: /health/liveness
+              port: 3000
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /health/readiness
+              port: 3000
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
 ---
 apiVersion: v1
 kind: Service

examples/kubernetes/cluster-install/worker.yaml

@@ -15,31 +15,42 @@ spec:
       imagePullSecrets:
         - name: docker-registry-secret
       containers:
-      - name: worker
-        image: ghcr.io/winkk-dev/firecrawl:latest
-        imagePullPolicy: Always
-        envFrom:
-        - configMapRef:
-            name: firecrawl-config
-        #- secretRef:
-        #    name: firecrawl-secret
-      - livenessProbe:
-          httpGet:
-            path: /v0/health/liveness
-            port: 3002
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-        name: worker-container
-      - readinessProbe:
-          httpGet:
-            path: /v0/health/readiness
-            port: 3002
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-        name: worker-container
+        - name: worker
+          image: ghcr.io/winkk-dev/firecrawl:latest
+          imagePullPolicy: Always
+          args: [ "pnpm", "run", "workers" ]
+          envFrom:
+            - configMapRef:
+                name: firecrawl-config
+          #- secretRef:
+          #    name: firecrawl-secret
+          livenessProbe:
+            httpGet:
+              path: /v0/health/liveness
+              port: 3003
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /v0/health/readiness
+              port: 3003
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: worker
+spec:
+  selector:
+    app: worker
+  ports:
+    - protocol: TCP
+      port: 3003
+      targetPort: 3003