From 70d50b3640812efceb6e3494b675a333c20ae5ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 20 Aug 2024 19:25:19 +0200
Subject: [PATCH 1/2] fix(queue-worker): move dotenv config up

---
 apps/api/src/services/queue-worker.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index cd4e03a1..afd80f42 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -1,10 +1,10 @@
+import "dotenv/config";
 import { CustomError } from "../lib/custom-error";
 import {
   getScrapeQueue,
   redisConnection,
   scrapeQueueName,
 } from "./queue-service";
-import "dotenv/config";
 import { logtail } from "./logtail";
 import { startWebScraperPipeline } from "../main/runWebScraper";
 import { callWebhook } from "./webhook";

From cf32893c2e3272f0ce737bc7e23b06c5bb1d2142 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 20 Aug 2024 19:31:26 +0200
Subject: [PATCH 2/2] add strict enforcement + move crawlerOptions to top-level
 in /crawl

---
 apps/api/src/controllers/v1/types.ts | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 3fe35891..afb8ab18 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -33,6 +33,8 @@ const url = z.preprocess(
     )
 );
 
+const strictMessage = "Unrecognized key in body -- please review the v1 API documentation for request body changes";
+
 export const scrapeOptions = z.object({
   formats: z
     .enum([
@@ -53,14 +55,14 @@ export const scrapeOptions = z.object({
   timeout: z.number().int().positive().finite().safe().default(30000), // default?
   waitFor: z.number().int().nonnegative().finite().safe().default(0),
   parsePDF: z.boolean().default(true),
-});
+}).strict(strictMessage);
 
 export type ScrapeOptions = z.infer<typeof scrapeOptions>;
 
 export const scrapeRequestSchema = scrapeOptions.extend({
   url,
   origin: z.string().optional().default("api"),
-});
+}).strict(strictMessage);
 
 // export type ScrapeRequest = {
 //   url: string;
@@ -83,7 +85,7 @@ const crawlerOptions = z.object({
   allowBackwardLinks: z.boolean().default(false), // >> TODO: CHANGE THIS NAME???
   allowExternalLinks: z.boolean().default(false),
   ignoreSitemap: z.boolean().default(true),
-});
+}).strict(strictMessage);
 
 // export type CrawlerOptions = {
 //   includePaths?: string[];
@@ -97,13 +99,12 @@ const crawlerOptions = z.object({
 
 export type CrawlerOptions = z.infer<typeof crawlerOptions>;
 
-export const crawlRequestSchema = z.object({
+export const crawlRequestSchema = crawlerOptions.extend({
   url,
   origin: z.string().optional().default("api"),
-  crawlerOptions: crawlerOptions.default({}),
   scrapeOptions: scrapeOptions.omit({ timeout: true }).default({}),
   webhook: z.string().url().optional(),
-});
+}).strict(strictMessage);
 
 // export type CrawlRequest = {
 //   url: string;
@@ -119,7 +120,7 @@ export const mapRequestSchema = crawlerOptions.extend({
   includeSubdomains: z.boolean().default(true),
   search: z.string().optional(),
   ignoreSitemap: z.boolean().default(false),
-});
+}).strict(strictMessage);
 
 // export type MapRequest = {
 //   url: string;