Merge pull request #265 from mendableai/feat/issue-264

[Feat] Added route to clean completed jobs and a github action cron that triggers every 24h

.github/workflows/clean-before-24h-complete-jobs.yml

@@ -0,0 +1,20 @@
+name: Clean Before 24h Completed Jobs
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+env:
+  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+    
+jobs:
+  clean-jobs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send GET request to clean jobs
+        run: |
+          response=$(curl --write-out '%{http_code}' --silent --output /dev/null https://api.firecrawl.dev/admin/${{ secrets.BULL_AUTH_KEY }}/clean-before-24h-complete-jobs)
+          if [ "$response" -ne 200 ]; then
+            echo "Failed to clean jobs. Response: $response"
+            exit 1
+          fi
+          echo "Successfully cleaned jobs. Response: $response"

apps/api/src/index.ts

@@ -164,6 +164,27 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
   }
 });
 
+app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, async (req, res) => {
+  try {
+    const webScraperQueue = getWebScraperQueue();
+    const completedJobs = await webScraperQueue.getJobs(['completed']);
+    const before24hJobs = completedJobs.filter(job => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000);
+    const jobIds = before24hJobs.map(job => job.id) as string[];
+    let count = 0;
+    for (const jobId of jobIds) {
+      try {
+        await webScraperQueue.removeJobs(jobId);
+        count++;
+      } catch (jobError) {
+        console.error(`Failed to remove job with ID ${jobId}:`, jobError);
+      }
+    }
+    res.status(200).send(`Removed ${count} completed jobs.`);
+  } catch (error) {
+    console.error('Failed to clean last 24h complete jobs:', error);
+    res.status(500).send('Failed to clean jobs');
+  }
+});
 
 app.get("/is-production", (req, res) => {
   res.send({ isProduction: global.isProduction });