diff --git a/.github/workflows/clean-before-24h-complete-jobs.yml b/.github/workflows/clean-before-24h-complete-jobs.yml new file mode 100644 index 00000000..2ced5373 --- /dev/null +++ b/.github/workflows/clean-before-24h-complete-jobs.yml @@ -0,0 +1,20 @@ +name: Clean Before 24h Completed Jobs +on: + schedule: + - cron: '0 0 * * *' + +env: + BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} + +jobs: + clean-jobs: + runs-on: ubuntu-latest + steps: + - name: Send GET request to clean jobs + run: | + response=$(curl --write-out '%{http_code}' --silent --output /dev/null https://api.firecrawl.dev/admin/${{ secrets.BULL_AUTH_KEY }}/clean-before-24h-complete-jobs) + if [ "$response" -ne 200 ]; then + echo "Failed to clean jobs. Response: $response" + exit 1 + fi + echo "Successfully cleaned jobs. Response: $response" diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 0246a1e0..cc8376be 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -164,6 +164,27 @@ app.get('/serverHealthCheck/notify', async (req, res) => { } }); +app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, async (req, res) => { + try { + const webScraperQueue = getWebScraperQueue(); + const completedJobs = await webScraperQueue.getJobs(['completed']); + const before24hJobs = completedJobs.filter(job => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000); + const jobIds = before24hJobs.map(job => job.id) as string[]; + let count = 0; + for (const jobId of jobIds) { + try { + await webScraperQueue.removeJobs(jobId); + count++; + } catch (jobError) { + console.error(`Failed to remove job with ID ${jobId}:`, jobError); + } + } + res.status(200).send(`Removed ${count} completed jobs.`); + } catch (error) { + console.error('Failed to clean last 24h complete jobs:', error); + res.status(500).send('Failed to clean jobs'); + } +}); app.get("/is-production", (req, res) => { res.send({ isProduction: global.isProduction });