firecrawl/.github/workflows/fly.yml

name: Fly Deploy
on:
  push:
    branches:
      - main

env:
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
  HOST: ${{ secrets.HOST }}
  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
  PORT: ${{ secrets.PORT }}
  REDIS_URL: ${{ secrets.REDIS_URL }}
  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
  PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
  PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}

jobs:
  pre-deploy-e2e-tests:
    name: Pre-deploy checks
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "20"
      - name: Install pnpm
        run: npm install -g pnpm
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Start the application
        run: npm start &
        working-directory: ./apps/api
        id: start_app
      - name: Start workers
        run: npm run workers &
        working-directory: ./apps/api
        id: start_workers
      - name: Run E2E tests
        run: |
          npm run test:prod
        working-directory: ./apps/api

  pre-deploy-test-suite:
    name: Test Suite
    needs: pre-deploy-e2e-tests
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "20"
      - name: Install pnpm
        run: npm install -g pnpm
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Start the application
        run: npm start &
        working-directory: ./apps/api
        id: start_app
      - name: Start workers
        run: npm run workers &
        working-directory: ./apps/api
        id: start_workers
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/test-suite
      - name: Run E2E tests
        run: |
          npm run test:suite
        working-directory: ./apps/test-suite

  python-sdk-tests:
    name: Python SDK Tests
    needs: pre-deploy-e2e-tests
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.x'
      - name: Install pnpm
        run: npm install -g pnpm
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Start the application
        run: npm start &
        working-directory: ./apps/api
        id: start_app
      - name: Start workers
        run: npm run workers &
        working-directory: ./apps/api
        id: start_workers
      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
        working-directory: ./apps/python-sdk
      - name: Run E2E tests for Python SDK
        run: |
          pytest firecrawl/__tests__/e2e_withAuth/test.py
        working-directory: ./apps/python-sdk

  js-sdk-tests:
    name: JavaScript SDK Tests
    needs: pre-deploy-e2e-tests
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "20"
      - name: Install pnpm
        run: npm install -g pnpm
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Start the application
        run: npm start &
        working-directory: ./apps/api
        id: start_app
      - name: Start workers
        run: npm run workers &
        working-directory: ./apps/api
        id: start_workers
      - name: Install dependencies for JavaScript SDK
        run: pnpm install
        working-directory: ./apps/js-sdk/firecrawl
      - name: Run E2E tests for JavaScript SDK
        run: npm run test
        working-directory: ./apps/js-sdk/firecrawl
        
  go-sdk-tests:
    name: Go SDK Tests
    needs: pre-deploy-e2e-tests
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: "go.mod"
      - name: Install pnpm
        run: npm install -g pnpm
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Start the application
        run: npm start &
        working-directory: ./apps/api
        id: start_app
      - name: Start workers
        run: npm run workers &
        working-directory: ./apps/api
        id: start_workers
      - name: Install dependencies for Go SDK
        run: go mod tidy
        working-directory: ./apps/go-sdk
      - name: Run tests for Go SDK
        run: go test -v ./... -timeout 180s
        working-directory: ./apps/go-sdk/firecrawl

  deploy:
    name: Deploy app
    runs-on: ubuntu-latest
    needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests]
    steps:
      - uses: actions/checkout@v3
      - uses: superfly/flyctl-actions/setup-flyctl@master
      - run: flyctl deploy --remote-only -a firecrawl-scraper-js --build-secret SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN
        working-directory: ./apps/api
        env:
          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
          BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}

  build-and-publish-python-sdk:
    name: Build and publish Python SDK
    runs-on: ubuntu-latest
    needs: deploy

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.x'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel twine build requests packaging

      - name: Run version check script
        id: version_check_script
        run: |
          PYTHON_SDK_VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
          echo "PYTHON_SDK_VERSION_INCREMENTED=$PYTHON_SDK_VERSION_INCREMENTED" >> $GITHUB_ENV

      - name: Build the package
        if: ${{ env.PYTHON_SDK_VERSION_INCREMENTED == 'true' }}
        run: |
          python -m build
        working-directory: ./apps/python-sdk

      - name: Publish to PyPI
        if: ${{ env.PYTHON_SDK_VERSION_INCREMENTED == 'true' }}
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          twine upload dist/*
        working-directory: ./apps/python-sdk

  build-and-publish-js-sdk:
    name: Build and publish JavaScript SDK
    runs-on: ubuntu-latest
    needs: deploy

    steps:
      - uses: actions/checkout@v3
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '20'
          registry-url: 'https://registry.npmjs.org/'
          scope: '@mendable'
          always-auth: true

      - name: Install pnpm
        run: npm install -g pnpm

      - name: Install python for running version check script
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel requests packaging

      - name: Install dependencies for JavaScript SDK
        run: pnpm install
        working-directory: ./apps/js-sdk/firecrawl
        
      - name: Run version check script
        id: version_check_script
        run: |
          VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py js ./apps/js-sdk/firecrawl @mendable/firecrawl-js)
          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV

      - name: Build and publish to npm
        if: ${{ env.VERSION_INCREMENTED == 'true' }}
        env:
          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
        run: |
          npm run build-and-publish
        working-directory: ./apps/js-sdk/firecrawl