firebase-ios-sdk/.github/workflows/firestore.yml

# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: firestore

on:
  workflow_dispatch:
  pull_request:
  schedule:
    # Run every day at 12am (PST) - cron uses UTC times
    - cron:  '0 8 * * *'

concurrency:
    group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
    cancel-in-progress: true

jobs:
  changes:
    runs-on: macos-14
    # Only when this is not a scheduled run
    if: github.event_name != 'schedule'
    outputs:
      changed: ${{ steps.firestore_src_changes.outputs.sources == 'true' || steps.related_changes.outputs.other_changes == 'true' }}
    steps:
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: firestore_src_changes
        with:
          predicate-quantifier: 'every'
          filters: |
            sources:
              # Firestore sources
              - 'Firestore/**'
              - '!Firestore/**/*.md'
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: related_changes
        with:
          filters: |
            other_changes:
              # Interop headers
              - 'FirebaseAuth/Interop/*.h'

              # FirebaseCore header change
              - 'FirebaseCore/Internal'
              - 'FirebaseCore/Sources/Public'

              # Podspec
              - 'FirebaseFirestoreInternal.podspec'
              - 'FirebaseFirestore.podspec'

              # Package.swift
              - 'Package.swift'

              # CMake
              - '**CMakeLists.txt'
              - 'cmake/**'

              # Build scripts to which Firestore is sensitive
              #
              # Note that this doesn't include check scripts because changing those will
              # already trigger the check workflow.
              - 'scripts/binary_to_array.py'
              - 'scripts/build.sh'
              - 'scripts/install_prereqs.sh'
              - 'scripts/localize_podfile.swift'
              - 'scripts/pod_lib_lint.rb'
              - 'scripts/run_firestore_emulator.sh'
              - 'scripts/setup_*'
              - 'scripts/sync_project.rb'
              - 'scripts/test_quickstart.sh'
              - 'scripts/xcresult_logs.py'

              # This workflow
              - '.github/workflows/firestore.yml'

              # Rebuild on Ruby infrastructure changes.
              - 'Gemfile*'

  check:
    needs: changes
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request' && needs.changes.outputs.changed == 'true')
    runs-on: macos-14
    steps:
    - uses: actions/checkout@v4

    - uses: actions/setup-python@v5
      with:
        python-version: 3.11

    - name: Setup check
      run:  scripts/setup_check.sh

    - name: Run check
      run:  scripts/check.sh --test-only

  cmake:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request' && needs.changes.outputs.changed == 'true')
    strategy:
      matrix:
        os: [macos-14, ubuntu-latest]

    env:
      MINT_PATH: ${{ github.workspace }}/mint
      USE_LATEST_CMAKE: false

    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v4

    - name: Prepare ccache
      uses: actions/cache@v4
      with:
        path: ${{ runner.temp }}/ccache
        key: firestore-ccache-${{ runner.os }}-${{ github.sha }}
        restore-keys: |
          firestore-ccache-${{ runner.os }}-

    - name: Cache Mint packages
      uses: actions/cache@v4
      with:
        path: ${{ env.MINT_PATH }}
        key: ${{ runner.os }}-mint-${{ hashFiles('**/Mintfile') }}
        restore-keys: ${{ runner.os }}-mint-

    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Setup cmake
      uses: jwlawson/actions-setup-cmake@v2
      with:
        cmake-version: '3.31.1'

    - name: Setup build
      run:  scripts/install_prereqs.sh Firestore ${{ runner.os }} cmake

    - name: Build and test
      run: |
        export EXPERIMENTAL_MODE=true
        export CCACHE_DIR=${{ runner.temp }}/ccache
        scripts/third_party/travis/retry.sh scripts/build.sh Firestore ${{ runner.os }} cmake


  cmake-prod-db:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request' && needs.changes.outputs.changed == 'true')

    strategy:
      matrix:
        os: [macos-14]
        databaseId: [(default), test-db]

    env:
      plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
      MINT_PATH: ${{ github.workspace }}/mint
      TARGET_DATABASE_ID: ${{ matrix.databaseId }}
      USE_LATEST_CMAKE: false

    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v4

    - name: Prepare ccache
      uses: actions/cache@v4
      with:
        path: ${{ runner.temp }}/ccache
        key: firestore-ccache-${{ matrix.databaseId }}-${{ runner.os }}-${{ github.sha }}
        restore-keys: |
          firestore-ccache-${{ matrix.databaseId }}-${{ runner.os }}-

    - name: Cache Mint packages
      uses: actions/cache@v4
      with:
        path: ${{ env.MINT_PATH }}
        key: ${{ runner.os }}-mint-${{ hashFiles('**/Mintfile') }}
        restore-keys: ${{ runner.os }}-mint-

    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Install Secret GoogleService-Info.plist
      run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/firestore.plist.gpg \
        Firestore/Example/App/GoogleService-Info.plist "$plist_secret"

    - name: Install Google Service Account key
      run: |
        scripts/decrypt_gha_secret.sh scripts/gha-encrypted/firestore-integration.json.gpg \
        google-service-account.json "$plist_secret"

    # create composite indexes with Terraform
    - name: Set up Google Cloud SDK
      uses: google-github-actions/setup-gcloud@v1
    - name: Setup Terraform
      uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2
    - name: Terraform Init
      run: |
        cd Firestore
        terraform init
    - name: Terraform Apply
      run: |
        cd Firestore

        # Define a temporary file, redirect both stdout and stderr to it
        output_file=$(mktemp)
        if ! terraform apply -var-file=../google-service-account.json -auto-approve > "$output_file" 2>&1 ; then
          cat "$output_file"
          if cat "$output_file" | grep -q "index already exists"; then
            echo "==================================================================================="
            echo "Terraform apply failed due to index already exists; We can safely ignore this error."
            echo "==================================================================================="
          fi
          exit 1
        fi
        rm -f "$output_file"
      env:
        GOOGLE_APPLICATION_CREDENTIALS: ../google-service-account.json
      continue-on-error: true

    - name: Setup cmake
      uses: jwlawson/actions-setup-cmake@v2
      with:
        cmake-version: '3.31.1'

    - name: Setup build
      run:  scripts/install_prereqs.sh Firestore ${{ runner.os }} cmake

    - name: Build and test
      run: |
        export CCACHE_DIR=${{ runner.temp }}/ccache
        scripts/third_party/travis/retry.sh scripts/build.sh Firestore ${{ runner.os }} cmake


  sanitizers-mac:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request' && needs.changes.outputs.changed == 'true')

    strategy:
      matrix:
        os: [macos-14]
        sanitizer: [asan, tsan]

    runs-on: ${{ matrix.os }}

    env:
      SANITIZERS: ${{ matrix.sanitizer }}
      USE_LATEST_CMAKE: false

    steps:
    - uses: actions/checkout@v4

    - name: Prepare ccache
      uses: actions/cache@v4
      with:
        path: ${{ runner.temp }}/ccache
        key: ${{ matrix.sanitizer }}-firestore-ccache-${{ runner.os }}-${{ github.sha }}
        restore-keys: |
          ${{ matrix.sanitizer }}-firestore-ccache-${{ runner.os }}-

    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Setup cmake
      uses: jwlawson/actions-setup-cmake@v2
      with:
        cmake-version: '3.31.1'

    - name: Setup build
      run:  scripts/install_prereqs.sh Firestore ${{ runner.os }} cmake

    - name: Build and test
      run: |
        export EXPERIMENTAL_MODE=true
        export CCACHE_DIR=${{ runner.temp }}/ccache
        scripts/third_party/travis/retry.sh scripts/build.sh Firestore ${{ runner.os }} cmake


  sanitizers-ubuntu:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request' && needs.changes.outputs.changed == 'true')

    strategy:
      matrix:
        os: [ubuntu-latest]
        # Excluding TSAN on ubuntu because of the warnings it generates around schedule.cc.
        # This could be due to Apple Clang provide additional support for synchronization
        # on Apple platforms, which is what we primarily care about.
        sanitizer: [asan]

    runs-on: ${{ matrix.os }}

    env:
      SANITIZERS: ${{ matrix.sanitizer }}
      ASAN_OPTIONS: detect_leaks=0
      USE_LATEST_CMAKE: false

    steps:
    - uses: actions/checkout@v3

    - name: Prepare ccache
      uses: actions/cache@v4
      with:
        path: ${{ runner.temp }}/ccache
        key: ${{ matrix.sanitizer }}-firestore-ccache-${{ runner.os }}-${{ github.sha }}
        restore-keys: |
          ${{ matrix.sanitizer }}-firestore-ccache-${{ runner.os }}-

    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Setup cmake
      uses: jwlawson/actions-setup-cmake@v2
      with:
        cmake-version: '3.31.1'

    - name: Setup build
      run:  scripts/install_prereqs.sh Firestore ${{ runner.os }} cmake

    - name: Build and test
      run: |
        export EXPERIMENTAL_MODE=true
        export CCACHE_DIR=${{ runner.temp }}/ccache
        scripts/third_party/travis/retry.sh scripts/build.sh Firestore ${{ runner.os }} cmake


  xcodebuild:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request')
    runs-on: macos-15

    strategy:
      matrix:
        target: [iOS, macOS, tvOS]

    steps:
    - uses: actions/checkout@v4
    - uses: mikehardy/buildcache-action@c87cea0ccd718971d6cc39e672c4f26815b6c126
      with:
        cache_key: ${{ matrix.target }}

    - uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1

    - name: Setup build
      run:  scripts/install_prereqs.sh Firestore ${{ matrix.target }} xcodebuild

    - name: Build and test
      run: |
        export EXPERIMENTAL_MODE=true
        scripts/third_party/travis/retry.sh scripts/build.sh Firestore ${{ matrix.target }} xcodebuild


  pod-lib-lint:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request')
    runs-on: macos-15
    strategy:
      matrix:
        podspec: [
          'FirebaseFirestoreInternal.podspec',
          'FirebaseFirestore.podspec',
        ]

    steps:
    - uses: actions/checkout@v4

    - uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1
    - name: Setup Bundler
      run: ./scripts/setup_bundler.sh
    - name: Xcode
      run: sudo xcode-select -s /Applications/Xcode_16.2.app/Contents/Developer

    - name: Pod lib lint
      # TODO(#9565, b/227461966): Remove --no-analyze when absl is fixed.
      run: |
        scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb ${{ matrix.podspec }} \
            --platforms=ios \
            --allow-warnings \
            --no-analyze

  # `pod lib lint` takes a long time so only run the other platforms and static frameworks build in the cron.
  pod-lib-lint-cron:
    needs: check
    if: github.event_name == 'schedule' && github.repository == 'Firebase/firebase-ios-sdk'
    strategy:
      matrix:
        podspec: [
          'FirebaseFirestoreInternal.podspec',
          'FirebaseFirestore.podspec',
        ]
        platforms: [
          'macos',
          'tvos',
          'ios',
        ]
        flags: [
          '--use-static-frameworks',
          '',
        ]
        os: [macos-15, macos-14]
        # Skip matrix cells covered by pod-lib-lint job.
        exclude:
          - os: macos-15
            platforms: 'ios'
    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/checkout@v4

    - uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1
    - name: Setup Bundler
      run: ./scripts/setup_bundler.sh
    - name: Xcode
      run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer

    - name: Pod lib lint
      # TODO(#9565, b/227461966): Remove --no-analyze when absl is fixed.
      run: |
        scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb ${{ matrix.podspec }}\
            ${{ matrix.flags }} \
            --platforms=${{ matrix.platforms }} \
            --allow-warnings \
            --no-analyze

  spm-package-resolved:
    runs-on: macos-14
    env:
      FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1
      FIREBASE_SOURCE_FIRESTORE: 1
    outputs:
      cache_key: ${{ steps.generate_cache_key.outputs.cache_key }}
    steps:
      - uses: actions/checkout@v4
      - name: Xcode
        run: sudo xcode-select -s /Applications/Xcode_16.2.app/Contents/Developer
      - name: Generate Swift Package.resolved
        id: swift_package_resolve
        run: |
          swift package resolve
      - name: Generate cache key
        id: generate_cache_key
        run: |
          cache_key="${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }}"
          echo "cache_key=${cache_key}" >> "$GITHUB_OUTPUT"
      - uses: actions/cache/save@v4
        id: cache
        with:
          path: .build
          key: ${{ steps.generate_cache_key.outputs.cache_key }}

  spm-source:
    needs: [check, spm-package-resolved]
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request')
    strategy:
      matrix:
        include:
          - os: macos-14
            xcode: Xcode_16.2
            target: iOS
          - os: macos-15
            xcode: Xcode_16.2
            target: iOS
          - os: macos-15
            xcode: Xcode_16.2
            target: tvOS
          - os: macos-15
            xcode: Xcode_16.2
            target: macOS
          - os: macos-15
            xcode: Xcode_16.2
            target: catalyst
          - os: macos-15
            xcode: Xcode_16.2
            target: visionOS
    runs-on: ${{ matrix.os }}
    env:
      FIREBASE_SOURCE_FIRESTORE: 1
    steps:
    - uses: actions/checkout@v4
    - uses: mikehardy/buildcache-action@c87cea0ccd718971d6cc39e672c4f26815b6c126
      with:
        cache_key: spm${{ matrix.os }}-${{ matrix.xcode }}-${{ matrix.target }}
    - name: Xcode
      run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer
    - name: Initialize xcodebuild
      run: scripts/setup_spm_tests.sh
    - name: iOS Build Test
      run: scripts/third_party/travis/retry.sh ./scripts/build.sh FirebaseFirestore ${{ matrix.target }} spmbuildonly

  spm-binary:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request' && needs.changes.outputs.changed == 'true')
    runs-on: macos-15
    steps:
    - uses: actions/checkout@v4
    - uses: mikehardy/buildcache-action@c87cea0ccd718971d6cc39e672c4f26815b6c126
      with:
        cache_key: spm-binary
    - name: Xcode
      run: sudo xcode-select -s /Applications/Xcode_16.2.app/Contents/Developer
    - name: Initialize xcodebuild
      run: scripts/setup_spm_tests.sh
    - name: iOS Build Test
      run: scripts/third_party/travis/retry.sh ./scripts/build.sh FirebaseFirestore iOS spmbuildonly

  check-firestore-internal-public-headers:
    needs: check
    # Either a scheduled run from public repo, or a pull request with firestore changes.
    if: |
      (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') ||
      (github.event_name == 'pull_request' && needs.changes.outputs.changed == 'true')
    runs-on: macos-14
    steps:
    - uses: actions/checkout@v4
    - name: Assert that Firestore and FirestoreInternal have identically named headers.
      run: |
        fst_dir=Firestore/Source/Public/FirebaseFirestore/
        fst_internal_dir=FirebaseFirestoreInternal/FirebaseFirestore/

        comparison=$(comm -3 <(ls $fst_dir | sort) <(ls $fst_internal_dir | sort))

        if [[ -z "$comparison" ]]; then
          echo "Success: Directories '$fst_dir' and '$fst_internal_dir' match."
        else
          echo "Error: Directories '$fst_dir' and '$fst_internal_dir' differ:"
          echo "Files only in '$fst_dir':"
          # Files in this set do not start with whitespace. Grep for them and a
          # dashed prefix for nicer formatting.
          echo "$comparison" | grep -v '^\s' | sed 's/^/- /'
          echo "Files only in '$fst_internal_dir':"
          # Files in this set start with whitespace. Grep for them and a dashed
          # prefix for nicer formatting.
          echo "$comparison" | grep '^\s' | sed 's/^	/- /'
          exit 1
        fi

  # TODO: Re-enable either in or after #11706.
  # spm-source-cron:
  #   # Don't run on private repo.
  #   if: github.event_name == 'schedule' && github.repository == 'Firebase/firebase-ios-sdk'
  #   runs-on: macos-14
  #   strategy:
  #     matrix:
  #       target: [tvOS, macOS, catalyst]
  #   env:
  #     FIREBASE_SOURCE_FIRESTORE: 1
  #   steps:
  #   - uses: actions/checkout@v4
  #   - uses: mikehardy/buildcache-action@c87cea0ccd718971d6cc39e672c4f26815b6c126
  #     with:
  #       cache_key: ${{ matrix.os }}
  #   - name: Initialize xcodebuild
  #     run: scripts/setup_spm_tests.sh
  #   - name: Build Test - Binary
  #     run: scripts/third_party/travis/retry.sh ./scripts/build.sh FirebaseFirestore ${{ matrix.target }} spmbuildonly

  spm-binary-cron:
    # Don't run on private repo.
    if: github.event_name == 'schedule' && github.repository == 'Firebase/firebase-ios-sdk'
    runs-on: macos-15
    strategy:
      matrix:
        target: [tvOS, macOS, catalyst]
    steps:
    - uses: actions/checkout@v4
    - uses: mikehardy/buildcache-action@c87cea0ccd718971d6cc39e672c4f26815b6c126
      with:
        cache_key: ${{ matrix.target }}
    - name: Xcode
      run: sudo xcode-select -s /Applications/Xcode_16.2.app/Contents/Developer
    - name: Initialize xcodebuild
      run: scripts/setup_spm_tests.sh
    - name: Build Test - Binary
      run: scripts/third_party/travis/retry.sh ./scripts/build.sh FirebaseFirestore ${{ matrix.target }} spmbuildonly

  # A job that fails if any required job in the test matrix fails,
  # to be used as a required check for merging.
  check-required-tests:
    runs-on: ubuntu-latest
    name: Check all required Firestore tests results
    needs: [cmake, cmake-prod-db, xcodebuild, spm-source, spm-binary]
    steps:
      - name: Check test matrix
        if: needs.*.result == 'failure'
        run: exit 1

  # Disable until FirebaseUI is updated to accept Firebase 9 and quickstart is updated to accept
  # Firebase UI 12
  # quickstart:
  #   # Don't run on private repo unless it is a PR.
  #   if: (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') || github.event_name == 'pull_request'
  #   env:
  #     plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
  #     signin_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
  #   runs-on: macos-14
  #   needs: check

  #   steps:
  #   - uses: actions/checkout@v4
  #   - name: Setup quickstart
  #     run: scripts/setup_quickstart.sh firestore
  #   - name: Install Secret GoogleService-Info.plist
  #     run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/qs-firestore.plist.gpg \
  #         quickstart-ios/firestore/GoogleService-Info.plist "$plist_secret"
  #   - name: Test swift quickstart
  #     run: ([ -z $plist_secret ] ||
  #           scripts/third_party/travis/retry.sh scripts/test_quickstart.sh Firestore false)