Set up multi jobs for spec testing. (#9148)

* Set up multi jobs for spec testing.

.github/workflows/abtesting.yml

@@ -118,14 +118,3 @@ jobs:
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb \
           FirebaseABTesting.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseABTesting.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/analytics.yml

@@ -33,14 +33,3 @@ jobs:
 
 # TODO: Consider pushing GoogleAppMeasurement.podspec.json to SpecsDev to enable similar test
 # for FirebaseAnalytics.podspec.json
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseAnalyticsSwift.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/auth.yml

@@ -141,14 +141,3 @@ jobs:
       run: scripts/configure_test_keychain.sh
     - name: PodLibLint Auth Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseAuth.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseAuth.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/core.yml

@@ -94,14 +94,3 @@ jobs:
       run: scripts/setup_bundler.sh
     - name: PodLibLint Core Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseCore.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseCore.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/crashlytics.yml

@@ -129,14 +129,3 @@ jobs:
       run: scripts/setup_bundler.sh
     - name: PodLibLint Auth Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseCrashlytics.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseCrashlytics.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/database.yml

@@ -150,14 +150,3 @@ jobs:
       run: scripts/setup_bundler.sh
     - name: PodLibLint database Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseDatabase.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseDatabase.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/dynamiclinks.yml

@@ -82,14 +82,3 @@ jobs:
     - name: Test swift quickstart
       if: ${{ always() }}
       run: ([ -z $plist_secret ] || scripts/third_party/travis/retry.sh scripts/test_quickstart.sh DynamicLinks true swift)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseDynamicLinks.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/firestore.yml

@@ -289,14 +289,3 @@ jobs:
     - name: Test swift quickstart
       run: ([ -z $plist_secret ] ||
             scripts/third_party/travis/retry.sh scripts/test_quickstart.sh Firestore false)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseFirestore.podspec --skip-tests --allow-warnings --platforms=ios --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/functions.yml

@@ -145,17 +145,3 @@ jobs:
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb \
           FirebaseFunctions.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    strategy:
-      matrix:
-        target: [ios, tvos, macos, watchos]
-        podspec: [ 'FirebaseFunctions.podspec', 'FirebaseFunctionsSwift.podspec']
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint ${{ matrix.podspec }} --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/google-utilities-components.yml

@@ -64,14 +64,3 @@ jobs:
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb \
           GoogleUtilitiesComponents.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint GoogleUtilitiesComponents.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/inappmessaging.yml

@@ -107,17 +107,3 @@ jobs:
     - name: Test swift quickstart
       run: ([ -z $plist_secret ] ||
             scripts/third_party/travis/retry.sh scripts/test_quickstart.sh InAppMessaging true swift)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    strategy:
-      matrix:
-        podspec: [FirebaseInAppMessaging.podspec, FirebaseInAppMessagingSwift.podspec]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint ${{ matrix.podspec}} --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/installations.yml

@@ -134,14 +134,3 @@ jobs:
        export FIS_INTEGRATION_TESTS_REQUIRED=${{ steps.secrets.outputs.val }}
        scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseInstallations.podspec \
          --platforms=${{ matrix.target }} ${{ matrix.flags }} \
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseInstallations.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/messaging.yml

@@ -198,14 +198,3 @@ jobs:
       run: scripts/install_prereqs.sh MessagingSample iOS
     - name: Build
       run: ([ -z $plist_secret ] || scripts/build.sh MessagingSample iOS)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseMessaging.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/mlmodeldownloader.yml

@@ -122,14 +122,3 @@ jobs:
       run: scripts/install_prereqs.sh MLModelDownloaderSample iOS
     - name: Build
       run: ([ -z $plist_secret ] || scripts/build.sh MLModelDownloaderSample iOS)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseMLModelDownloader.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/performance.yml

@@ -121,17 +121,6 @@ jobs:
     - name: Setup project and Build Catalyst
       run: scripts/test_catalyst.sh FirebasePerformance build
 
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebasePerformance.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
-
   performance-cron-only:
     # Don't run on private repo.
     if: github.event_name == 'schedule' && github.repository == 'Firebase/firebase-ios-sdk'

.github/workflows/remoteconfig.yml

@@ -158,14 +158,3 @@ jobs:
     - name: PodLibLint RemoteConfig Cron
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseRemoteConfig.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseRemoteConfig.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

.github/workflows/spectesting.yml

@@ -0,0 +1,62 @@
+name: spectesting
+
+on:
+  pull_request
+
+jobs:
+  specs_checking:
+    # Don't run on private repo unless it is a PR.
+    if: github.repository == 'Firebase/firebase-ios-sdk'
+    runs-on: macos-11
+    outputs:
+      matrix: ${{ steps.check_files.outputs.matrix }}
+      podspecs: ${{ steps.check_files.outputs.podspecs }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: check files
+      id: check_files
+      env:
+        pr_branch: ${{ github.event.pull_request.head.ref }}
+      run: |
+        # The output.json will have podspecs that will be tested.
+        # ``` output.json
+        # [{"podspec":"FirebaseABTesting.podspec"},{"podspec":"FirebaseAnalytics.podspec.json"}]
+        # ```
+        # This array will help generate a matrix for jobs in GHA, taking
+        # advantage of `include`.
+        # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-including-additional-values-into-combinations
+        # The final matrix will be set as an env var in the GHA workflow,
+        # This env var, matrix, will be passed as a matrix in the next job of
+        # the workflow.
+        ./scripts/health_metrics/get_updated_files.sh -p output.json
+        echo "::set-output name=matrix::{\"include\":$( cat output.json )}"
+        # `podspecs` is to help determine if specs_testing job should be run.
+        echo "::set-output name=podspecs::$(cat output.json)"
+  specs_testing:
+    needs: specs_checking
+    if: ${{ needs.specs_checking.outputs.podspecs != '[]' }}
+    runs-on: macos-11
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.specs_checking.outputs.matrix)}}
+    env:
+      PODSPEC: ${{ matrix.podspec }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Init podspecs and source
+      run: |
+        mkdir specTestingLogs
+        cd ReleaseTooling
+        swift run podspecs-tester --git-root "${GITHUB_WORKSPACE}" --podspec ${PODSPEC} --skip-tests --temp-log-dir "${GITHUB_WORKSPACE}/specTestingLogs"
+    - name: Upload Failed Testing Logs
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: specTestingLogs
+        path: specTestingLogs/*.txt

.github/workflows/storage.yml

@@ -170,20 +170,3 @@ jobs:
     - name: PodLibLint Storage Cron
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseStorageSwift.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    strategy:
-      matrix:
-        spec: [
-          'FirebaseStorage.podspec --test-specs=unit', # The integration tests need more set up.
-          'FirebaseStorageSwift.podspec --skip-tests' # No current unit tests.
-        ]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint ${{ matrix.spec }} --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'

ReleaseTooling/Sources/PodspecsTester/InitializeSource.swift

@@ -55,10 +55,10 @@ struct InitializeSpecTesting {
                                   podRepoName: String = Constants.localSpecRepoName) {
     let result = Shell.executeCommandFromScript("pod repo remove \(podRepoName)")
     switch result {
-    case let .error(code, output):
+    case let .error(_, output):
       print("\(podRepoName) was not properly removed. \(podRepoName) probably" +
         "does not exist in local.\n \(output)")
-    case let .success(output):
+    case .success:
       print("\(podRepoName) was removed.")
     }
     Shell.executeCommand("pod repo add \(podRepoName) \(repoURL)")
@@ -66,9 +66,20 @@ struct InitializeSpecTesting {
 
   // Add a testing tag to the head of the branch.
   private static func addTestingTag(path sdkRepoPath: URL, manifest: FirebaseManifest.Manifest) {
-    let testingTag = Constants.testingTagPrefix + manifest.version
-    // Add or update the testing tag to the local sdk repo.
-    Shell.executeCommand("git tag -af \(testingTag) -m 'spectesting'", workingDir: sdkRepoPath)
+    // Pods could have different versions, like `8.11.0` and `8.11.0-beta`.
+    // These versions should be part of tags, so a warning from `pod spec lint`
+    // could be avoided.
+    // ```
+    //   The version should be included in the Git tag.
+    // ```
+    // The tag should include `s.version`, e.g.
+    // If "s.version = '8.11.0-beta'", the tag should include 8.11.0-beta to
+    // avoid trigerring the warning.
+    for pod in manifest.pods {
+      let testingTag = Constants.testingTagPrefix + manifest.versionString(pod)
+      // Add or update the testing tag to the local sdk repo.
+      Shell.executeCommand("git tag -af \(testingTag) -m 'spectesting'", workingDir: sdkRepoPath)
+    }
   }
 
   // Update the podspec source.
@@ -85,11 +96,11 @@ struct InitializeSpecTesting {
         // After `sed`:
         //  s.source           = {
         //    :git => '\(path.path)',
-        //    :tag => 'testing-\(manifest.version)',
+        //    :tag => 'testing-\(version)',
         //  }
         Shell.executeCommand(
           "sed -i.bak -e \"s|\\(.*\\:git =>[[:space:]]*\\).*|\\1'\(path.path)',| ; " +
-            "s|\\(.*\\:tag =>[[:space:]]*\\).*|\\1'\(Constants.testingTagPrefix + manifest.version)',|\" \(pod.name).podspec",
+            "s|\\(.*\\:tag =>[[:space:]]*\\).*|\\1'\(Constants.testingTagPrefix + version)',|\" \(pod.name).podspec",
           workingDir: path
         )
       }

ReleaseTooling/Sources/PodspecsTester/main.swift

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-import Foundation
-
 import ArgumentParser
+import FirebaseManifest
+import Foundation
 import Utils
 
 struct PodspecsTester: ParsableCommand {
@@ -25,19 +25,115 @@ struct PodspecsTester: ParsableCommand {
           transform: URL.init(fileURLWithPath:))
   var gitRoot: URL
 
+  /// A targeted testing pod, e.g. FirebaseAuth.podspec
+  @Option(help: "A podspec that will be tested.")
+  var podspec: String
+
+  /// The root of the Firebase git repo.
+  @Option(help: "Spec testing log dir", transform: URL.init(fileURLWithPath:))
+  var tempLogDir: URL?
+
+  @Flag(help: "Skip unit tests.")
+  var skipTests: Bool
+
   mutating func validate() throws {
     guard FileManager.default.fileExists(atPath: gitRoot.path) else {
       throw ValidationError("git-root does not exist: \(gitRoot.path)")
     }
   }
 
+  /// Trigger spec test with `spec` under the `workingDir` and return an error
+  /// code and log.
+  ///
+  /// - Parameters:
+  ///   - spec: The podspec name, e.g. FirebaseAnalytics.podspec.json.
+  ///   - workingDir: The dir of the testing spec.
+  ///   - args: A dict including options with its value or/and flags with nil.
+  /// - Returns: A tuple with an error code and log.
+  func specTest(spec: String, workingDir: URL,
+                args: [String: String?]) -> (code: Int32, output: String) {
+    var exitCode: Int32 = 0
+    var logOutput: String = ""
+    // If value is nil, the key will be a flag.
+    let arguments = args.map { key, value in
+      if let v = value {
+        return "--\(key)=\(v)"
+      } else {
+        return "--\(key)"
+      }
+    }.joined(separator: " ")
+    let command =
+      "pod spec lint \(spec) \(arguments) --sources=https://github.com/firebase/SpecsTesting,https://cdn.cocoapods.org/"
+    print(command)
+    let result = Shell.executeCommandFromScript(
+      command,
+      outputToConsole: false,
+      workingDir: workingDir
+    )
+    switch result {
+    case let .error(code, output):
+      print("---- Failed Spec Testing: \(spec) Start ----")
+      print("\(output)")
+      print("---- Failed Spec Testing: \(spec) End ----")
+      exitCode = code
+      logOutput = output
+    case let .success(output):
+      print("\(spec) passed validation.")
+      exitCode = 0
+      logOutput = output
+    }
+
+    if let logDir = tempLogDir {
+      do {
+        try logOutput.write(
+          to: logDir.appendingPathComponent("\(spec).txt"),
+          atomically: true,
+          encoding: String.Encoding.utf8
+        )
+      } catch {
+        print(error)
+      }
+    }
+    return (exitCode, logOutput)
+  }
+
   func run() throws {
     let startDate = Date()
+    var exitCode: Int32 = 0
     print("Started at: \(startDate.dateTimeString())")
     InitializeSpecTesting.setupRepo(sdkRepoURL: gitRoot)
+    let manifest = FirebaseManifest.shared
+    var minutes = 0
+    var timer: DispatchSourceTimer = {
+      let t = DispatchSource.makeTimerSource()
+      t.schedule(deadline: .now(), repeating: 60)
+      t.setEventHandler(handler: {
+        print("Tests have run \(minutes) min(s).")
+        minutes += 1
+      })
+      return t
+    }()
+    timer.resume()
+    let testingPod = podspec.components(separatedBy: ".")[0]
+    for pod in manifest.pods {
+      if testingPod == pod.name {
+        var args: [String: String?] = [:]
+        args["platforms"] = pod.platforms.joined(separator: ",")
+        if pod.allowWarnings {
+          args.updateValue(nil, forKey: "allow-warnings")
+        }
+        if skipTests {
+          args.updateValue(nil, forKey: "skip-tests")
+        }
+        let code = specTest(spec: podspec, workingDir: gitRoot, args: args).code
+        exitCode = code
+      }
+    }
+    timer.cancel()
     let finishDate = Date()
     print("Finished at: \(finishDate.dateTimeString()). " +
       "Duration: \(startDate.formattedDurationSince(finishDate))")
+    Foundation.exit(exitCode)
   }
 }
 

scripts/health_metrics/code_coverage_file_list.json

@@ -1,6 +1,7 @@
 [
   {
     "sdk": "abtesting",
+    "podspecs": ["FirebaseABTesting.podspec"],
     "filePatterns": [
       "^FirebaseABTesting.*",
       "Interop/Analytics/Public/[^/]+\\.h",
@@ -9,6 +10,7 @@
   },
   {
     "sdk": "analytics",
+    "podspecs": ["FirebaseAnalytics.podspec.json", "FirebaseAnalyticsSwift.podspec", "GoogleAppMeasurement.podspec.json"],
     "filePatterns": [
       "^FirebaseAnalytics.*",
       "^GoogleAppMeasurement.*"
@@ -16,6 +18,7 @@
   },
   {
     "sdk": "appcheck",
+    "podspecs": ["FirebaseAppCheck.podspec"],
     "filePatterns": [
       "^FirebaseAppCheck.*",
       "\\.github/workflows/app_check\\.yml"
@@ -23,12 +26,14 @@
   },
   {
     "sdk": "appdistribution",
+    "podspecs": ["FirebaseAppDistribution.podspec"],
     "filePatterns": [
       "^FirebaseAppDistribution.*"
     ]
   },
   {
     "sdk": "auth",
+    "podspecs": ["FirebaseAuth.podspec"],
     "filePatterns": [
       "^FirebaseAuth.*",
       "Interop/Auth/Public/[^/]+\\.h",
@@ -37,6 +42,7 @@
   },
   {
     "sdk": "crashlytics",
+    "podspecs": ["FirebaseCrashlytics.podspec"],
     "filePatterns": [
       "^Crashlytics.*",
       "FirebaseCrashlytics.podspec"
@@ -44,6 +50,7 @@
   },
   {
     "sdk": "database",
+    "podspecs": ["FirebaseDatabase.podspec", "FirebaseDatabaseSwift.podspec"],
     "filePatterns": [
       "^FirebaseDatabase.*",
       "\\.github/workflows/database\\.yml",
@@ -53,6 +60,7 @@
   },
   {
     "sdk": "dynamiclinks",
+    "podspecs": ["FirebaseDynamicLinks.podspec"],
     "filePatterns": [
       "^FirebaseDynamicLinks.*",
       "\\.github/workflows/dynamiclinks\\.yml",
@@ -61,6 +69,7 @@
   },
   {
     "sdk": "firestore",
+    "podspecs": ["FirebaseFirestore.podspec", "FirebaseFirestoreSwift.podspec"],
     "filePatterns": [
       "^Firestore/.*",
       "FirebaseAppCheck/Sources/Interop/[^/]+\\.h",
@@ -75,6 +84,7 @@
   },
   {
     "sdk": "functions",
+    "podspecs": ["FirebaseFunctions.podspec", "FirebaseFunctionsSwift.podspec"],
     "filePatterns": [
       "^Functions.*",
       "\\.github/workflows/functions\\.yml",
@@ -84,6 +94,7 @@
   },
   {
     "sdk": "inappmessaging",
+    "podspecs": ["FirebaseInAppMessaging.podspec", "FirebaseInAppMessagingSwift.podspec"],
     "filePatterns": [
       "^FirebaseInAppMessaging.*",
       "Interop/Analytics/Public/[^/]+\\.h",
@@ -92,12 +103,14 @@
   },
   {
     "sdk": "installations",
+    "podspecs": ["FirebaseInstallations.podspec"],
     "filePatterns": [
       "^FirebaseInstallations.*"
     ]
   },
   {
     "sdk": "messaging",
+    "podspecs": ["FirebaseMessaging.podspec"],
     "filePatterns": [
       "^FirebaseMessaging/.*",
       "Interop/Analytics/Public/[^/]+\\.h",
@@ -107,6 +120,7 @@
   },
   {
     "sdk": "performance",
+    "podspecs": ["FirebasePerformance.podspec"],
     "filePatterns": [
       "^FirebasePerformance/.*",
       "FirebasePerformance\\.podspec",
@@ -115,6 +129,7 @@
   },
   {
     "sdk": "remoteconfig",
+    "podspecs": ["FirebaseRemoteConfig.podspec"],
     "filePatterns": [
       "^FirebaseRemoteConfig.*",
       "Interop/Analytics/Public/[^/]+\\.h",
@@ -124,6 +139,7 @@
   },
   {
     "sdk": "storage",
+    "podspecs": ["FirebaseStorage.podspec", "FirebaseStorageSwift.podspec"],
     "filePatterns": [
       "^FirebaseStorage.*",
       "Interop/Auth/Public/[^/]+\\.h",

scripts/health_metrics/generate_code_coverage_report/Sources/UpdatedFilesCollector/main.swift

@@ -19,9 +19,18 @@ import Foundation
 
 struct SDKFilePattern: Codable {
   let sdk: String
+  let podspecs: [String]
   let filePatterns: [String]
 }
 
+/// SDKPodspec is to help generate an array of podspec in json file, e.g.
+/// ``` output.json
+/// [{"podspec":"FirebaseABTesting.podspec"},{"podspec":"FirebaseAnalytics.podspec.json"}]
+/// ```
+struct SDKPodspec: Codable {
+  let podspec: String
+}
+
 struct UpdatedFilesCollector: ParsableCommand {
   @Option(help: "A txt File with updated files.",
           transform: { str in
@@ -39,7 +48,16 @@ struct UpdatedFilesCollector: ParsableCommand {
           })
   var codeCoverageFilePatterns: [SDKFilePattern]
 
+  @Option(help: "A output file with all Podspecs with related changed files",
+          transform: { str in
+            print(FileManager.default.currentDirectoryPath)
+            let documentDir = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
+            return documentDir.appendingPathComponent(str)
+          })
+  var outputSDKFileURL: URL?
+
   func run() throws {
+    var podspecsWithChangedFiles: [SDKPodspec] = []
     print("=============== list changed files ===============")
     print(changedFilePaths.joined(separator: "\n"))
     // Initiate all run_job flag to false.
@@ -59,6 +77,9 @@ struct UpdatedFilesCollector: ParsableCommand {
           if regex.firstMatch(in: changedFilePath, options: [], range: range) != nil {
             print("=============== paths of changed files ===============")
             print("::set-output name=\(sdkPatterns.sdk)_run_job::true")
+            for podspec in sdkPatterns.podspecs {
+              podspecsWithChangedFiles.append(SDKPodspec(podspec: podspec))
+            }
             print("\(sdkPatterns.sdk): \(changedFilePath) is updated under the pattern, \(pattern)")
             trigger_pod_test_for_coverage_report = true
             // Once this sdk run_job flag is turned to true, then the loop
@@ -70,6 +91,23 @@ struct UpdatedFilesCollector: ParsableCommand {
         if trigger_pod_test_for_coverage_report { break }
       }
     }
+    if let outputPath = outputSDKFileURL {
+      do {
+        // Instead of directly writing Data to a file, trasnferring Data to
+        // String can help trimming whitespaces and newlines in advance.
+        let str = try String(
+          decoding: JSONEncoder().encode(podspecsWithChangedFiles),
+          as: UTF8.self
+        )
+        try str.trimmingCharacters(in: .whitespacesAndNewlines).write(
+          to: outputPath,
+          atomically: true,
+          encoding: String.Encoding.utf8
+        )
+      } catch {
+        fatalError("Error while writting in \(outputPath.path).\n\(error)")
+      }
+    }
   }
 }
 

scripts/health_metrics/get_updated_files.sh

@@ -16,6 +16,15 @@ set -ex
 # Updated files in paths in code_coverage_file_list.json will trigger code coverage workflows.
 # Updates in a pull request will generate a code coverage report in a PR.
 
+while getopts p: flag
+do
+    case "${flag}" in
+        p) spec_output_file=${OPTARG};;
+    esac
+done
+
+dir=$(pwd)
+
 # Get most rescent ancestor commit.
 common_commit=$(git merge-base remotes/origin/${pr_branch} remotes/origin/${GITHUB_BASE_REF})
 target_branch_head=$(git rev-parse remotes/origin/${GITHUB_BASE_REF})
@@ -30,4 +39,9 @@ cd scripts/health_metrics/generate_code_coverage_report
 # merge commit to the head commit from the target branch.
 git diff --name-only remotes/origin/${GITHUB_BASE_REF} ${GITHUB_SHA} > updated_files.txt
 
-swift run UpdatedFilesCollector --changed-file-paths updated_files.txt --code-coverage-file-patterns ../code_coverage_file_list.json
+if [ -z $spec_output_file] ; then
+  swift run UpdatedFilesCollector --changed-file-paths updated_files.txt --code-coverage-file-patterns ../code_coverage_file_list.json
+else
+  swift run UpdatedFilesCollector --changed-file-paths updated_files.txt --code-coverage-file-patterns ../code_coverage_file_list.json --output-sdk-file-url "${spec_output_file}"
+  mv "${spec_output_file}" "${dir}"
+fi