Firestore Travis: add test runs using sanitizers (#1128)

Also move most of `before_install` actions from Travis config into scripts to reduce duplication.

.travis.yml

@@ -22,16 +22,20 @@ jobs:
         # Google C++ style compliance
         - ./scripts/lint.sh $TRAVIS_COMMIT_RANGE
 
+    # The order of builds matters (even though they are run in parallel):
+    # Travis will schedule them in the same order they are listed here.
+
+    # Primary platforms
+
     - stage: test
       env:
         - PROJECT=Firebase PLATFORM=iOS
       before_install:
-        # Add next line back with updated DeviceUDID for xcode9.1 if stability issues with simulator
+        # Add next line back with updated DeviceUDID for xcode9.1 if stability
+        # issues with simulator:
         # - open -a "simulator" --args -CurrentDeviceUDID ABBD7191-486B-462F-80B4-AE08C5820DA1
-        - bundle install
-        - gem install xcpretty
-        - ./scripts/if_changed.sh bundle exec pod install --project-directory=Example --repo-update
-        - ./scripts/if_changed.sh bundle exec pod install --project-directory=Functions/Example
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
       script:
         - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM
 
@@ -49,32 +53,30 @@ jobs:
       env:
         - PROJECT=Firestore PLATFORM=iOS METHOD=xcodebuild
       before_install:
-        - bundle install
-        - gem install xcpretty
-        - ./scripts/if_changed.sh bundle exec pod install --project-directory=Firestore/Example --repo-update
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
       script:
         - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM $METHOD
 
+    # Alternative platforms
+
     - stage: test
       env:
         - PROJECT=Firestore PLATFORM=macOS METHOD=cmake
       before_install:
-        - bundle install
-        - gem install xcpretty
-        - brew outdated cmake || brew upgrade cmake
-        - brew outdated go || brew upgrade go # Somehow the build for Abseil requires this.
-        - ./scripts/if_changed.sh bundle exec pod install --project-directory=Example --repo-update
-        - ./scripts/if_changed.sh bundle exec pod install --project-directory=Firestore/Example --no-repo-update
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
       script:
         - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM $METHOD
 
+    # Community-supported platforms
+
     - stage: test
       env:
         - PROJECT=Firebase PLATFORM=macOS
       before_install:
-        - bundle install
-        - gem install xcpretty
-        - ./scripts/if_changed.sh bundle exec pod install --project-directory=Example --repo-update
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
       script:
         - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM
 
@@ -82,12 +84,72 @@ jobs:
       env:
         - PROJECT=Firebase PLATFORM=tvOS
       before_install:
-        - bundle install
-        - gem install xcpretty
-        - ./scripts/if_changed.sh bundle exec pod install --project-directory=Example --repo-update
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
       script:
         - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM
 
+    # Firestore sanitizers
+
+    - stage: test
+      env:
+        - PROJECT=Firestore PLATFORM=iOS METHOD=xcodebuild SANITIZERS=asan
+      before_install:
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
+      script:
+        - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM $METHOD
+
+    - stage: test
+      env:
+        - PROJECT=Firestore PLATFORM=iOS METHOD=xcodebuild SANITIZERS=tsan
+      before_install:
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
+      script:
+        - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM $METHOD
+
+    # TODO(varconst): enable UBSan in xcodebuild. Right now if fails during
+    # linkage (it works if enabled together with ASan, but it's supposed to be
+    # usable on its own, too).
+
+    - stage: test
+      env:
+        - PROJECT=Firestore PLATFORM=macOS METHOD=cmake SANITIZERS=asan
+      before_install:
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
+      script:
+        - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM $METHOD
+
+    - stage: test
+      env:
+        - PROJECT=Firestore PLATFORM=macOS METHOD=cmake SANITIZERS=tsan
+      before_install:
+        - ./scripts/if_changed.sh ./scripts/install_prereqs.sh
+        - ./scripts/if_changed.sh ./scripts/pod_install.sh
+      script:
+        - ./scripts/if_changed.sh ./scripts/build.sh $PROJECT $PLATFORM $METHOD
+
+    # TODO(varconst): UBSan for CMake. UBSan failures are non-fatal by default,
+    # need to make them fatal for the purposes of the test run.
+
+  # TODO(varconst): disallow sanitizers to fail once we fix all existing issues.
+  allow_failures:
+    - env:
+      - PROJECT=Firestore PLATFORM=macOS METHOD=cmake SANITIZERS=asan
+    - env:
+      - PROJECT=Firestore PLATFORM=macOS METHOD=cmake SANITIZERS=tsan
+    - env:
+      - PROJECT=Firestore PLATFORM=iOS METHOD=xcodebuild SANITIZERS=asan
+    - env:
+      - PROJECT=Firestore PLATFORM=iOS METHOD=xcodebuild SANITIZERS=tsan
+
+  # TODO(varconst): enable if it's possible to make this flag work on build
+  # stages. It's supposed to avoid waiting for jobs that are allowed to fail
+  # before reporting the results.
+  # fast_finish: true
+
 branches:
   only:
     - master

CMakeLists.txt

@@ -17,6 +17,10 @@
 cmake_minimum_required(VERSION 2.8.11)
 project(firebase C CXX)
 
+option(WITH_ASAN "Build with Address Sanitizer" OFF)
+option(WITH_TSAN "Build with Thread Sanitizer (mutually exculsive with other sanitizers)" OFF)
+option(WITH_UBSAN "Build with Undefined Behavior sanitizer" OFF)
+
 # If no build type is specified, make it a debug build
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Debug)

cmake/external/firestore.cmake

@@ -32,6 +32,9 @@ ExternalProject_Add(
   CMAKE_ARGS
     -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
     -DCMAKE_INSTALL_PREFIX:PATH=${FIREBASE_INSTALL_DIR}
+    -DWITH_ASAN:BOOL=${WITH_ASAN}
+    -DWITH_TSAN:BOOL=${WITH_TSAN}
+    -DWITH_UBSAN:BOOL=${WITH_UBSAN}
 
   BUILD_ALWAYS ON
 

scripts/build.sh

@@ -36,6 +36,13 @@ platform can be one of:
 method can be one of:
   xcodebuild (default)
   cmake
+
+Optionally, reads the environment variable SANITIZERS. If set, it is expected to
+be a string containing a space-separated list with some of the following
+elements:
+  asan
+  tsan
+  ubsan
 EOF
   exit 1
 fi
@@ -53,6 +60,9 @@ if [[ $# -gt 2 ]]; then
 fi
 
 echo "Building $product for $platform using $method"
+if [[ -n "${SANITIZERS:-}" ]]; then
+  echo "Using sanitizers: $SANITIZERS"
+fi
 
 # Runs xcodebuild with the given flags, piping output to xcpretty
 # If xcodebuild fails with known error codes, retries once.
@@ -103,6 +113,51 @@ xcb_flags+=(
   CODE_SIGNING_REQUIRED=NO
 )
 
+# TODO(varconst): --warn-unused-vars - right now, it makes the log overflow on
+# Travis.
+cmake_options=(
+  -Wdeprecated
+  --warn-uninitialized
+)
+
+if [[ -n "${SANITIZERS:-}" ]]; then
+  for sanitizer in $SANITIZERS; do
+    case "$sanitizer" in
+      asan)
+        xcb_flags+=(
+          -enableAddressSanitizer YES
+        )
+        cmake_options+=(
+          -DWITH_ASAN=ON
+        )
+        ;;
+
+      tsan)
+        xcb_flags+=(
+          -enableThreadSanitizer YES
+        )
+        cmake_options+=(
+          -DWITH_TSAN=ON
+        )
+        ;;
+
+      ubsan)
+        xcb_flags+=(
+          -enableUndefinedBehaviorSanitizer YES
+        )
+        cmake_options+=(
+          -DWITH_UBSAN=ON
+        )
+        ;;
+
+      *)
+        echo "Unknown sanitizer '$sanitizer'" 1>&2
+        exit 1
+        ;;
+    esac
+  done
+fi
+
 case "$product-$method-$platform" in
   Firebase-xcodebuild-*)
     RunXcodebuild \
@@ -167,7 +222,7 @@ case "$product-$method-$platform" in
   Firestore-cmake-macOS)
     test -d build || mkdir build
     echo "Preparing cmake build ..."
-    (cd build; cmake ..)
+    (cd build; cmake "${cmake_options[@]}" ..)
 
     echo "Building cmake build ..."
     cpus=$(sysctl -n hw.ncpu)

scripts/install_prereqs.sh

@@ -0,0 +1,47 @@
+# Copyright 2018 Google
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Within Travis, installs prerequisites for a build.
+
+# Examines the following configured environment variables that should be
+# specified in an env: block
+#   - PROJECT - Firebase or Firestore
+#   - METHOD - xcodebuild or cmake; default is xcodebuild
+
+if [[ -z "$METHOD" ]]; then
+  METHOD="xcodebuild"
+fi
+
+case "$PROJECT-$METHOD" in
+  *-xcodebuild)
+    bundle install
+    gem install xcpretty
+    ;;
+
+  Firestore-cmake)
+    bundle install
+    # xcpretty is helpful for the intermediate step which builds FirebaseCore
+    # using xcodebuild.
+    gem install xcpretty
+    brew outdated cmake || brew upgrade cmake
+    brew outdated go || brew upgrade go # Somehow the build for Abseil requires this.
+    ;;
+
+  *)
+    echo "Unknown project-method combo" 1>&2
+    echo "  PROJECT=$PROJECT" 1>&2
+    echo "  METHOD=$METHOD" 1>&2
+    exit 1
+    ;;
+esac

scripts/pod_install.sh

@@ -0,0 +1,54 @@
+# Copyright 2018 Google
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Within Travis, installs prerequisites for a build.
+
+# Examines the following configured environment variables that should be
+# specified in an env: block
+#   - PROJECT - Firebase or Firestore
+#   - METHOD - xcodebuild or cmake; default is xcodebuild
+#   - PLATFORM - iOS, macOS, or tvOS
+
+if [[ -z "$METHOD" ]]; then
+  METHOD="xcodebuild"
+fi
+
+case "$PROJECT-$METHOD-$PLATFORM" in
+  Firebase-xcodebuild-iOS)
+    bundle exec pod install --project-directory=Example --repo-update
+    bundle exec pod install --project-directory=Functions/Example
+    ;;
+
+  Firebase-xcodebuild-*)
+    bundle exec pod install --project-directory=Example --repo-update
+    ;;
+
+  Firestore-xcodebuild-*)
+    bundle exec pod install --project-directory=Firestore/Example --repo-update
+    ;;
+
+  Firestore-cmake-*)
+    bundle exec pod install --project-directory=Example --repo-update
+    bundle exec pod install --project-directory=Firestore/Example \
+        --no-repo-update
+    ;;
+
+  *)
+    echo "Unknown project-method combo" 1>&2
+    echo "  PROJECT=$PROJECT" 1>&2
+    echo "  METHOD=$METHOD" 1>&2
+    exit 1
+    ;;
+esac
+