Travis fix to collect coredumps from build configuration subdirs - #720 (#776)

* Travis fix to collect coredumps from build configuration subdirs - #720

Previously, Travis would only attempt to collect coredumps from the last
directory the build script was in, or the home directory. This meant
that most coredumps were missed, and debug information was lost.

The new setup configures apport to watch for crashes on any executables
(by default it watches for executables from Ubuntu packages). These
crash dumps are picked up, extracted, and analysed using GDB to provide
a backtrace (this needs the executable to be passed to GDB as well so
that debug symbols are provided).

* Restructured .travis.yml to avoid underspecified builds.

.travis.yml

 language: cpp
 
+os: linux
 dist: bionic
-sudo: true
 
 branches:
   only:
     - master
 
-arch:
-  - amd64
-  - ppc64le
-  - s390x
-  - arm64
-
-matrix:
+# Note that we're not making use of the matrix expansion that Travis provides -
+# it doesn't work very well for our collection of builds that each require a
+# combination of environment variables and packages that need to be defined
+# per-job in the jobs.include list. Jobs defined in the include key don't
+# modify or extend job specifications from the matrix, and actually add a whole
+# new job to the matrix (!), so using the matrix would leave us with a
+# subset of jobs being under-specified.
+# To get around this, we're using YAML anchors and aliases to avoid repeating
+# config across different builds.
+
+# The following keys prefixed with an underscore are not directly interpreted by
+# Travis, and are used to provide a centralised config store for config that
+# gets referenced inside the jobs.include map.
+# The apt addon can take nested lists in its subkeys (and flattens them), so we
+# use that to our advantage to merge a list of generic packages with a list of
+# job-specific packages in YAML.
+_apt_pkgs: &_apt_pkgs
+  - 'apport'
+  - 'cmake'
+  - 'libssl-dev'
+  - 'libcurl4-openssl-dev'
+  - 'gdb'
+  - 'lcov'
+  - 'cppcheck'
+
+_apt_srcs: &_apt_srcs
+  - 'ubuntu-toolchain-r-test'
+
+_env_clang_4: &_env_clang_4
+  - COMPILER=clang++-4.0
+  - COV_TOOL=llvm-cov-4.0
+  - COV_TOOL_ARGS=gcov
+
+_env_clang_5: &_env_clang_5
+  - COMPILER=clang++-5.0
+  - COV_TOOL=llvm-cov-5.0
+  - COV_TOOL_ARGS=gcov
+
+_env_clang_6: &_env_clang_6
+  - COMPILER=clang++-6.0
+  - COV_TOOL=llvm-cov-6.0
+  - COV_TOOL_ARGS=gcov
+
+_env_gcc_7: &_env_gcc_7
+  - COMPILER=g++-7
+  - COV_TOOL=gcov-7
+  - COV_TOOL_ARGS=
+
+_env_gcc_8: &_env_gcc_8
+  - COMPILER=g++-8
+  - COV_TOOL=gcov-8
+  - COV_TOOL_ARGS=
+
+jobs:
   include:
   # Linux clang builds
-  - os: linux
-    compiler: clang
-    env:
-      - COMPILER=clang++-4.0
-      - COV_TOOL=llvm-cov-4.0
-      - COV_TOOL_ARGS=gcov
+  - arch: amd64
+    env: *_env_clang_4
     addons:
       apt:
-        sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-bionic-4.0']
-        packages: [ 'apport', 'cmake', 'clang-4.0', 'llvm-4.0-tools', 'libstdc++-6-dev', 'libssl-dev', 'libcurl4-openssl-dev', 'gdb', 'lcov', 'cppcheck' ]
-
-  - os: linux
-    compiler: clang
-    env:
-      - COMPILER=clang++-5.0
-      - COV_TOOL=llvm-cov-5.0
-      - COV_TOOL_ARGS=gcov
+        sources: 
+          - *_apt_srcs
+          - ['llvm-toolchain-bionic-4.0']
+        packages: 
+          - *_apt_pkgs
+          - ['clang-4.0', 'llvm-4.0-tools', 'libstdc++-6-dev']
+
+  - arch: amd64
+    env: *_env_clang_5
     addons:
       apt:
-        sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-bionic-5.0']
-        packages: [ 'apport', 'cmake', 'clang-5.0', 'llvm-5.0-tools', 'libstdc++-6-dev', 'libssl-dev', 'libcurl4-openssl-dev', 'gdb', 'lcov', 'cppcheck' ]
-
-  - os: linux
-    compiler: clang
-    env:
-      - COMPILER=clang++-6.0
-      - COV_TOOL=llvm-cov-6.0
-      - COV_TOOL_ARGS=gcov
+        sources:
+          - *_apt_srcs
+          - ['llvm-toolchain-bionic-5.0', 'fish']
+        packages: 
+          - *_apt_pkgs
+          - ['clang-5.0', 'llvm-5.0-tools', 'libstdc++-6-dev']
+
+  - arch: amd64
+    env: *_env_clang_6
     addons:
       apt:
-        sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-bionic-6.0']
-        packages: [ 'apport', 'cmake', 'clang-6.0', 'llvm-6.0-tools', 'libstdc++-6-dev', 'libssl-dev', 'libcurl4-openssl-dev', 'gdb', 'lcov', 'cppcheck' ]
+        sources:
+          - *_apt_srcs
+          - ['llvm-toolchain-bionic-6.0']
+        packages: 
+          - *_apt_pkgs
+          - ['clang-6.0', 'llvm-6.0-tools', 'libstdc++-6-dev']
 
   # Linux GCC builds
-  - os: linux
-    compiler: gcc
-    env:
-      - COMPILER=g++-7
-      - COV_TOOL=gcov-7
-      - COV_TOOL_ARGS=
+  - arch: amd64
+    env: *_env_gcc_7
+    addons:
+      apt:
+        sources:
+          - *_apt_srcs
+        packages: 
+          - *_apt_pkgs
+          - ['g++-7']
+
+  - arch: amd64
+    env: *_env_gcc_8
+    addons:
+      apt:
+        sources:
+          - *_apt_srcs
+        packages: 
+          - *_apt_pkgs
+          - ['g++-8']
+
+  - arch: ppc64le
+    env: *_env_gcc_8
     addons:
       apt:
-        sources: ['ubuntu-toolchain-r-test']
-        packages: ['apport', 'g++-7', 'libssl-dev', 'libcurl4-openssl-dev', 'gdb', 'lcov', 'cppcheck']
-
-  - os: linux
-    compiler: gcc
-    env:
-      - COMPILER=g++-8
-      - COV_TOOL=gcov-8
-      - COV_TOOL_ARGS=
+        sources:
+          - *_apt_srcs
+        packages: 
+          - *_apt_pkgs
+          - ['g++-8']
+
+  - arch: s390x
+    env: *_env_gcc_8
     addons:
       apt:
-        sources: ['ubuntu-toolchain-r-test']
-        packages: ['apport', 'g++-8', 'libssl-dev', 'libcurl4-openssl-dev', 'gdb', 'lcov', 'cppcheck']
+        sources:
+          - *_apt_srcs
+        packages: 
+          - *_apt_pkgs
+          - ['g++-8']
+
+  - arch: arm64
+    env: *_env_gcc_8
+    addons:
+      apt:
+        sources:
+          - *_apt_srcs
+        packages: 
+          - *_apt_pkgs
+          - ['g++-8']
 
 install:
   - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps"
@@ -84,6 +159,9 @@ before_script:
   - ulimit -c
   - ulimit -a -S
   - ulimit -a -H
+    # Make sure that crash reports are generated for all executables.
+  - mkdir -p ~/.config/apport
+  - printf '[main]\nunpackaged=true\n' >> ~/.config/apport/settings
 
   # Print debug system information
   - cat /proc/sys/kernel/core_pattern
@@ -130,17 +208,19 @@ script:
   - make -j 2
 
 after_failure:
-  - ls -lta /var/crash
-  - COREFILE=$(find ./ -maxdepth 1 -name "core*" -print | head -n 1) 
-  - if [[ -f "$COREFILE" ]]; then
-    gdb -c "$COREFILE" -ex "thread apply all bt" -ex "set pagination 0" -batch;
-    fi
-
+  - CRASHFILES=$(find /var/crash/ -mindepth 1 -maxdepth 1 -print)
+  - echo "$CRASHFILES"
+    # Unpack the apport crash file into a temporary directory and get the
+    # executable and core dump to pass to GDB.
+    # Call gdb with the absolute path since it isn't on the PATH.
+  - |
+    if [ -n "$CRASHFILES" ]; then echo "$CRASHFILES" | while IFS= read -r CRASH; do echo "Crash report: $CRASH"; DIR="/tmp/$(basename $CRASH).d"; apport-unpack "$CRASH" "$DIR"; EXE=$(cat $DIR/ExecutablePath); CORE="$DIR/CoreDump"; echo "Coredump $CORE for $EXE"; /usr/bin/gdb $EXE $CORE -ex "thread apply all bt" -ex "set pagination 0" -batch; done; fi
 
 after_success:
-- cd ../Build-Debug
-- sudo su -c "echo 'if [ \"\$1\" = \"-v\" ] ; then $COV_TOOL --version ; else $COV_TOOL $COV_TOOL_ARGS \$@ ; fi' > /usr/local/bin/cov-tool" && sudo chmod +x /usr/local/bin/cov-tool
-- lcov --capture --gcov-tool cov-tool --directory . --output-file coverage.info
-- lcov --remove coverage.info '/usr/*' '*tests/*' '*third-party/*' --output-file coverage.info
-- lcov --list coverage.info
-- bash <(curl -s https://codecov.io/bash) -f coverage.info || echo "Codecov did not collect coverage reports"
+  - cd ../Build-Debug
+  - sudo su -c "echo 'if [ \"\$1\" = \"-v\" ] ; then $COV_TOOL --version ; else $COV_TOOL $COV_TOOL_ARGS \$@ ; fi' > /usr/local/bin/cov-tool" && sudo chmod +x /usr/local/bin/cov-tool
+  - lcov --capture --gcov-tool cov-tool --directory . --output-file coverage.info
+  - lcov --remove coverage.info '/usr/*' '*tests/*' '*third-party/*' --output-file coverage.info
+  - lcov --list coverage.info
+  - bash <(curl -s https://codecov.io/bash) -f coverage.info || echo "Codecov did not collect coverage reports"
+