[github-actions] upload core dump, so libs and binaries for crashed programs (#5861)

This commit adds action to upload core dumps, so libs and binaries for
1.2 certification. This is intended to solve issues where programs
intermittently crash however hard to reproduce it.

This is added first for thread-1-2 and low-power-packet-verification
first as an experimental work. We can add same actions for other items
when it's necessary.

.github/workflows/simulation-1.2.yml

@@ -70,13 +70,27 @@ jobs:
         ./script/test build
     - name: Run
       run: |
+        ulimit -c unlimited
+        ./script/test prepare_coredump_upload
         ./script/test unit
         ./script/test cert_suite tests/scripts/thread-cert/v1_2_*
+    - name: Copy Shared Libraries
+      if: ${{ failure() }}
+      run: |
+          ./script/test copy_so_lib
     - uses: actions/upload-artifact@v2
       if: ${{ failure() }}
       with:
         name: thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-pcaps
         path: "*.pcap"
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: core-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}
+        path: |
+          ./build/cmake/openthread-simulation-1.2/examples/apps/cli/ot-cli-*
+          ./ot-core-dump/*
+          ./so-lib/*
     - name: Keep-1-2-only
       run: |
         ./script/test tar simulation 1.1
@@ -136,10 +150,16 @@ jobs:
         ./script/test get_thread_wireshark
     - name: Run
       run: |
+        ulimit -c unlimited
+        ./script/test prepare_coredump_upload
         for i in {1..10}
         do
           ./script/test cert_suite ./tests/scripts/thread-cert/v1_2_LowPower*.py
         done
+    - name: Copy Shared Libraries
+      if: ${{ failure() }}
+      run: |
+          ./script/test copy_so_lib
     - uses: actions/upload-artifact@v2
       if: ${{ failure() }}
       with:
@@ -147,6 +167,15 @@ jobs:
         path: |
           *.pcap
           *.json
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: core-packet-verification-low-power
+        path: |
+          ./build/cmake/openthread-simulation-1.2/examples/apps/cli/ot-cli-*
+          ./build/cmake/openthread-simulation-1.1/examples/apps/cli/ot-cli-*
+          ./ot-core-dump/*
+          ./so-lib/*
     - name: Keep-1-2-only
       run: |
         ./script/test tar simulation 1.2-bbr

CONTRIBUTING.md

@@ -129,6 +129,40 @@ This will trigger continuous-integration checks using GitHub Actions. You can vi
 
 Once you've validated that all continuous-integration checks have passed, go to the page for your fork on GitHub, select your development branch, and click the pull request button. If you need to make any adjustments to your pull request, just push the updates to GitHub. Your pull request will automatically track the changes on your development branch and update.
 
+#### Checks fail
+
+Once you've submitted a pull request, all continuous-integration checks are triggered again. If some of these checks fail, it could be either problems with the pull request or an intermittent failure of some test cases. For more information on the failure, check the output and download artifacts. (After all jobs in one group are completed, an `Artifacts` button appears beside the `Re-run` jobs button.) If the failure is intermittent, the check will usually pass after rerunning once or twice.
+
+We want to eliminate intermittent failures as well, so when you experience such a failure, please log an issue and attach any relevant artifacts. If the artifacts are too big, provide the link of the failed run (do not rerun checks again, or it will be overwritten). Alternatively, upload the artifacts to a file-sharing service like Google Drive and share a link to it.
+
+##### Analyze core dumps in failed checks
+
+For some checks, core dumps for crashed programs are uploaded as artifacts in a failed check. Besides core dumps, binaries and shared libraries are also uploaded so that we can analyze the dumps locally. To analyze the dumps, download the artifact `core-xxx` and unzip it. The package is in the following format:
+
+```
+|-- build
+|   `-- cmake
+|       `-- openthread-simulation-1.2
+|           `-- examples
+|               `-- apps
+|                   `-- cli
+|                       |-- ot-cli-ftd
+|                       `-- ot-cli-mtd
+|-- ot-core-dump
+|   `-- corefile-ot-cli-ftd-11323-1606274703
+`-- so-lib
+    |-- ld-linux-x86-64.so.2
+    |-- libc.so.6
+    `-- libgcc_s.so.1
+```
+
+Once unzipped:
+
+1. `cd` to the unzipped directory
+2. Run `gdb build/cmake/openthread-simulation-1.2/examples/apps/cli/ot-cli-ftd ./ot-core-dump/corefile-ot-cli-ftd-XXX`.
+3. Set the absolute path of `so-lib`. In gdb, run `set solib-absolute-prefix /ABSOLUTE/PATH/TO/so-lib/`, then run `set solib-search-path /ABSOLUTE/PATH/TO/so-lib/`.
+4. In gdb, run `backtrace` or `bt`. Then you should see the stack of the crashed program. Find and fix the problem!
+
 ## Contributing Documentation
 
 Documentation undergoes the same review process as code and contributions may be mirrored on our [openthread.io](https://openthread.io) website. See the [Documentation Style Guide](/doc/STYLE_GUIDE.md) for more information on how to author and format documentation for contribution.

script/test

@@ -402,6 +402,20 @@ do_untar()
     rm "${build_dir:?}/${target_name}.tar"
 }
 
+do_prepare_coredump_upload()
+{
+    echo "$PWD/ot-core-dump/corefile-%e-%p-%t" | sudo tee /proc/sys/kernel/core_pattern
+    mkdir ot-core-dump
+}
+
+do_copy_so_lib()
+{
+    mkdir so-lib
+    cp /lib/x86_64-linux-gnu/libgcc_s.so.1 ./so-lib
+    cp /lib/x86_64-linux-gnu/libc.so.6 ./so-lib
+    cp /lib64/ld-linux-x86-64.so.2 ./so-lib
+}
+
 do_generate_coverage()
 {
     mkdir -p tmp/
@@ -554,6 +568,12 @@ main()
                 shift
                 do_untar "$@"
                 ;;
+            prepare_coredump_upload)
+                do_prepare_coredump_upload
+                ;;
+            copy_so_lib)
+                do_copy_so_lib
+                ;;
             generate_coverage)
                 shift
                 do_generate_coverage "$1"