Allow UI tests to fail in CI.

This commit makes passing compile UI tests optional, allowing the CI to
succeed even when UI tests fail. This change was made because UI tests
are highly susceptible to false negatives due to benign rustc compiler
output changes. A failure resulting from such a benign change inhibits
progress in the main branch due to failing PR testing which would have
otherwise passed.

.github/workflows/ci.yml

@@ -8,10 +8,12 @@ env:
 jobs:
   test:
     name: "${{ matrix.platform.name }} ${{ matrix.test.name }} (${{ matrix.platform.toolchain }})"
+    runs-on: ${{ matrix.platform.distro }}
 
     strategy:
       fail-fast: false
       matrix:
+        fallible: [false]
         platform:
           - { name: Linux, distro: ubuntu-latest, toolchain: stable }
           - { name: Windows, distro: windows-latest, toolchain: stable }
@@ -26,8 +28,12 @@ jobs:
             test: { name: Core, flag: "--core" }
           - platform: { name: Linux, distro: ubuntu-latest, toolchain: stable }
             test: { name: Release, flag: "--release" }
-
-    runs-on: ${{ matrix.platform.distro }}
+          - platform: { name: Linux, distro: ubuntu-latest, toolchain: stable }
+            test: { name: UI, flag: "--ui" }
+            fallible: true
+          - platform: { name: Linux, distro: ubuntu-latest, toolchain: nightly }
+            test: { name: UI, flag: "--ui" }
+            fallible: true
 
     steps:
       - name: Checkout Sources
@@ -80,5 +86,6 @@ jobs:
           key: ${{ matrix.test.name }}
 
       - name: Run Tests
+        continue-on-error: ${{ matrix.fallible }}
         run: ./scripts/test.sh ${{ matrix.test.flag }} -q
         shell: bash

contrib/db_pools/codegen/tests/ui-fail.rs

@@ -1,4 +1,5 @@
 #[test]
+#[ignore]
 fn ui() {
     let path = match version_check::is_feature_flaggable() {
         Some(true) => "ui-fail-nightly",

contrib/sync_db_pools/codegen/tests/ui-fail.rs

@@ -1,4 +1,5 @@
 #[test]
+#[ignore]
 fn ui() {
     let path = match version_check::is_feature_flaggable() {
         Some(true) => "ui-fail-nightly",

core/codegen/tests/ui-fail.rs

@@ -1,4 +1,5 @@
 #[test]
+#[ignore]
 fn ui() {
     let path = match version_check::is_feature_flaggable() {
         Some(true) => "ui-fail-nightly",

scripts/test.sh

@@ -145,7 +145,7 @@ function test_examples() {
   indir "${EXAMPLES_DIR}" $CARGO update
   ROCKET_SECRET_KEY="itlYmFR2vYKrOmFhupMIn/hyB6lYCCTXz4yaQX89XVg=" \
     indir "${EXAMPLES_DIR}" $CARGO test --all $@
-  }
+}
 
 function test_default() {
   echo ":: Building and testing core libraries..."
@@ -160,6 +160,11 @@ function test_default() {
   indir "${FUZZ_ROOT}" $CARGO check --all --all-features $@
 }
 
+function test_ui() {
+  echo ":: Testing compile-time UI output..."
+  indir "${PROJECT_ROOT}" $CARGO test ui --all --all-features -- --ignored $@
+}
+
 function run_benchmarks() {
   echo ":: Running benchmarks..."
   indir "${BENCHMARKS_ROOT}" $CARGO update
@@ -173,7 +178,7 @@ fi
 
 # The kind of test we'll be running.
 TEST_KIND="default"
-KINDS=("contrib" "benchmarks" "core" "examples" "default" "all")
+KINDS=("contrib" "benchmarks" "core" "examples" "default" "ui" "all")
 
 if [[ " ${KINDS[@]} " =~ " ${1#"--"} " ]]; then
   TEST_KIND=${1#"--"}
@@ -208,17 +213,20 @@ case $TEST_KIND in
   examples) test_examples $@ ;;
   default) test_default $@ ;;
   benchmarks) run_benchmarks $@ ;;
+  ui) test_ui $@ ;;
   all)
     test_default $@ & default=$!
     test_examples $@ & examples=$!
     test_core $@ & core=$!
     test_contrib $@ & contrib=$!
+    test_ui $@ & ui=$!
 
     failures=()
     if ! wait $default ; then failures+=("DEFAULT"); fi
     if ! wait $examples ; then failures+=("EXAMPLES"); fi
     if ! wait $core ; then failures+=("CORE"); fi
     if ! wait $contrib ; then failures+=("CONTRIB"); fi
+    if ! wait $ui ; then failures+=("UI"); fi
 
     if [ ${#failures[@]} -ne 0 ]; then
       tput setaf 1;