Allow UI tests to fail in CI.

This commit makes passing compile UI tests optional, allowing the CI to
succeed even when UI tests fail. This change was made because UI tests
are highly susceptible to false negatives due to benign rustc compiler
output changes. A failure resulting from such a benign change inhibits
progress in the main branch due to failing PR testing which would have
otherwise passed.
This commit is contained in:
Sergio Benitez 2023-01-30 14:15:24 -08:00
parent 162fafa0f1
commit ca4b38c0d0
5 changed files with 22 additions and 4 deletions

View File

@ -8,10 +8,12 @@ env:
jobs:
test:
name: "${{ matrix.platform.name }} ${{ matrix.test.name }} (${{ matrix.platform.toolchain }})"
runs-on: ${{ matrix.platform.distro }}
strategy:
fail-fast: false
matrix:
fallible: [false]
platform:
- { name: Linux, distro: ubuntu-latest, toolchain: stable }
- { name: Windows, distro: windows-latest, toolchain: stable }
@ -26,8 +28,12 @@ jobs:
test: { name: Core, flag: "--core" }
- platform: { name: Linux, distro: ubuntu-latest, toolchain: stable }
test: { name: Release, flag: "--release" }
runs-on: ${{ matrix.platform.distro }}
- platform: { name: Linux, distro: ubuntu-latest, toolchain: stable }
test: { name: UI, flag: "--ui" }
fallible: true
- platform: { name: Linux, distro: ubuntu-latest, toolchain: nightly }
test: { name: UI, flag: "--ui" }
fallible: true
steps:
- name: Checkout Sources
@ -80,5 +86,6 @@ jobs:
key: ${{ matrix.test.name }}
- name: Run Tests
continue-on-error: ${{ matrix.fallible }}
run: ./scripts/test.sh ${{ matrix.test.flag }} -q
shell: bash

View File

@ -1,4 +1,5 @@
#[test]
#[ignore]
fn ui() {
let path = match version_check::is_feature_flaggable() {
Some(true) => "ui-fail-nightly",

View File

@ -1,4 +1,5 @@
#[test]
#[ignore]
fn ui() {
let path = match version_check::is_feature_flaggable() {
Some(true) => "ui-fail-nightly",

View File

@ -1,4 +1,5 @@
#[test]
#[ignore]
fn ui() {
let path = match version_check::is_feature_flaggable() {
Some(true) => "ui-fail-nightly",

View File

@ -160,6 +160,11 @@ function test_default() {
indir "${FUZZ_ROOT}" $CARGO check --all --all-features $@
}
function test_ui() {
echo ":: Testing compile-time UI output..."
indir "${PROJECT_ROOT}" $CARGO test ui --all --all-features -- --ignored $@
}
function run_benchmarks() {
echo ":: Running benchmarks..."
indir "${BENCHMARKS_ROOT}" $CARGO update
@ -173,7 +178,7 @@ fi
# The kind of test we'll be running.
TEST_KIND="default"
KINDS=("contrib" "benchmarks" "core" "examples" "default" "all")
KINDS=("contrib" "benchmarks" "core" "examples" "default" "ui" "all")
if [[ " ${KINDS[@]} " =~ " ${1#"--"} " ]]; then
TEST_KIND=${1#"--"}
@ -208,17 +213,20 @@ case $TEST_KIND in
examples) test_examples $@ ;;
default) test_default $@ ;;
benchmarks) run_benchmarks $@ ;;
ui) test_ui $@ ;;
all)
test_default $@ & default=$!
test_examples $@ & examples=$!
test_core $@ & core=$!
test_contrib $@ & contrib=$!
test_ui $@ & ui=$!
failures=()
if ! wait $default ; then failures+=("DEFAULT"); fi
if ! wait $examples ; then failures+=("EXAMPLES"); fi
if ! wait $core ; then failures+=("CORE"); fi
if ! wait $contrib ; then failures+=("CONTRIB"); fi
if ! wait $ui ; then failures+=("UI"); fi
if [ ${#failures[@]} -ne 0 ]; then
tput setaf 1;