From 37740ccd073f7e20ff8cb0d43d7f6b9cea5e69b3 Mon Sep 17 00:00:00 2001 From: Dan Rivett Date: Mon, 16 Mar 2015 23:35:49 -0700 Subject: [PATCH 1/5] Updating the asserts so if the assert fails the rc of the assert command is 1 and not 0. This allows us to check if an assertion has passed or not without having to wrap in a '_clean' and 'assert_end' block like the current tests.sh script does. See the good usage of this new behavior in new file 'test-extras.sh' in a subsequent commit. --- assert.sh | 1 + tests.sh | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assert.sh b/assert.sh index ffd2b95..e56df2c 100644 --- a/assert.sh +++ b/assert.sh @@ -139,6 +139,7 @@ _assert_fail() { fi tests_errors[$tests_failed]="$report" (( tests_failed++ )) || : + return 1 } skip_if() { diff --git a/tests.sh b/tests.sh index e8c1425..ccaab1a 100755 --- a/tests.sh +++ b/tests.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -e - . assert.sh assert "echo" # no output expected @@ -96,8 +94,9 @@ assert "_clean; x=0; assert 'x=1'; assert_raises 'x=2'; echo \$x" 0 assert "_clean; x=0; assert 'export x=1'; assert_raises 'export x=2'; echo \$x" 0 # options do not leak -assert_raises "set +e" -assert_raises "shopt -o errexit" +assert_raises "shopt -o errexit" 1 +assert_raises "set -e" +assert_raises "shopt -o errexit" 1 # skip properly resets all options assert_raises "_clean; set +e; skip; assert_raises false; shopt -o errexit" 1 assert_raises "_clean; set -e; skip; assert_raises false; shopt -o errexit" From 5656971c584594325a89cbd03b1d36d469ebb4ba Mon Sep 17 00:00:00 2001 From: Dan Rivett Date: Tue, 17 Mar 2015 19:46:40 -0700 Subject: [PATCH 2/5] Fixing bugs in assert and assert_raises which doesn't wrap the command in quotes. This breaks commands that pass in multiline input such as the following simple examples: assert 'echo "this is a multiline echo" | wc -l | tr -d "[[:space:]]"' "2" assert 'echo -e "this\nis a multiline echo" | wc -l | tr -d "[[:space:]]"' "2" With this commit, the following tests above works. --- assert.sh | 6 +++--- tests.sh | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/assert.sh b/assert.sh index e56df2c..0f3ea09 100644 --- a/assert.sh +++ b/assert.sh @@ -75,7 +75,7 @@ assert_end() { tests_endtime="$(date +%s%N)" # required visible decimal place for seconds (leading zeros if needed) local tests_time="$( \ - printf "%010d" "$(( ${tests_endtime/%N/000000000} + printf "%010d" "$(( ${tests_endtime/%N/000000000} - ${tests_starttime/%N/000000000} ))")" # in ns tests="$tests_ran ${*:+$* }tests" [[ -n "$DISCOVERONLY" ]] && echo "collected $tests." && _assert_reset && return @@ -103,7 +103,7 @@ assert() { (( tests_ran++ )) || : [[ -z "$DISCOVERONLY" ]] || return expected=$(echo -ne "${2:-}") - result="$(eval 2>/dev/null $1 <<< ${3:-})" || true + result="$(eval 2>/dev/null "$1" <<< ${3:-})" || true if [[ "$result" == "$expected" ]]; then [[ -z "$DEBUG" ]] || echo -n . return @@ -119,7 +119,7 @@ assert_raises() { (( tests_ran++ )) || : [[ -z "$DISCOVERONLY" ]] || return status=0 - (eval $1 <<< ${3:-}) > /dev/null 2>&1 || status=$? + (eval "$1" <<< ${3:-}) > /dev/null 2>&1 || status=$? expected=${2:-0} if [[ "$status" -eq "$expected" ]]; then [[ -z "$DEBUG" ]] || echo -n . diff --git a/tests.sh b/tests.sh index ccaab1a..2ceee66 100755 --- a/tests.sh +++ b/tests.sh @@ -160,4 +160,9 @@ date() { # date mock assert '_clean DEBUG=1 INVARIANT=; tests_starttime="0N"; assert_end' \ '\nall 0 tests passed in 123.000s.' unset -f date # bring back original date +# commit: Supporting multiline inputs +# We trim the output from wc as the BSD version (on Mac OS) contains leading spaces +assert 'echo "this +is a multiline echo" | wc -l | tr -d "[[:space:]]"' "2" +assert 'echo -e "this\nis a multiline echo" | wc -l | tr -d "[[:space:]]"' "2" assert_end regression From ce33aeae9bcd7dbf9c1086f6356d0bc1b2044117 Mon Sep 17 00:00:00 2001 From: Dan Rivett Date: Tue, 17 Mar 2015 20:01:29 -0700 Subject: [PATCH 3/5] Updating tests.sh to fix test failures when run on Mac OS X. The tests should now work multi-platform. --- tests.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests.sh b/tests.sh index 2ceee66..c18b903 100755 --- a/tests.sh +++ b/tests.sh @@ -58,7 +58,7 @@ assert "_clean; skip_if false; assert_raises true; assert_end;" \ assert "_clean; skip_if bash -c 'exit 1'; assert_raises false; assert_end;" \ "all 0 tests passed." # subshells and pipes can be used in skip as well (albeit escaped) -assert "_clean; skip_if 'cat /etc/passwd | grep \$(echo \$USER)'; +assert "_clean; skip_if 'id | grep \$(echo \$USER)'; assert_raises false; assert_end;" \ "all 0 tests passed." assert_end output @@ -152,7 +152,8 @@ _date=22; assert_end" "all 1 tests passed in 2.000s." # commit: supported formatting codes assert "echo %s" "%s" -assert "echo -n %s | wc -c" "2" +# We trim the output from wc as the BSD version (on Mac OS) contains leading spaces +assert "echo -n %s | wc -c | tr -d '[[:space:]]'" "2" # date with no nanosecond support date() { # date mock echo "123N" From 50816b23ee194b19c3e54a3d34c3e02b7f2b2194 Mon Sep 17 00:00:00 2001 From: Dan Rivett Date: Mon, 16 Mar 2015 23:37:02 -0700 Subject: [PATCH 4/5] Adding in assert-extras.sh which contains convenience assertions such as: assert_contains, assert_matches, assert_success, assert_failure. --- assert-extras.sh | 76 +++++++++++++++++++++++++++++ test-extras.sh | 122 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 assert-extras.sh create mode 100755 test-extras.sh diff --git a/assert-extras.sh b/assert-extras.sh new file mode 100644 index 0000000..b4f4769 --- /dev/null +++ b/assert-extras.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# assert-extras.sh 1.1 - supplementary bash unit testing functions +# Note: This script should be sourced together with assert.sh, +# it is dependent on the functionality provided by that script. + +# assert_success [stdin] +assert_success() { + assert_raises "$1" 0 "${2:-}" +} + +# assert_failure [stdin] +assert_failure() { + (( tests_ran++ )) || : + [[ -z "$DISCOVERONLY" ]] || return + status=0 + (eval "$1" <<< ${2:-}) > /dev/null 2>&1 || status=$? + if [[ "$status" != "0" ]]; then + [[ -z "$DEBUG" ]] || echo -n . + return + fi + _assert_fail "program terminated with a zero return code; expecting non-zero return code" \ + "$1" "$2" +} + +# assert_contains +assert_contains() { + _assert_with_grep '-F' "$@" +} + +# assert_matches +assert_matches() { + _assert_with_grep '-E' "$@" +} + +# assert_startswith +assert_startswith() { + assert_success "[[ '$($1)' == '$2'* ]]" +} + +# assert_endswith +assert_endswith() { + assert_success "[[ '$($1)' == *'$2' ]]" +} + +# _assert_with_grep +_assert_with_grep() { + local grep_modifier="$1" + local output="$($2)" + shift 2 + + while [ $# != 0 ]; do + assert_raises "echo '$output' | $GREP $grep_modifier '$1'" 0 || return 1 + shift + done +} + +# Returns the resolved command, preferring any gnu-versions of the cmd (prefixed with 'g') on +# non-Linux systems such as Mac OS, and falling back to the standard version if not. +_cmd() { + local cmd="$1" + + local gnu_cmd="g$cmd" + local gnu_cmd_found=$(which "$gnu_cmd" 2> /dev/null) + if [ "$gnu_cmd_found" ]; then + echo "$gnu_cmd_found" + else + if [ "$(uname)" == 'Darwin' ]; then + echo "Warning: Cannot find gnu version of command '$cmd' ($gnu_cmd) on path." \ + "Falling back to standard command" >&2 + fi + echo "cmd" + fi +} + +GREP=$(_cmd grep) diff --git a/test-extras.sh b/test-extras.sh new file mode 100755 index 0000000..3247264 --- /dev/null +++ b/test-extras.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +source assert.sh +source assert-extras.sh + +assert "echo foo" "foo" +assert_raises "true" 0 +assert_raises "exit 127" 127 + +assert_end sanity + +### +### assert_success tests +### + +# Tests expecting success +assert_success "true" +assert_success "echo foo" +assert_success "cat" "foo" + +# Tests expecting failure +assert_raises 'assert_success "false"' 1 +assert_raises 'assert_success "exit 1"' 1 + +assert_end assert_success + +### +### assert_failure tests +### + +# Tests expecting success +assert_failure "false" +assert_failure "exit 1" +assert_failure "exit -1" +assert_failure "exit 42" +assert_failure "exit -42" + +# Tests expecting failure +assert_raises 'assert_failure "true"' 1 +assert_raises 'assert_failure "echo foo"' 1 + +assert_end assert_failure + +### +### assert_contains tests +### + +# Tests expecting success +assert_contains "echo foo" "foo" +assert_contains "echo foobar" "foo" +assert_contains "echo foo bar" "foo" +assert_contains "echo foo bar" "bar" +assert_contains "echo foo bar" "foo bar" + +# Tests expecting failure +assert_failure 'assert_contains "echo foo" "foot"' +assert_failure 'assert_contains "echo foo" "f.."' + +# Multi-word argument tests +assert_contains "echo foo bar" "foo bar" +assert_failure 'assert_contains "echo foo; echo bar" "foo bar"' + +# Multi-argument tests +assert_contains "echo foo bar baz" "foo" "baz" +assert_failure 'assert_contains "echo foo bar baz" "bar" "foo baz"' + +assert_end assert_contains + +### +### assert_matches tests +### + +# Tests expecting success +assert_matches "echo foo" "f.." +assert_matches "echo foobar" "f." +assert_matches "echo foo bar" "^foo bar$" +assert_matches "echo foo bar" "[az ]+" + +# Tests expecting failure +assert_failure 'assert_matches "echo foot" "foo$"' + +# Multi-word argument tests +assert_matches "echo foo bar" "foo .*" +assert_failure 'assert_matches "echo foo; echo bar" "foo .*"' + +# Multi-argument tests +assert_matches "echo foo bar baz" "^f.." "baz$" +assert_failure 'assert_matches "echo foo bar baz" "bar" "foo baz"' + +assert_end assert_matches + +### +### assert_startswith tests +### + +# Tests expecting success +assert_startswith "echo foo" "f" +assert_startswith "echo foo" "foo" +assert_startswith "echo foo; echo bar" "foo" + +# Tests expecting failure +assert_failure 'assert_startswith "echo foo" "oo"' +assert_failure 'assert_startswith "echo foo; echo bar" "foo bar"' +assert_failure 'assert_startswith "echo foo" "."' + +assert_end assert_startswith + +### +### assert_endswith tests +### + +# Tests expecting success +assert_endswith "echo foo" "oo" +assert_endswith "echo foo" "foo" +assert_endswith "echo foo; echo bar" "bar" + +# Tests expecting failure +assert_failure 'assert_endswith "echo foo" "f"' +assert_failure 'assert_endswith "echo foo; echo bar" "foo bar"' +assert_failure 'assert_endswith "echo foo" "."' + +assert_end assert_endswith \ No newline at end of file From 0667d5691a1e79ddc6a39d37b4dcfc67ba8af92a Mon Sep 17 00:00:00 2001 From: Dan Rivett Date: Sat, 21 Mar 2015 15:13:18 -0700 Subject: [PATCH 5/5] Keep track to the total number of tests ran and failed in an entire test run, not just in a test suite. --- assert.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/assert.sh b/assert.sh index 0f3ea09..0b1d2d5 100644 --- a/assert.sh +++ b/assert.sh @@ -94,6 +94,8 @@ assert_end() { echo "$tests_failed of $tests failed$report_time." fi tests_failed_previous=$tests_failed + tests_ran_total=$(($tests_ran_total + $tests_ran)) + tests_failed_total=$(($tests_failed_total + $tests_failed)) [[ $tests_failed -gt 0 ]] && tests_suite_status=1 _assert_reset } @@ -179,6 +181,8 @@ _skip() { _assert_reset : ${tests_suite_status:=0} # remember if any of the tests failed so far +: ${tests_ran_total:=0} # remember the total number of tests ran (inc. failures) +: ${tests_failed_total:=0} # remember the total number of test failures _assert_cleanup() { local status=$? # modify exit code if it's not already non-zero