run-checks

#!/bin/bash -eu

export LANG=C.UTF-8
export LANGUAGE=en

this_file="$0"
COVERMODE=${COVERMODE:-atomic}
COVERAGE_SUFFIX=${GO_BUILD_TAGS:-notags}
COVERAGE_OUT=${COVERAGE_OUT:-.coverage/coverage-$COVERAGE_SUFFIX.cov}
CHANGED_FILES=${CHANGED_FILES:-""}
TIMEOUT=${TIMEOUT:-15}
AUTO_INSTALL=${AUTO_INSTALL:-1}     # Automatically install supported tool dependencies (default: install)
IGNORE_MISSING=${IGNORE_MISSING:-0} # Do not error on missing tool dependencies (default: error)

tool_version() {
    local tool=$1
    case "$tool" in
    go)
        $tool version
        ;;
    shellcheck)
        $tool --version | grep version
        ;;
    golangci-lint) ;& # fallthrough
    python3) ;&       # fallthrough
    pytest-3) ;&      # fallthrough
    clang-format)
        $tool --version
        ;;
    *)
        return 1
        ;;
    esac
}

tool_install() {
    local tool=$1
    case "$tool" in
    golangci-lint)
        go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.0.2 >/dev/null
        ;;
    go) ;&         # fallthrough
    shellcheck) ;& # fallthrough
    python3) ;&    # fallthrough
    pytest-3) ;&   # fallthrough
    clang-format) ;;
    *)
        return 1
        ;;
    esac
}

tool_configure() {
    local tool=$1
    case "$tool" in
    go)
        # If GOPATH is set in the shell environment, the path will be reflected
        # in $(go env GOPATH). If no shell path was set, Go will default the internal
        # GOPATH to $HOME/go. Note that GOPATH may contain a colon delimited set
        # of paths, so in order to run any binary from any of the installed GOPATHs
        # we must add all the possible bin locations.
        GOBINS=$(go env GOPATH | sed 's|:|/bin:|g' | sed 's|.*|\0/bin|g')
        export PATH="$PATH:$GOBINS"

        # when *not* running inside github, and go-1.18 is available, use it
        if [ "${CI:-}" != "true" ] && [ -e "/usr/lib/go-1.18/bin" ]; then
            export PATH=/usr/lib/go-1.18/bin:"${PATH}"
	    echo "WARNING: Forcing the use of Go 1.18 (preferred version for this project)" >&2
        fi
        ;;
    shellcheck) ;&    # fallthrough
    golangci-lint) ;& # fallthrough
    python3) ;&       # fallthrough
    pytest-3) ;&      # fallthrough
    clang-format) ;;
    *)
        return 1
        ;;
    esac
}

install_conf_show_tool() {
    local tool=$1
    local auto_install=$2
    local ignore_missing=$3

    echo ">> Depends on \"$tool\""

    if [ "$auto_install" -eq 1 ] && ! command -v "$tool" >/dev/null; then # ignore-tool
        if ! tool_install "$tool"; then
            echo "WARNING: Cannot install tool $tool" >&2
        fi
    fi

    if ! executable=$(command -v "$tool"); then # ignore-tool
        if [ "$ignore_missing" -ne 1 ]; then
            echo "ERROR: Tool $tool is not available" >&2
            return 1
        else
            echo "WARNING: Tool $tool is not available" >&2
        fi
    else
        if ! tool_configure "$tool"; then
            echo "ERROR: Cannot configure $tool" >&2
            return 1
        fi

        if ! version=$(tool_version "$tool"); then
            echo "ERROR: Cannot retrieve version of $tool" >&2
            return 1
        else
            echo "Version: $version"
            echo "Executable: $executable"
        fi
    fi
}

verify_tools() {
    local auto_install=$1
    local ignore_missing=$2

    # find tool dependencies but ignore those marked with `# check-tools-ignore`
    tools=$(grep -v '# ignore-tool' "$this_file" | grep -oP 'command -v \K[\w-]+' | sort -u)

    # check go first that includes setting up go paths required to find or install
    # other go based tools, error early when missing
    install_conf_show_tool "go" "$auto_install" "0"

    # check other tools
    for tool in $tools; do
        install_conf_show_tool "$tool" "$auto_install" "$ignore_missing"
    done
}

missing_interface_spread_test() {
    snap_yaml="tests/lib/snaps/test-snapd-policy-app-consumer/meta/snap.yaml"
    core_snap_yaml="tests/main/interfaces-many-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml"
    classic_snap_yaml="tests/main/interfaces-many-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml"
    for iface in $(go run ./tests/lib/list-interfaces.go); do
        search="plugs: \\[ $iface \\]"
        case "$iface" in
        bool-file | gpio | pwm | dsp | netlink-driver | hidraw | i2c | iio | serial-port | spi | confdb)
            # skip gadget provided interfaces for now
            continue
            ;;
        cuda-driver-libs | egl-driver-libs)
            # skip interfaces with plug side only in rootfs for now
            continue
            ;;
        dbus | content)
            search="interface: $iface"
            ;;
        autopilot)
            search='plugs: \[ autopilot-introspection \]'
            ;;
        esac

        # check if a standalone test already exists and that it at least
        # connects and disconnects the interface
        dedicated_test=$(find tests/main/ -maxdepth 1 -name "interfaces-$iface")
        if [ -n "$dedicated_test" ]; then
            if grep -q "$search" "$snap_yaml"; then
                echo "Dedicated test '$dedicated_test' found for '$iface'." >&2
                echo "Please remove '$iface' from '$snap_yaml'." >&2
                exit 1
            fi
            # dedicated test already exists, skip high-level test check below
            continue
        fi

        # check if high-level minimal test exists for interface
        if ! grep -q "$search" "$snap_yaml"; then
            echo "Missing high-level test for interface '$iface'. Please add to:" >&2
            echo "* $snap_yaml" >&2
            echo "* $core_snap_yaml (if needed)" >&2
            echo "* $classic_snap_yaml (if needed)" >&2
            exit 1
        fi
    done
}

CURRENT_TRAP="true"
EXIT_CODE=99

store_exit_code() {
    EXIT_CODE=$?
}

exit_with_exit_code() {
    exit $EXIT_CODE
}

addtrap() {
    CURRENT_TRAP="$CURRENT_TRAP ; $1"
    # shellcheck disable=SC2064
    trap "store_exit_code; $CURRENT_TRAP ; exit_with_exit_code" EXIT
}

endmsg() {
    if [ $EXIT_CODE -eq 0 ]; then
        p="success.txt"
        m="All good, what could possibly go wrong."
    else
        p="failure.txt"
        m="Crushing failure and despair."
    fi
    echo
    if [ -t 1 ] && [ -z "$STATIC" ]; then
        cat "data/$p"
    else
        echo "$m"
    fi
}
addtrap endmsg

short=
STATIC=
UNIT=

case "${1:-all}" in
all)
    STATIC=1
    UNIT=1
    ;;
--static)
    STATIC=1
    ;;
--unit)
    UNIT=1
    ;;
--short-unit)
    UNIT=1
    short="-short"
    ;;
*)
    echo "Wrong flag ${1}. To run a single suite use --static, --unit."
    exit 1
    ;;
esac

echo "> Verify tool dependencies"
verify_tools "$AUTO_INSTALL" "$IGNORE_MISSING"

if [ "$STATIC" = 1 ]; then
    ./get-deps.sh
    echo "> Running static tests"

    echo ">> Checking docs"
    ./mdlint.py ./*.md ./**/*.md

    # XXX: remove once we can use an action, see workflows/test.yaml for
    #      details why we still use this script
    if [ -n "${GITHUB_PULL_REQUEST_TITLE:-}" ]; then
        echo ">> Checking pull request summary"
        ./check-pr-title.py "${GITHUB_PULL_REQUEST_TITLE}"
    else
        echo ">> Skipping pull request summary check: not a pull request"
    fi

    echo ">> [Git] Checking commit author/committer name for unicode"
    ./check-commit-email.py

    if [ -z "${SKIP_GOFMT:-}" ]; then
        echo ">> [Go] Checking go formatting"
        fmt=""
        for dir in $(go list -f '{{.Dir}}' ./...); do
            s="$(gofmt -s -d "$dir" || true)"
            if [ -n "$s" ]; then
                fmt="$s\\n$fmt"
            fi
        done
        if [ -n "$fmt" ]; then
            echo "Formatting wrong in following files:"
            # shellcheck disable=SC2001
            echo "$fmt" | sed -e 's/\\n/\n/g'
            exit 1
        fi
    else
        echo ">> [Go] Skipping go formatting check"
    fi

    echo '>> [Go] Checking for usages of http.Status*'
    got=""
    for dir in $(go list -f '{{.Dir}}' ./...); do
        s="$(grep -nP 'http\.Status(?!Text)' "$dir"/*.go || true)"
        if [ -n "$s" ]; then
            got="$s\\n$got"
        fi
    done

    if [ -n "$got" ]; then
        echo 'Usages of http.Status*, we prefer the numeric values directly:' >&2
        echo "$got" >&2
        exit 1
    fi

    echo ">> [Go] Checking for direct usages of math/rand"
    got=""
    for dir in $(go list -f '{{.Dir}}' ./...); do
        # shellcheck disable=SC2063
        s="$(grep -nP --exclude '*_test.go' --exclude 'randutil/*.go' math/rand "$dir"/*.go || true)"
        if [ -n "$s" ]; then
            got="$s\\n$got"
        fi
    done

    if [ -n "$got" ]; then
        echo 'Direct usages of math/rand, we prefer randutil:' >&2
        echo "$got" >&2
        exit 1
    fi

    echo ">> [Go] Checking for usages of deprecated io/ioutil"
    got=""
    for dir in $(go list -f '{{.Dir}}' ./...); do
        # shellcheck disable=SC2063
        s="$(grep -nP io/ioutil "$dir"/*.go || true)"
        if [ -n "$s" ]; then
            got="$s\\n$got"
        fi
    done

    if [ -n "$got" ]; then
        echo 'Found usages of deprecated io/ioutil, please use "io" or "os" equivalents' >&2
        echo "$got" >&2
        exit 1
    fi

    if command -v shellcheck >/dev/null; then
        exclude_tools_path=tests/lib/external/snapd-testing-tools
        echo ">> [Bash] Checking shell scripts"
        if [ -n "$CHANGED_FILES" ]; then
            echo "Checking just the changed bash files"
            echo "Changed files: $CHANGED_FILES"
            # shellcheck disable=SC2086
            INITIAL_FILES="$(file -N $CHANGED_FILES | awk -F": " '$2~/shell.script/{print $1}')"
        else
            echo "Checking all the bash files"
            INITIAL_FILES="$( (git ls-files -z 2>/dev/null || find . \( -name .git -o -name vendor -o -name c-vendor \) -prune -o -print0) | xargs -0 file -N | awk -F": " '$2~/shell.script/{print $1}')"
        fi

        echo "Filtering files"
        FILTERED_FILES=
        for file in $INITIAL_FILES; do
            if ! echo "$file" | grep -q "$exclude_tools_path"; then
                FILTERED_FILES="$FILTERED_FILES $file"
            fi
        done
        if [ -n "$FILTERED_FILES" ]; then
            echo "Running shellcheck for files: $FILTERED_FILES"
            # shellcheck disable=SC2086
            shellcheck -x $FILTERED_FILES
        else
            echo "Skipping shellcheck, no files to check"
        fi

        regexp='GOPATH(?!%%:\*)(?!:)[^= ]*/'
        if grep -qPr --exclude HACKING.md --exclude 'Makefile.*' --exclude-dir .git --exclude-dir vendor "$regexp"; then
            echo "Using GOPATH as if it were a single entry and not a list:" >&2
            grep -PHrn -C1 --color=auto --exclude HACKING.md --exclude 'Makefile.*' --exclude-dir .git --exclude-dir vendor "$regexp"
            echo "Use GOHOME, or {GOPATH%%:*}, instead." >&2
            exit 1
        fi
        unset regexp

        # also run spread-shellcheck
        FILTERED_FILES="spread.yaml"
        if [ -n "$CHANGED_FILES" ]; then
            # shellcheck disable=SC2086
            for changed_file in $CHANGED_FILES; do
                if [[ $changed_file == */task.yaml ]]; then
                    FILTERED_FILES="$FILTERED_FILES $(pwd)/$changed_file"
                fi
            done
        else
            FILTERED_FILES="$FILTERED_FILES tests"
        fi
        # XXX: exclude core20-preseed test as its environment block confuses shellcheck, and it's not possible to disable shellcheck there.
        # shellcheck disable=SC2086
        ./tests/lib/external/snapd-testing-tools/utils/spread-shellcheck $FILTERED_FILES --exclude "$exclude_tools_path" --exclude "tests/nested/manual/core20-preseed"
    else
        echo ">> [Bash] Skipping shellcheck check"
    fi

    echo ">> [Spread] Checking all interfaces have a spread test"
    missing_interface_spread_test

    echo ">> [Spread] Checking for incorrect multiline strings in spread tests"
    badmultiline=$(find tests -name 'task.yaml' -print0 -o -name 'spread.yaml' -print0 |
        xargs -0 grep -R -n -E '(restore*|prepare*|execute|debug):\s*$' || true)
    if [ -n "$badmultiline" ]; then
        echo "Incorrect multiline strings at the following locations:" >&2
        echo "$badmultiline" >&2
        exit 1
    fi

    echo ">> [Spread] Checking for potentially incorrect use of MATCH -v"
    badMATCH=$(find tests -name 'task.yaml' -print0 -o -name 'spread.yaml' -print0 |
        xargs -0 grep -R -n -E 'MATCH +-v' || true)
    if [ -n "$badMATCH" ]; then
        echo "Potentially incorrect use of MATCH -v at the following locations:" >&2
        echo "$badMATCH" >&2
        exit 1
    fi

    # FIXME: re-add staticcheck with a matching version for the used go-version

    if [ -z "${SKIP_TESTS_FORMAT_CHECK:-}" ] || [ "$SKIP_TESTS_FORMAT_CHECK" = 0 ]; then
        echo ">> [Spread] Checking tests formatting"
        CHANGED_TESTS=""
        FILTERED_TESTS=""
        EXCLUDE_PATH=tests/lib/external/snapd-testing-tools
        if [ -n "$CHANGED_FILES" ]; then
            # shellcheck disable=SC2086
            for changed_file in $CHANGED_FILES; do
                if [[ $changed_file == */task.yaml ]]; then
                    CHANGED_TESTS="$CHANGED_TESTS $changed_file"
                fi
            done
        fi
        # shellcheck disable=SC2086
        for test in $CHANGED_TESTS; do
            if ! echo "$test" | grep -q "$EXCLUDE_PATH"; then
                if [ -z "$FILTERED_TESTS" ]; then
                    FILTERED_TESTS="$test"
                else
                    FILTERED_TESTS="$FILTERED_TESTS $test"
                fi
            fi
        done
        if [ -n "$FILTERED_TESTS" ]; then
            # shellcheck disable=SC2086
            ./tests/lib/external/snapd-testing-tools/utils/check-test-format --tests $FILTERED_TESTS
        fi
    else
        echo ">> [Spread] Skipping tests formatting check"
    fi

    echo ">> [Go] Checking for usages of !=, == or Equals with ErrNoState"
    if got=$(grep -n -R -E "(\!=|==|Equals,) (state\.)?ErrNoState" --include=*.go); then
        echo "Don't use equality checks with ErrNoState, use errors.Is() instead" >&2
        echo "$got" >&2
        exit 1
    fi

    if gcil=$(command -v golangci-lint) || [ -z "${SKIP_GOLANGCI_LINT:-}" ]; then
        echo ">> [Go] Checking golangci-lint"
        if echo "$gcil" | grep -q '/snap/bin/'; then
            # golangci-lint was installed from the snap
            if snap refresh --list | grep -q golangci-lint; then
                echo "WARNING: your golangci-lint snap is out of date. The CI will install a fresh version, which may differ from yours." >&2
            fi
            if ! snap list golangci-lint | grep -q latest; then
                echo "WARNING: your golangci-lint snap is not installed from the latest/* channel." >&2
            fi
        fi

        # only linters can be disabled, formatters require configuration change
        disable=""
        if [ -n "${SKIP_INEFFASSIGN:-}" ]; then
            disable+=" ineffassign"
        fi
        if [ -n "${SKIP_MISSPELL:-}" ]; then
            disable+=" misspell"
        fi
        if [ -n "${SKIP_NAKEDRET:-}" ]; then
            disable+=" nakedret"
        fi

        enabled=$(golangci-lint linters | awk '
        /^Enabled by your configuration linters:/ { in_section=1; next }
        /^Disabled by your configuration linters:/ { in_section=0 }
        in_section && NF {
        sub(/:.*/, "", $1)
        linters = linters ? linters "," $1 : $1
        }
        END { print linters }')

        echo ">> [Go] Enabled golangci-lint linters: $enabled"

        if [ -n "${disable:-}" ]; then
            disable="$(echo "$disable" | sed 's/^ *//' | tr ' ' ',')"
            echo ">> [Go] Disabling the following golangci-lint linters on request: $disable"
            golangci-lint --path-prefix= run --disable "$disable"
        else
            # don't run with --new-from-rev as the diff might not be enough to tell
            # the change introduces problems (e.g., removing the last call to a function)
            golangci-lint --path-prefix= run
        fi
    else
        echo ">> [Go] Skipping golangci-lint check"
    fi

    echo ">> [C] Checking C source code formatting"
    if command -v clang-format >/dev/null; then
        current=$(pwd)
        cd cmd/
        ./autogen.sh
        if ! make fmt-check; then
            echo "C files are not formatted correctly, run 'make fmt'" >&2
            exit 1
        fi
        cd "$current"
    else
        echo ">> [C] Skipping C source code formatting check"
    fi
fi

if [ "$UNIT" = 1 ]; then
    ./get-deps.sh
    echo "> Running unit tests"

    tags=
    race=
    timeout="${TIMEOUT}m"
    if [ -n "${GO_BUILD_TAGS:-}" ]; then
        echo "Using build tags: $GO_BUILD_TAGS"
        tags="-tags $GO_BUILD_TAGS"
    fi
    if [ -n "${GO_TEST_RACE:-}" ]; then
        echo "Using go test -race"
        race="-race"
        timeout="$((TIMEOUT * 2))m"
    fi

    echo Building
    # shellcheck disable=SC2086
    go build -v $tags $race github.com/snapcore/snapd/...

    # tests
    echo ">> [Go] Running go unit tests from $PWD"
    if [ "$short" = "-short" ] || [ -n "${SKIP_COVERAGE:-}" ]; then
        echo ">> [Go] Skipping coverage"
        # shellcheck disable=SC2046,SC2086
        GOTRACEBACK=1 go test ./... $tags $race $short -timeout $timeout
    else
        coverage="-coverprofile=$COVERAGE_OUT -covermode=$COVERMODE"
        echo ">> [Go] Checking coverage with params: $coverage"
        mkdir -p "$(dirname "$COVERAGE_OUT")"
        # shellcheck disable=SC2046,SC2086
        GOTRACEBACK=1 go test ./... $tags $race -timeout $timeout $coverage
    fi
    # python unit test for mountinfo.query and version-compare
    if command -v python3; then
        echo ">> [Python] Running python3 unit tests for mountinfo.query and version-compare"
        python3 ./tests/lib/tools/mountinfo.query --run-unit-tests
        python3 ./tests/lib/tools/version-compare --run-unit-tests
    else
        echo ">> [Python] Skipping python3 unit tests for mountinfo.query and version-compare"
    fi
    if command -v pytest-3; then
        echo ">> [Python] Running pytest-3 test for release-tools"
        PYTHONDONTWRITEBYTECODE=1 pytest-3 ./release-tools
    else
        echo ">> [Python] Skipping pytest-3 test for release-tools"
    fi
fi

if [ -n "${SKIP_UNCLEAN:-}" ]; then
    echo "> [Git] Skipping leftover files in git tree check"
else
    echo "> [Git] Checking for leftover files in git tree"
    UNCLEAN="$(git status -s | grep '^??')" || true
    if [ -n "$UNCLEAN" ]; then
        cat >&2 <<EOF

There are files left in the git tree after the tests:

$UNCLEAN
EOF
        exit 1
    fi
fi

if [ -n "${SKIP_DIRTY_CHECK:-}" ]; then
    echo "> [Git] Skipping dirty check"
    exit 0
fi
echo "> [Git] Checking for dirty build tree"
if git describe --always --dirty | grep -q dirty; then
    echo "Build tree is dirty" >&2
    git diff >&2
    exit 1
fi