scripts/tests/test_suites.sh - third_party/github/project-chip/connectedhomeip - Git at Google

 #!/usr/bin/env bash

 #
 # Copyright (c) 2020 Project CHIP Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #

 # Fail if one of our sub-commands fails.
 set -e

 # Fail if anything in a pipeline fails, not just the last command (which for
 # us tends to be 'tee').
 set -o pipefail

 declare INPUT_ARGS=$*

 declare -i iterations=2
 declare -i delay=0
 declare -i node_id=0x12344321
 declare -i background_pid=0
 declare -i use_netns=0
 declare -i root_remount=0
 declare -i clean_netns=0
 declare test_case_wrapper=()

 usage() {
     echo "test_suites.sh [-a APPLICATION] [-i ITERATIONS] [-h] [-s CASE_NAME] [-w COMMAND] [-d DELAY]"
     echo "  -a APPLICATION: runs chip-tool against 'chip-<APPLICATION>-app' (default: all-clusters)"
     echo "  -d DELAY: milliseconds to wait before running an individual test step (default: $delay)"
     echo "  -h: this help message"
     echo "  -i ITERATIONS: number of iterations to run (default: $iterations)"
     echo "  -s CASE_NAME: runs single test case name (e.g. Test_TC_OO_2_2"
     echo "                for Test_TC_OO_2_2.yaml) (by default, all are run)"
     echo "  -w COMMAND: prefix all instantiations with a command (e.g. valgrind) (default: '')"
     echo "  -n Use linux netns to isolate app and tool executables"
     echo "  -c execute a netns cleanup and exit"
     echo "  -r Execute a remount (INTERNAL USE for netns)"
     echo ""
     exit 0
 }

 declare app_run_prefix=""
 declare tool_run_prefix=""

 netns_setup() {
     # 2 virtual hosts: for app and for the tool
     ip netns add app
     ip netns add tool

     # create links for switch to net connections
     ip link add eth-app type veth peer name eth-app-switch
     ip link add eth-tool type veth peer name eth-tool-switch

     # link the connections together
     ip link set eth-app netns app
     ip link set eth-tool netns tool

     ip link add name br1 type bridge
     ip link set br1 up
     ip link set eth-app-switch master br1
     ip link set eth-tool-switch master br1

     # mark connections up
     ip netns exec app ip addr add 10.10.10.1/24 dev eth-app
     ip netns exec app ip link set dev eth-app up
     ip netns exec app ip link set dev lo up
     ip link set dev eth-app-switch up

     ip netns exec tool ip addr add 10.10.10.2/24 dev eth-tool
     ip netns exec tool ip link set dev eth-tool up
     ip netns exec tool ip link set dev lo up
     ip link set dev eth-tool-switch up

     # Force IPv6 to use ULAs that we control
     ip netns exec tool ip -6 addr flush eth-tool
     ip netns exec app ip -6 addr flush eth-app
     ip netns exec tool ip -6 a add fd00:0:1:1::2/64 dev eth-tool
     ip netns exec app ip -6 a add fd00:0:1:1::3/64 dev eth-app

     # TODO(andy314): IPv6 does Duplicate Address Detection even though
     # we know these addresses are isolated. For a while IPv6 addresses
     # will be in 'transitional' state and cannot be used.
     #
     # This sleep waits for the addresses to become 'global'. Ideally
     # we should loop/wait here instead.
     sleep 2
 }

 netns_cleanup() {
     ip netns del app || true
     ip netns del tool || true
     ip link del br1 || true

     # attempt  to delete orphaned items just in case
     ip link del eth-tool || true
     ip link del eth-tool-switch || true
     ip link del eth-app || true
     ip link del eth-app-switch || true
 }

 while getopts a:d:i:hs:w:ncr flag; do
     case "$flag" in
     a) application=$OPTARG ;;
     d) delay=$OPTARG ;;
     h) usage ;;
     i) iterations=$OPTARG ;;
     s) single_case=$OPTARG ;;
     w) test_case_wrapper=("$OPTARG") ;;
     n) use_netns=1 ;;
     c) clean_netns=1 ;;
     r) root_remount=1 ;;
     esac
 done

 if [[ $clean_netns != 0 ]]; then
     echo "Cleaning network namespaces"
     netns_cleanup
     exit 0
 fi

 if [[ $root_remount != 0 ]]; then
     echo 'Creating a separate mount'
     mount --make-private /
     mount -t tmpfs tmpfs /run
 fi

 if [[ $use_netns != 0 ]]; then
     echo "Using network namespaces"

     if [[ `id -u` -ne 0 ]]; then
         echo 'Executing in a new namespace: ' $0 -r $INPUT_ARGS
         unshare --map-root-user -n -m $0 -r $INPUT_ARGS
         exit 0
     else
         if [[ $root_remount -eq 0 ]]; then
           # Running as root may be fine in docker/vm however this is not advised
           # on workstations as changes are global and harder to undo
           echo 'Running as root: this changes global network namespaces, not ideal'
         fi
     fi

     netns_setup

     app_run_prefix="ip netns exec app"
     tool_run_prefix="ip netns exec tool"

     trap netns_cleanup EXIT
 fi

 if [[ $application == "tv" ]]; then
     declare test_filenames="${single_case-TV_*}.yaml"
     cp examples/tv-app/linux/include/endpoint-configuration/chip_tv_config.ini /tmp/chip_tv_config.ini
 # in case there's no application argument
 # always default to all-cluters app
 else
     application="all-clusters"
     declare test_filenames="${single_case-Test*}.yaml"
 fi
 declare -a test_array="($(find src/app/tests/suites -type f -name "$test_filenames" -not -name "*Simulated*" -exec basename {} .yaml \;))"

 if [[ $iterations == 0 ]]; then
     echo "Invalid iteration count: '$1'"
     exit 1
 fi

 echo "Running tests for application: $application, with iterations set to: $iterations and delay set to $delay"

 cleanup() {
     if [[ $background_pid != 0 ]]; then
         # In case we died on a failure before we cleaned up our background task.
         kill -9 "$background_pid" || true
     fi

     if [[ $use_netns != 0 ]]; then
         netns_cleanup
     fi
 }
 trap cleanup EXIT

 echo "Found tests:"
 for i in "${test_array[@]}"; do
     echo "   $i"
 done
 echo ""
 echo ""

 ulimit -c unlimited || true

 rm -rf /tmp/test_suites_app_logs
 mkdir -p /tmp/test_suites_app_logs

 declare -a iter_array="($(seq "$iterations"))"
 for j in "${iter_array[@]}"; do
     echo " ===== Iteration $j starting"
     for i in "${test_array[@]}"; do
         echo "  ===== Running test: $i"
         echo "          * Starting cluster server"
         rm -rf /tmp/chip_tool_config.ini
         # This part is a little complicated.  We want to
         # 1) Start chip-app in the background
         # 2) Pipe its output through tee so we can wait until it's ready for a
         #    PASE handshake.
         # 3) Save its pid off so we can kill it.
         #
         # The subshell with echoing of $! to a file descriptor and
         # then reading things out of there accomplishes item 3;
         # otherwise $! would be the last-started command which would
         # be the tee.  This part comes from https://stackoverflow.com/a/3786955
         # and better ideas are welcome.
         #
         # The stdbuf -o0 is to make sure our output is flushed through
         # tee expeditiously; otherwise it will buffer things up and we
         # will never see the string we want.

         application_log_file=/tmp/test_suites_app_logs/"$application-$i-$j"-log
         pairing_log_file=/tmp/test_suites_app_logs/pairing-"$application-$i-$j"-log
         chip_tool_log_file=/tmp/test_suites_app_logs/chip-tool-"$application-$i-$j"-log
         touch "$application_log_file"
         touch "$pairing_log_file"
         touch "$chip_tool_log_file"
         rm -rf /tmp/pid
         (
             ${app_run_prefix} stdbuf -o0 "${test_case_wrapper[@]}" out/debug/standalone/chip-"$application"-app &
             echo $! >&3
         ) 3>/tmp/pid | tee "$application_log_file" &
         while ! grep -q "Server Listening" "$application_log_file"; do
             :
         done
         # Now read $background_pid from /tmp/pid; presumably it's
         # landed there by now.  If we try to read it immediately after
         # kicking off the subshell, sometimes we try to do it before
         # the data is there yet.
         background_pid="$(</tmp/pid)"
         # Only look for commissionable nodes if dns-sd is available
         if command -v dns-sd &>/dev/null; then
             echo "          * [CI DEBUG] Looking for commissionable Nodes"
             # Ignore the error that timeout generates
             cat <(timeout 1 dns-sd -B _matterc._udp)
         fi
         echo "          * Pairing to device"
         ${tool_run_prefix} "${test_case_wrapper[@]}" out/debug/standalone/chip-tool pairing qrcode "$node_id" MT:D8XA0CQM00KA0648G00 | tee "$pairing_log_file"
         echo "          * Starting test run: $i"
         ${tool_run_prefix} "${test_case_wrapper[@]}" out/debug/standalone/chip-tool tests "$i" "$node_id" --delayInMs "$delay" | tee "$chip_tool_log_file"
         # Prevent cleanup trying to kill a process we already killed.
         temp_background_pid=$background_pid
         background_pid=0
         kill -9 "$temp_background_pid"
         echo "  ===== Test complete: $i"
     done
     echo " ===== Iteration $j completed"
 done
	#!/usr/bin/env bash

	#
	# Copyright (c) 2020 Project CHIP Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#

	# Fail if one of our sub-commands fails.
	set -e

	# Fail if anything in a pipeline fails, not just the last command (which for
	# us tends to be 'tee').
	set -o pipefail

	declare INPUT_ARGS=$*

	declare -i iterations=2
	declare -i delay=0
	declare -i node_id=0x12344321
	declare -i background_pid=0
	declare -i use_netns=0
	declare -i root_remount=0
	declare -i clean_netns=0
	declare test_case_wrapper=()

	usage() {
	echo "test_suites.sh [-a APPLICATION] [-i ITERATIONS] [-h] [-s CASE_NAME] [-w COMMAND] [-d DELAY]"
	echo " -a APPLICATION: runs chip-tool against 'chip-<APPLICATION>-app' (default: all-clusters)"
	echo " -d DELAY: milliseconds to wait before running an individual test step (default: $delay)"
	echo " -h: this help message"
	echo " -i ITERATIONS: number of iterations to run (default: $iterations)"
	echo " -s CASE_NAME: runs single test case name (e.g. Test_TC_OO_2_2"
	echo " for Test_TC_OO_2_2.yaml) (by default, all are run)"
	echo " -w COMMAND: prefix all instantiations with a command (e.g. valgrind) (default: '')"
	echo " -n Use linux netns to isolate app and tool executables"
	echo " -c execute a netns cleanup and exit"
	echo " -r Execute a remount (INTERNAL USE for netns)"
	echo ""
	exit 0
	}

	declare app_run_prefix=""
	declare tool_run_prefix=""

	netns_setup() {
	# 2 virtual hosts: for app and for the tool
	ip netns add app
	ip netns add tool

	# create links for switch to net connections
	ip link add eth-app type veth peer name eth-app-switch
	ip link add eth-tool type veth peer name eth-tool-switch

	# link the connections together
	ip link set eth-app netns app
	ip link set eth-tool netns tool

	ip link add name br1 type bridge
	ip link set br1 up
	ip link set eth-app-switch master br1
	ip link set eth-tool-switch master br1

	# mark connections up
	ip netns exec app ip addr add 10.10.10.1/24 dev eth-app
	ip netns exec app ip link set dev eth-app up
	ip netns exec app ip link set dev lo up
	ip link set dev eth-app-switch up

	ip netns exec tool ip addr add 10.10.10.2/24 dev eth-tool
	ip netns exec tool ip link set dev eth-tool up
	ip netns exec tool ip link set dev lo up
	ip link set dev eth-tool-switch up

	# Force IPv6 to use ULAs that we control
	ip netns exec tool ip -6 addr flush eth-tool
	ip netns exec app ip -6 addr flush eth-app
	ip netns exec tool ip -6 a add fd00:0:1:1::2/64 dev eth-tool
	ip netns exec app ip -6 a add fd00:0:1:1::3/64 dev eth-app

	# TODO(andy314): IPv6 does Duplicate Address Detection even though
	# we know these addresses are isolated. For a while IPv6 addresses
	# will be in 'transitional' state and cannot be used.
	#
	# This sleep waits for the addresses to become 'global'. Ideally
	# we should loop/wait here instead.
	sleep 2
	}

	netns_cleanup() {
	ip netns del app \|\| true
	ip netns del tool \|\| true
	ip link del br1 \|\| true

	# attempt to delete orphaned items just in case
	ip link del eth-tool \|\| true
	ip link del eth-tool-switch \|\| true
	ip link del eth-app \|\| true
	ip link del eth-app-switch \|\| true
	}

	while getopts a:d:i:hs:w:ncr flag; do
	case "$flag" in
	a) application=$OPTARG ;;
	d) delay=$OPTARG ;;
	h) usage ;;
	i) iterations=$OPTARG ;;
	s) single_case=$OPTARG ;;
	w) test_case_wrapper=("$OPTARG") ;;
	n) use_netns=1 ;;
	c) clean_netns=1 ;;
	r) root_remount=1 ;;
	esac
	done

	if [[ $clean_netns != 0 ]]; then
	echo "Cleaning network namespaces"
	netns_cleanup
	exit 0
	fi

	if [[ $root_remount != 0 ]]; then
	echo 'Creating a separate mount'
	mount --make-private /
	mount -t tmpfs tmpfs /run
	fi

	if [[ $use_netns != 0 ]]; then
	echo "Using network namespaces"

	if [[ `id -u` -ne 0 ]]; then
	echo 'Executing in a new namespace: ' $0 -r $INPUT_ARGS
	unshare --map-root-user -n -m $0 -r $INPUT_ARGS
	exit 0
	else
	if [[ $root_remount -eq 0 ]]; then
	# Running as root may be fine in docker/vm however this is not advised
	# on workstations as changes are global and harder to undo
	echo 'Running as root: this changes global network namespaces, not ideal'
	fi
	fi

	netns_setup

	app_run_prefix="ip netns exec app"
	tool_run_prefix="ip netns exec tool"

	trap netns_cleanup EXIT
	fi

	if [[ $application == "tv" ]]; then
	declare test_filenames="${single_case-TV_*}.yaml"
	cp examples/tv-app/linux/include/endpoint-configuration/chip_tv_config.ini /tmp/chip_tv_config.ini
	# in case there's no application argument
	# always default to all-cluters app
	else
	application="all-clusters"
	declare test_filenames="${single_case-Test*}.yaml"
	fi
	declare -a test_array="($(find src/app/tests/suites -type f -name "$test_filenames" -not -name "Simulated" -exec basename {} .yaml \;))"

	if [[ $iterations == 0 ]]; then
	echo "Invalid iteration count: '$1'"
	exit 1
	fi

	echo "Running tests for application: $application, with iterations set to: $iterations and delay set to $delay"

	cleanup() {
	if [[ $background_pid != 0 ]]; then
	# In case we died on a failure before we cleaned up our background task.
	kill -9 "$background_pid" \|\| true
	fi

	if [[ $use_netns != 0 ]]; then
	netns_cleanup
	fi
	}
	trap cleanup EXIT

	echo "Found tests:"
	for i in "${test_array[@]}"; do
	echo " $i"
	done
	echo ""
	echo ""

	ulimit -c unlimited \|\| true

	rm -rf /tmp/test_suites_app_logs
	mkdir -p /tmp/test_suites_app_logs

	declare -a iter_array="($(seq "$iterations"))"
	for j in "${iter_array[@]}"; do
	echo " ===== Iteration $j starting"
	for i in "${test_array[@]}"; do
	echo " ===== Running test: $i"
	echo " * Starting cluster server"
	rm -rf /tmp/chip_tool_config.ini
	# This part is a little complicated. We want to
	# 1) Start chip-app in the background
	# 2) Pipe its output through tee so we can wait until it's ready for a
	# PASE handshake.
	# 3) Save its pid off so we can kill it.
	#
	# The subshell with echoing of $! to a file descriptor and
	# then reading things out of there accomplishes item 3;
	# otherwise $! would be the last-started command which would
	# be the tee. This part comes from https://stackoverflow.com/a/3786955
	# and better ideas are welcome.
	#
	# The stdbuf -o0 is to make sure our output is flushed through
	# tee expeditiously; otherwise it will buffer things up and we
	# will never see the string we want.

	application_log_file=/tmp/test_suites_app_logs/"$application-$i-$j"-log
	pairing_log_file=/tmp/test_suites_app_logs/pairing-"$application-$i-$j"-log
	chip_tool_log_file=/tmp/test_suites_app_logs/chip-tool-"$application-$i-$j"-log
	touch "$application_log_file"
	touch "$pairing_log_file"
	touch "$chip_tool_log_file"
	rm -rf /tmp/pid
	(
	${app_run_prefix} stdbuf -o0 "${test_case_wrapper[@]}" out/debug/standalone/chip-"$application"-app &
	echo $! >&3
	) 3>/tmp/pid \| tee "$application_log_file" &
	while ! grep -q "Server Listening" "$application_log_file"; do
	:
	done
	# Now read $background_pid from /tmp/pid; presumably it's
	# landed there by now. If we try to read it immediately after
	# kicking off the subshell, sometimes we try to do it before
	# the data is there yet.
	background_pid="$(</tmp/pid)"
	# Only look for commissionable nodes if dns-sd is available
	if command -v dns-sd &>/dev/null; then
	echo " * [CI DEBUG] Looking for commissionable Nodes"
	# Ignore the error that timeout generates
	cat <(timeout 1 dns-sd -B _matterc._udp)
	fi
	echo " * Pairing to device"
	${tool_run_prefix} "${test_case_wrapper[@]}" out/debug/standalone/chip-tool pairing qrcode "$node_id" MT:D8XA0CQM00KA0648G00 \| tee "$pairing_log_file"
	echo " * Starting test run: $i"
	${tool_run_prefix} "${test_case_wrapper[@]}" out/debug/standalone/chip-tool tests "$i" "$node_id" --delayInMs "$delay" \| tee "$chip_tool_log_file"
	# Prevent cleanup trying to kill a process we already killed.
	temp_background_pid=$background_pid
	background_pid=0
	kill -9 "$temp_background_pid"
	echo " ===== Test complete: $i"
	done
	echo " ===== Iteration $j completed"
	done