mirror of
https://gitlab.com/prpl-foundation/prplos/prplos.git
synced 2025-12-20 00:56:07 +08:00
5f5cf26405
Currently it's not easy to identify errors in CI job log output, so
let's introduce centralized logging helpers with colored output and
visual indicators to make errors more prominent.
The helpers are consolidated in .gitlab/scripts/helpers.sh and replace
inline logging in dut-init scripts. Error messages in cdrouter.yml and
testbed.yml now use log_error() for hopefully better visibility in CI
logs.
References: PCF-2157
Signed-off-by: Petr Štetiar <petr.stetiar@prplfoundation.org>
(cherry picked from commit a4ff6b0ccf)
228 lines
9.0 KiB
YAML
228 lines
9.0 KiB
YAML
.cdrouter:
|
|
stage: cdrouter
|
|
timeout: 40 minutes
|
|
image: "$CI_REGISTRY_IMAGE/$CI_DESIGNATED_BRANCH_SLUG/testbed:latest"
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule"'
|
|
when: never
|
|
- when: manual
|
|
variables:
|
|
CDROUTER_CONFIG_WIFI_BEACON_PHY: 'ax b g n'
|
|
CDROUTER_CONFIG_WIFI_BEACON_RSN_CIPHER: 'CCMP-128'
|
|
CDROUTER_CONFIG_WIFI_LAN_CHANNEL: 6
|
|
CDROUTER_CONFIG_WIFI_LAN_SSID: 'prplOS'
|
|
CDROUTER_TEST_CONFIG: generic
|
|
DUT_SLEEP_AFTER_BOOT: 90
|
|
TEST_SETUP_TEARDOWN_DURATION: 360
|
|
CDROUTER_LAN_INTERFACE: eth1
|
|
CDROUTER_WAN_INTERFACE: eth2
|
|
CDROUTER_ETH0_MAC_ADDRESS: 00:03:2d:49:2e:d8
|
|
CDROUTER_ETH0_MAC_ADDRESS_TESTBED_02: 00:03:2d:49:2e:c8
|
|
|
|
before_script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
- section_start "testbed-setup" "Testbed Setup" "collapsed"
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
|
|
sudo etherwake -i $TESTBED_MNG_INTERFACE $CDROUTER_ETH0_MAC_ADDRESS 2> /dev/null
|
|
else
|
|
sudo etherwake -i $TESTBED_MNG_INTERFACE $CDROUTER_ETH0_MAC_ADDRESS_TESTBED_02 2> /dev/null
|
|
fi
|
|
- sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null
|
|
- sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null
|
|
- sleep 10
|
|
|
|
- mkdir -p ~/.ssh; chmod 700 ~/.ssh
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
|
|
eval $(ssh-agent -s)
|
|
echo "$TESTBED_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
|
ssh-keyscan $TESTBED_UART_RELAY_HOST > ~/.ssh/known_hosts 2> /dev/null
|
|
chmod 644 ~/.ssh/known_hosts
|
|
fi
|
|
|
|
- |
|
|
echo "Using firmware image $TFTP_IMAGE_PATH/$TFTP_IMAGE_FILENAME"
|
|
cp "$TFTP_IMAGE_PATH/$TFTP_IMAGE_FILENAME" "$TESTBED_TFTP_PATH"
|
|
|
|
if [ -n "$TFTP_IMAGE_UNPACK_COMMAND" ]; then
|
|
echo "Running $TFTP_IMAGE_UNPACK_COMMAND"
|
|
eval "$TFTP_IMAGE_UNPACK_COMMAND"
|
|
fi
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- section_end "testbed-setup"
|
|
- section_start "dut-bringup" "DUT Bringup" "collapsed"
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET boot_into shell
|
|
- sudo ip link set $TESTBED_WAN_INTERFACE down 2> /dev/null
|
|
- >
|
|
.gitlab/scripts/testbed-device.py
|
|
--target $LABGRID_TARGET check_network
|
|
--network lan
|
|
--remote-host $TARGET_LAN_TEST_HOST
|
|
|
|
- sleep $DUT_SLEEP_AFTER_BOOT
|
|
|
|
- section_end "dut-bringup"
|
|
- section_start "dut-system-info" "DUT System Information" "collapsed"
|
|
|
|
- echo "Waiting for SSH connection availability with ${DUT_SSH_CONNECT_TIMEOUT}sec timeout, with $DUT_SSH_CONNECT_ATTEMPTS attempts..."
|
|
- time ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectionAttempts=$DUT_SSH_CONNECT_ATTEMPTS -o ConnectTimeout=$DUT_SSH_CONNECT_TIMEOUT root@$TARGET_LAN_IP 'exit 0'
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "System accessible over SSH, $(uptime -p)"' || true
|
|
- ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt 2> /dev/null || true
|
|
- ssh root@$TARGET_LAN_IP "ubus call system board" | tee system-$LABGRID_TARGET.json
|
|
|
|
- section_end "dut-system-info"
|
|
- section_start "dut-init" "DUT Initialization" "collapsed"
|
|
|
|
- >
|
|
for dut_init_script in $DUT_INIT_SCRIPTS;
|
|
do echo "Running $dut_init_script"; "${CI_PROJECT_DIR}/$dut_init_script";
|
|
done
|
|
|
|
- section_end "dut-init"
|
|
- section_start "cdrouter-configuration" "CDRouter Configuration" "collapsed"
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
|
|
export CDROUTER_CONFIG_WIFI_LAN_CHANNEL=11
|
|
export CDROUTER_CONFIG_WIFI_LAN_SSID=prplOS-2g-11
|
|
else
|
|
export CDROUTER_CONFIG_WIFI_LAN_CHANNEL=6
|
|
export CDROUTER_CONFIG_WIFI_LAN_SSID=prplOS-2g-6
|
|
fi
|
|
|
|
- .gitlab/scripts/testbed-cdrouter.py check_connectivity
|
|
- .gitlab/scripts/testbed-cdrouter.py package_stop
|
|
- .gitlab/scripts/testbed-cdrouter.py wait_for_netif $CDROUTER_LAN_INTERFACE
|
|
- .gitlab/scripts/testbed-cdrouter.py wait_for_netif $CDROUTER_WAN_INTERFACE
|
|
|
|
script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
|
|
fi
|
|
|
|
- export TEST_PACKAGE="$(echo $CI_JOB_NAME | sed 's/\(\w\) .*/\1/')"
|
|
- .gitlab/scripts/testbed-cdrouter.py config_import $CDROUTER_TEST_CONFIG
|
|
- .gitlab/scripts/testbed-cdrouter.py package_import $TEST_PACKAGE.gz
|
|
|
|
- section_end "cdrouter-configuration"
|
|
- >
|
|
ssh root@$TARGET_LAN_IP 'f=$(logread | grep "procd.*init complete"); echo "${f:-Warning: init is not complete yet,} $(uptime -p)"' || true
|
|
- ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing starts, $(uptime -p)"' || true
|
|
- sudo ip link set $TESTBED_LAN_INTERFACE down 2> /dev/null
|
|
|
|
- |
|
|
TEST_CUTOFF_TIMEOUT=$((CI_JOB_TIMEOUT - DUT_SLEEP_AFTER_BOOT - TEST_SETUP_TEARDOWN_DURATION))
|
|
echo "Job timeout: $CI_JOB_TIMEOUT seconds, test cutoff timeout: $TEST_CUTOFF_TIMEOUT seconds"
|
|
|
|
- section_start "test-execution" "Test Execution" "expanded"
|
|
- |
|
|
.gitlab/scripts/testbed-cdrouter.py \
|
|
package_run $TEST_PACKAGE \
|
|
--device generic \
|
|
--configuration $CDROUTER_TEST_CONFIG \
|
|
--system-info system-$LABGRID_TARGET.json \
|
|
--timeout ${TEST_CUTOFF_TIMEOUT} || cdrouter_exit_code=$?
|
|
- section_end "test-execution"
|
|
|
|
- sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null || true
|
|
- sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null || true
|
|
- sleep 5
|
|
|
|
- python3 -m cram --verbose .gitlab/tests/cram/post/generic/50-check-for-crashes.t || cram_exit_code=$?
|
|
|
|
- >
|
|
if test -n "$cdrouter_exit_code"; then
|
|
log_error "CDRouter exited with $cdrouter_exit_code exit code."
|
|
exit "$cdrouter_exit_code"
|
|
fi
|
|
|
|
if test -n "$cram_exit_code"; then
|
|
log_error "cram exited with $cram_exit_code exit code."
|
|
exit "$cram_exit_code"
|
|
fi
|
|
|
|
after_script:
|
|
- >
|
|
test -f *-logdir.tgz &&
|
|
mkdir -p cdrouter-results &&
|
|
tar xf *-logdir.tgz --strip-components=1 --directory=cdrouter-results &&
|
|
cat cdrouter-results/final.txt &&
|
|
.gitlab/scripts/cdrouter-pretty-failures.awk cdrouter-results/*.txt
|
|
|
|
- mkdir -p ~/.ssh; chmod 700 ~/.ssh
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
|
|
eval $(ssh-agent -s)
|
|
echo "$TESTBED_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
|
ssh-keyscan $TESTBED_UART_RELAY_HOST > ~/.ssh/known_hosts 2> /dev/null
|
|
chmod 644 ~/.ssh/known_hosts
|
|
fi
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- >
|
|
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15 root@$TARGET_LAN_IP 'exit 0' || {
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_dump_system_state
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_recover_ssh_access
|
|
} || true
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing finished, $(uptime -p)"' || true
|
|
- >
|
|
ssh root@$TARGET_LAN_IP exit && {
|
|
ssh root@$TARGET_LAN_IP ps > processes-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP dmesg > dmesg-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP opkg list > opkg_list-$LABGRID_TARGET.txt
|
|
script --quiet --command "ssh -t root@$TARGET_LAN_IP /bin/getDebugInformation --all"
|
|
scp -r root@${TARGET_LAN_IP}:/tmp/debug-*_all.tar.gz . > /dev/null
|
|
scp -r root@${TARGET_LAN_IP}:/etc etc > /dev/null
|
|
scp -r root@${TARGET_LAN_IP}:/tmp/beerocks/logs prplmesh_beerocks_logs > /dev/null
|
|
} || true
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET power off
|
|
- mv console_$LABGRID_TARGET console_$LABGRID_TARGET.txt || true
|
|
- tar xzf debug-*_all.tar.gz || true
|
|
- mv debug-*.txt debug-information-$LABGRID_TARGET.txt || true
|
|
- ( [ $(tar -tf coredumps_all.tar.gz | wc -l) -eq 0 ] && rm coredumps_all.tar.gz || true ) 2>/dev/null
|
|
- mv coredumps_all.tar.gz coredumps-$LABGRID_TARGET.tar.gz 2> /dev/null || true
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
|
|
fi
|
|
- .gitlab/scripts/testbed-cdrouter.py package_stop
|
|
|
|
artifacts:
|
|
expire_in: 1 month
|
|
when: always
|
|
paths:
|
|
- cdrouter-results
|
|
- etc
|
|
- prplmesh_beerocks_logs
|
|
- debug-information-$LABGRID_TARGET.txt
|
|
- processes-$LABGRID_TARGET.txt
|
|
- dmesg-$LABGRID_TARGET.txt
|
|
- syslog-$LABGRID_TARGET.txt
|
|
- system-$LABGRID_TARGET.json
|
|
- console_$LABGRID_TARGET.txt
|
|
- opkg_list-$LABGRID_TARGET.txt
|
|
- coredumps-$LABGRID_TARGET.tar.gz
|