mirror of
https://gitlab.com/prpl-foundation/prplos/prplos.git
synced 2025-12-20 00:56:07 +08:00
5f5cf26405
Currently it's not easy to identify errors in CI job log output, so
let's introduce centralized logging helpers with colored output and
visual indicators to make errors more prominent.
The helpers are consolidated in .gitlab/scripts/helpers.sh and replace
inline logging in dut-init scripts. Error messages in cdrouter.yml and
testbed.yml now use log_error() for hopefully better visibility in CI
logs.
References: PCF-2157
Signed-off-by: Petr Štetiar <petr.stetiar@prplfoundation.org>
(cherry picked from commit a4ff6b0ccf)
211 lines
8.4 KiB
YAML
211 lines
8.4 KiB
YAML
.testbed:
|
|
stage: run
|
|
image: "$CI_REGISTRY_IMAGE/$CI_DESIGNATED_BRANCH_SLUG/testbed:latest"
|
|
timeout: 38 minutes
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule"'
|
|
when: never
|
|
- when: on_success
|
|
variables:
|
|
DUT_WAN_INTERFACE: eth1
|
|
TESTBED_MNG_INTERFACE: enp1s0
|
|
TESTBED_LAN_INTERFACE: enp2s0
|
|
TESTBED_WAN_INTERFACE: enp3s0
|
|
TESTBED_TFTP_PATH: /var/lib/tftpboot
|
|
TFTP_IMAGE_PATH: bin/targets/$DUT_TARGET/$DUT_SUBTARGET
|
|
PACKAGE_DIR: bin/packages
|
|
DUT_ARCH_PACKAGES_PATH: $PACKAGE_DIR/${DUT_ARCH}_${DUT_CPU_TYPE}_${DUT_CPU_SUBTYPE}
|
|
TARGET_LAN_IP: 192.168.1.1
|
|
TARGET_LAN_TEST_HOST: 192.168.1.2
|
|
DUT_SLEEP_AFTER_BOOT: 90
|
|
DUT_SSH_CONNECT_TIMEOUT: 30
|
|
DUT_SSH_CONNECT_ATTEMPTS: 3
|
|
DUT_TR181_WAN_INTERFACE: Device.Ethernet.Link.2.
|
|
DUT_TR181_LAN_INTERFACE: Device.Ethernet.Link.3.
|
|
TEST_SETUP_TEARDOWN_DURATION: 360
|
|
CRAM_REMOTE_COMMAND: ssh root@$TARGET_LAN_IP
|
|
CRAM_REMOTE_COPY: scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
CRAM_TEST_SUITE: |
|
|
.gitlab/tests/cram/generic/acceleration-plan-components
|
|
.gitlab/tests/cram/$DUT_BOARD/acceleration-plan-components
|
|
.gitlab/tests/cram/generic
|
|
.gitlab/tests/cram/$DUT_BOARD
|
|
CRAM_TEST_SUITE_POST: |
|
|
.gitlab/tests/cram/post/generic
|
|
|
|
before_script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
- section_start "testbed-setup" "Testbed Setup" "collapsed"
|
|
|
|
- touch .run_failed
|
|
|
|
- sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null
|
|
- sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null
|
|
- sleep 10
|
|
|
|
- mkdir -p ~/.ssh; chmod 700 ~/.ssh
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
|
|
eval $(ssh-agent -s)
|
|
echo "$TESTBED_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
|
ssh-keyscan $TESTBED_UART_RELAY_HOST > ~/.ssh/known_hosts 2> /dev/null
|
|
chmod 644 ~/.ssh/known_hosts
|
|
fi
|
|
|
|
- |
|
|
echo "Using firmware image $TFTP_IMAGE_PATH/$TFTP_IMAGE_FILENAME"
|
|
cp "$TFTP_IMAGE_PATH/$TFTP_IMAGE_FILENAME" "$TESTBED_TFTP_PATH"
|
|
|
|
if [ -n "$TFTP_IMAGE_UNPACK_COMMAND" ]; then
|
|
echo "Running $TFTP_IMAGE_UNPACK_COMMAND"
|
|
eval "$TFTP_IMAGE_UNPACK_COMMAND"
|
|
fi
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- section_end "testbed-setup"
|
|
- section_start "dut-bringup" "DUT Bringup" "collapsed"
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET boot_into shell
|
|
- >
|
|
.gitlab/scripts/testbed-device.py
|
|
--target $LABGRID_TARGET check_network
|
|
--network lan
|
|
--remote-host $TARGET_LAN_TEST_HOST
|
|
|
|
- sleep $DUT_SLEEP_AFTER_BOOT
|
|
|
|
- section_end "dut-bringup"
|
|
- section_start "dut-system-info" "DUT System Information" "collapsed"
|
|
|
|
- echo "Waiting for SSH connection availability with ${DUT_SSH_CONNECT_TIMEOUT}sec timeout, with $DUT_SSH_CONNECT_ATTEMPTS attempts..."
|
|
- time ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectionAttempts=$DUT_SSH_CONNECT_ATTEMPTS -o ConnectTimeout=$DUT_SSH_CONNECT_TIMEOUT root@$TARGET_LAN_IP 'exit 0'
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "System accessible over SSH, $(uptime -p)"' || true
|
|
- ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt 2> /dev/null || true
|
|
- ssh root@$TARGET_LAN_IP "ubus call system board" | tee system-$LABGRID_TARGET.json
|
|
|
|
- section_end "dut-system-info"
|
|
|
|
script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
- section_start "test-execution" "Test Execution" "expanded"
|
|
|
|
- |
|
|
TEST_CUTOFF_TIMEOUT=$((CI_JOB_TIMEOUT - DUT_SLEEP_AFTER_BOOT - TEST_SETUP_TEARDOWN_DURATION))
|
|
echo "Job timeout: $CI_JOB_TIMEOUT seconds, test cutoff timeout: $TEST_CUTOFF_TIMEOUT seconds"
|
|
|
|
- |
|
|
rm -fr .run_check_failed
|
|
|
|
run_check() {
|
|
"$@" || {
|
|
local statuses=("${PIPESTATUS[@]}")
|
|
for status in "${statuses[@]}"; do
|
|
if [ "$status" -ne 0 ]; then
|
|
log_error "run_check: $* | FAILED! (status: $status)" | tee -a .run_check_failed
|
|
fi
|
|
done
|
|
}
|
|
}
|
|
|
|
run_check ssh root@$TARGET_LAN_IP 'f=$(logread | grep "procd.*init complete"); echo "${f:-Warning: init is not complete yet,} $(uptime -p)"'
|
|
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing starts, $(uptime -p)"'
|
|
run_check \
|
|
timeout --kill-after=10s --verbose ${TEST_CUTOFF_TIMEOUT}s \
|
|
python3 -m cram --xunit-file=cram-result-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE $CRAM_TEST_SUITE_EXTRA | tee cram-result-$LABGRID_TARGET.txt
|
|
run_check python3 -m cram --xunit-file=cram-result-post-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE_POST | tee --append cram-result-$LABGRID_TARGET.txt
|
|
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing finished, $(uptime -p)"'
|
|
|
|
- section_end "test-execution"
|
|
|
|
- |
|
|
[ -f .run_check_failed ] && cat .run_check_failed && exit 1
|
|
rm .run_failed
|
|
|
|
after_script:
|
|
- >-
|
|
curl --fail \
|
|
--output .gitlab/scripts/cram-xunit-merger.py \
|
|
https://gitlab.com/prpl-foundation/tooling/cram-xunit-merger/-/raw/9bc14318ab88c2a024ad0ff8bf09c21d048e1e3b/cram-xunit-merger.py
|
|
|
|
- python3 .gitlab/scripts/cram-xunit-merger.py \
|
|
--output "${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.junit.xml" \
|
|
"${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.xunit.xml" \
|
|
"${CI_PROJECT_DIR}/cram-result-post-$LABGRID_TARGET.xunit.xml" || true
|
|
|
|
- >
|
|
[ "$TMATE_ENABLE_SHELL" = "YES" ] && {
|
|
echo "Starting tmate session..."
|
|
tmate -n "${TMATE_SESSION_TOKEN}-$CI_JOB_ID" -k "$TMATE_API_KEY" -F new "ssh root@$TARGET_LAN_IP" > /tmp/tmate-session.log 2>&1
|
|
true
|
|
}
|
|
|
|
- mkdir -p ~/.ssh; chmod 700 ~/.ssh
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
|
|
eval $(ssh-agent -s)
|
|
echo "$TESTBED_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
|
ssh-keyscan $TESTBED_UART_RELAY_HOST > ~/.ssh/known_hosts 2> /dev/null
|
|
chmod 644 ~/.ssh/known_hosts
|
|
fi
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- >
|
|
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15 root@$TARGET_LAN_IP 'exit 0' || {
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_dump_system_state
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_recover_ssh_access
|
|
} || true
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- >
|
|
ssh root@$TARGET_LAN_IP exit && {
|
|
ssh root@$TARGET_LAN_IP ps > processes-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP dmesg > dmesg-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP opkg list > opkg_list-$LABGRID_TARGET.txt
|
|
script --quiet --command "ssh -t root@$TARGET_LAN_IP /bin/getDebugInformation --all"
|
|
scp -r root@${TARGET_LAN_IP}:/tmp/debug-*_all.tar.gz . > /dev/null
|
|
scp -r root@${TARGET_LAN_IP}:/etc etc > /dev/null
|
|
scp -r root@${TARGET_LAN_IP}:/tmp/beerocks/logs prplmesh_beerocks_logs > /dev/null
|
|
} || true
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET power off
|
|
- tar xzf debug-*_all.tar.gz || true
|
|
- mv debug-*.txt debug-information-$LABGRID_TARGET.txt || true
|
|
- ( [ $(tar -tf coredumps_all.tar.gz | wc -l) -eq 0 ] && rm coredumps_all.tar.gz || true ) 2>/dev/null
|
|
- mv coredumps_all.tar.gz coredumps-$LABGRID_TARGET.tar.gz 2> /dev/null || true
|
|
- mv console_$LABGRID_TARGET console_$LABGRID_TARGET.txt || true
|
|
|
|
artifacts:
|
|
expire_in: 1 month
|
|
when: always
|
|
paths:
|
|
- .gitlab/tests/cram/**/*.t.err
|
|
- etc
|
|
- prplmesh_beerocks_logs
|
|
- debug-information-$LABGRID_TARGET.txt
|
|
- processes-$LABGRID_TARGET.txt
|
|
- dmesg-$LABGRID_TARGET.txt
|
|
- syslog-$LABGRID_TARGET.txt
|
|
- system-$LABGRID_TARGET.json
|
|
- console_$LABGRID_TARGET.txt
|
|
- cram-result-*
|
|
- opkg_list-$LABGRID_TARGET.txt
|
|
- coredumps-$LABGRID_TARGET.tar.gz
|
|
reports:
|
|
junit: cram-result-$LABGRID_TARGET.junit.xml
|
|
|
|
.testbed true.cz:
|
|
extends: .testbed
|
|
variables:
|
|
TESTBED_UART_RELAY_HOST: uart-relay.testbed.vpn.true.cz
|