From 6acbe63aea2b9ae6eb0bf13d564e8198b0ae6476 Mon Sep 17 00:00:00 2001 From: Andrew Boie Date: Fri, 17 Jul 2015 12:03:52 -0700 Subject: [PATCH] initial commit for next-gen sanity checks The online help ./scripts/sanitycheck --help describes usage. Most users will simply want to run with no arguments. Change-Id: Icedbbfc22599a64a6e3dbbb808ff3276db06f2e0 Signed-off-by: Andrew Boie --- .gitignore | 1 + samples/bluetooth/beacon/testcase.ini | 12 + samples/bluetooth/central/testcase.ini | 7 + samples/bluetooth/init/testcase.ini | 7 + samples/bluetooth/peripheral/testcase.ini | 12 + samples/bluetooth/shell/testcase.ini | 12 + samples/bluetooth/tester/testcase.ini | 4 + .../microkernel/apps/hello_world/testcase.ini | 4 + .../microkernel/apps/nfc_hello/testcase.ini | 7 + .../apps/philosophers/testcase.ini | 4 + .../benchmark/app_kernel/src/Makefile | 2 +- .../benchmark/app_kernel/testcase.ini | 6 + .../benchmark/boot_time/testcase.ini | 4 + .../benchmark/footprint/testcase.ini | 17 + .../benchmark/latency_measure/testcase.ini | 4 + .../benchmark/sys_kernel/testcase.ini | 4 + .../test/test_bluetooth/testcase.ini | 3 + .../test/test_critical/testcase.ini | 3 + .../microkernel/test/test_events/testcase.ini | 3 + .../microkernel/test/test_fifo/testcase.ini | 3 + .../microkernel/test/test_libs/testcase.ini | 3 + .../microkernel/test/test_mail/testcase.ini | 3 + .../microkernel/test/test_map/testcase.ini | 3 + .../microkernel/test/test_mutex/testcase.ini | 3 + .../microkernel/test/test_pipe/testcase.ini | 3 + .../microkernel/test/test_pool/testcase.ini | 3 + .../microkernel/test/test_rand32/testcase.ini | 3 + .../microkernel/test/test_sema/testcase.ini | 3 + .../test/test_sprintf/testcase.ini | 3 + .../test/test_stackprot/testcase.ini | 3 + .../test/test_static_idt/testcase.ini | 4 + .../microkernel/test/test_task/testcase.ini | 3 + .../test/test_task_irq/testcase.ini | 3 + .../test/test_tickless/testcase.ini | 4 + .../microkernel/test/test_timer/testcase.ini | 3 + .../microkernel/test/test_xip/testcase.ini | 3 + .../apps/bluetooth/init/tescase.ini | 4 + .../apps/bluetooth/shell/tescase.ini | 4 + .../nanokernel/apps/hello_world/testcase.ini | 4 + .../nanokernel/apps/philosophers/testcase.ini | 4 + .../benchmark/boot_time/testcase.ini | 4 + .../benchmark/footprint/testcase.ini | 17 + .../benchmark/latency_measure/testcase.ini | 4 + .../benchmark/sys_kernel/testcase.ini | 4 + .../test_arm_m3_irq_vector_table/testcase.ini | 4 + .../test/test_bluetooth/testcase.ini | 3 + .../nanokernel/test/test_context/testcase.ini | 3 + .../nanokernel/test/test_fifo/testcase.ini | 3 + .../nanokernel/test/test_lifo/testcase.ini | 3 + .../nanokernel/test/test_sema/testcase.ini | 3 + .../nanokernel/test/test_stack/testcase.ini | 3 + .../test/test_stackprot/testcase.ini | 3 + .../test/test_static_idt/testcase.ini | 4 + .../nanokernel/test/test_timer/testcase.ini | 3 + samples/nanokernel/test/test_xip/testcase.ini | 3 + scripts/sanity_chk/.gitignore | 2 + scripts/sanity_chk/arches/arm.ini | 9 + scripts/sanity_chk/arches/x86.ini | 12 + scripts/sanity_chk/sanity_last_release.csv | 229 +++ scripts/sanitycheck | 1509 +++++++++++++++++ 60 files changed, 2009 insertions(+), 1 deletion(-) create mode 100644 samples/bluetooth/beacon/testcase.ini create mode 100644 samples/bluetooth/central/testcase.ini create mode 100644 samples/bluetooth/init/testcase.ini create mode 100644 samples/bluetooth/peripheral/testcase.ini create mode 100644 samples/bluetooth/shell/testcase.ini create mode 100644 samples/bluetooth/tester/testcase.ini create mode 100644 samples/microkernel/apps/hello_world/testcase.ini create mode 100644 samples/microkernel/apps/nfc_hello/testcase.ini create mode 100644 samples/microkernel/apps/philosophers/testcase.ini create mode 100644 samples/microkernel/benchmark/app_kernel/testcase.ini create mode 100644 samples/microkernel/benchmark/boot_time/testcase.ini create mode 100644 samples/microkernel/benchmark/footprint/testcase.ini create mode 100644 samples/microkernel/benchmark/latency_measure/testcase.ini create mode 100644 samples/microkernel/benchmark/sys_kernel/testcase.ini create mode 100644 samples/microkernel/test/test_bluetooth/testcase.ini create mode 100644 samples/microkernel/test/test_critical/testcase.ini create mode 100644 samples/microkernel/test/test_events/testcase.ini create mode 100644 samples/microkernel/test/test_fifo/testcase.ini create mode 100644 samples/microkernel/test/test_libs/testcase.ini create mode 100644 samples/microkernel/test/test_mail/testcase.ini create mode 100644 samples/microkernel/test/test_map/testcase.ini create mode 100644 samples/microkernel/test/test_mutex/testcase.ini create mode 100644 samples/microkernel/test/test_pipe/testcase.ini create mode 100644 samples/microkernel/test/test_pool/testcase.ini create mode 100644 samples/microkernel/test/test_rand32/testcase.ini create mode 100644 samples/microkernel/test/test_sema/testcase.ini create mode 100644 samples/microkernel/test/test_sprintf/testcase.ini create mode 100644 samples/microkernel/test/test_stackprot/testcase.ini create mode 100644 samples/microkernel/test/test_static_idt/testcase.ini create mode 100644 samples/microkernel/test/test_task/testcase.ini create mode 100644 samples/microkernel/test/test_task_irq/testcase.ini create mode 100644 samples/microkernel/test/test_tickless/testcase.ini create mode 100644 samples/microkernel/test/test_timer/testcase.ini create mode 100644 samples/microkernel/test/test_xip/testcase.ini create mode 100644 samples/nanokernel/apps/bluetooth/init/tescase.ini create mode 100644 samples/nanokernel/apps/bluetooth/shell/tescase.ini create mode 100644 samples/nanokernel/apps/hello_world/testcase.ini create mode 100644 samples/nanokernel/apps/philosophers/testcase.ini create mode 100644 samples/nanokernel/benchmark/boot_time/testcase.ini create mode 100644 samples/nanokernel/benchmark/footprint/testcase.ini create mode 100644 samples/nanokernel/benchmark/latency_measure/testcase.ini create mode 100644 samples/nanokernel/benchmark/sys_kernel/testcase.ini create mode 100644 samples/nanokernel/test/test_arm_m3_irq_vector_table/testcase.ini create mode 100644 samples/nanokernel/test/test_bluetooth/testcase.ini create mode 100644 samples/nanokernel/test/test_context/testcase.ini create mode 100644 samples/nanokernel/test/test_fifo/testcase.ini create mode 100644 samples/nanokernel/test/test_lifo/testcase.ini create mode 100644 samples/nanokernel/test/test_sema/testcase.ini create mode 100644 samples/nanokernel/test/test_stack/testcase.ini create mode 100644 samples/nanokernel/test/test_stackprot/testcase.ini create mode 100644 samples/nanokernel/test/test_static_idt/testcase.ini create mode 100644 samples/nanokernel/test/test_timer/testcase.ini create mode 100644 samples/nanokernel/test/test_xip/testcase.ini create mode 100644 scripts/sanity_chk/.gitignore create mode 100644 scripts/sanity_chk/arches/arm.ini create mode 100644 scripts/sanity_chk/arches/x86.ini create mode 100644 scripts/sanity_chk/sanity_last_release.csv create mode 100755 scripts/sanitycheck diff --git a/.gitignore b/.gitignore index bec6497e812..78585e0e266 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ doc/_build xml html/ doc/latex/ +sanity-out/ diff --git a/samples/bluetooth/beacon/testcase.ini b/samples/bluetooth/beacon/testcase.ini new file mode 100644 index 00000000000..7946e54c26a --- /dev/null +++ b/samples/bluetooth/beacon/testcase.ini @@ -0,0 +1,12 @@ +[test_x86] +tags = bluetooth +build_only = true +arch_whitelist = x86 +# FIXME Doesn't work for ia32_pci +config_whitelist = CONFIG_PLATFORM="ia32" + +[test_arm] +tags = bluetooth +build_only = true +arch_whitelist = arm + diff --git a/samples/bluetooth/central/testcase.ini b/samples/bluetooth/central/testcase.ini new file mode 100644 index 00000000000..84e8d8ee478 --- /dev/null +++ b/samples/bluetooth/central/testcase.ini @@ -0,0 +1,7 @@ +[test] +tags = bluetooth +build_only = true +arch_whitelist = x86 +# FIXME Doesn't work for ia32_pci +config_whitelist = CONFIG_PLATFORM="ia32" + diff --git a/samples/bluetooth/init/testcase.ini b/samples/bluetooth/init/testcase.ini new file mode 100644 index 00000000000..4ea5f6e01e3 --- /dev/null +++ b/samples/bluetooth/init/testcase.ini @@ -0,0 +1,7 @@ +[test] +tags = bluetooth +build_only = true +arch_whitelist = x86 +# Doesn't work for ia32_pci +config_whitelist = CONFIG_PLATFORM="ia32" + diff --git a/samples/bluetooth/peripheral/testcase.ini b/samples/bluetooth/peripheral/testcase.ini new file mode 100644 index 00000000000..7946e54c26a --- /dev/null +++ b/samples/bluetooth/peripheral/testcase.ini @@ -0,0 +1,12 @@ +[test_x86] +tags = bluetooth +build_only = true +arch_whitelist = x86 +# FIXME Doesn't work for ia32_pci +config_whitelist = CONFIG_PLATFORM="ia32" + +[test_arm] +tags = bluetooth +build_only = true +arch_whitelist = arm + diff --git a/samples/bluetooth/shell/testcase.ini b/samples/bluetooth/shell/testcase.ini new file mode 100644 index 00000000000..7946e54c26a --- /dev/null +++ b/samples/bluetooth/shell/testcase.ini @@ -0,0 +1,12 @@ +[test_x86] +tags = bluetooth +build_only = true +arch_whitelist = x86 +# FIXME Doesn't work for ia32_pci +config_whitelist = CONFIG_PLATFORM="ia32" + +[test_arm] +tags = bluetooth +build_only = true +arch_whitelist = arm + diff --git a/samples/bluetooth/tester/testcase.ini b/samples/bluetooth/tester/testcase.ini new file mode 100644 index 00000000000..9c84ca2ef6e --- /dev/null +++ b/samples/bluetooth/tester/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = bluetooth +build_only = true +platform_whitelist = ti_lm3s6965 diff --git a/samples/microkernel/apps/hello_world/testcase.ini b/samples/microkernel/apps/hello_world/testcase.ini new file mode 100644 index 00000000000..8ee37ed2f85 --- /dev/null +++ b/samples/microkernel/apps/hello_world/testcase.ini @@ -0,0 +1,4 @@ +[test] +build_only = true +tags = apps + diff --git a/samples/microkernel/apps/nfc_hello/testcase.ini b/samples/microkernel/apps/nfc_hello/testcase.ini new file mode 100644 index 00000000000..f839a3106bc --- /dev/null +++ b/samples/microkernel/apps/nfc_hello/testcase.ini @@ -0,0 +1,7 @@ +[test] +build_only = true +tags = apps +arch_whitelist = x86 +# Doesn't work for ia32_pci +config_whitelist = CONFIG_PLATFORM="ia32" + diff --git a/samples/microkernel/apps/philosophers/testcase.ini b/samples/microkernel/apps/philosophers/testcase.ini new file mode 100644 index 00000000000..8ee37ed2f85 --- /dev/null +++ b/samples/microkernel/apps/philosophers/testcase.ini @@ -0,0 +1,4 @@ +[test] +build_only = true +tags = apps + diff --git a/samples/microkernel/benchmark/app_kernel/src/Makefile b/samples/microkernel/benchmark/app_kernel/src/Makefile index ec93a2cdb16..609033ac1bc 100644 --- a/samples/microkernel/benchmark/app_kernel/src/Makefile +++ b/samples/microkernel/benchmark/app_kernel/src/Makefile @@ -1,5 +1,5 @@ ccflags-y += -I$(CURDIR)/misc/generated/sysgen -ccflags-y += -I$(CURDIR)/../../latency_measure/src +ccflags-y += -I$(srctree)/samples/microkernel/benchmark/latency_measure/src obj-y := fifo_b.o mailbox_b.o master.o mempool_b.o \ nop_b.o pipe_r.o sema_r.o event_b.o \ diff --git a/samples/microkernel/benchmark/app_kernel/testcase.ini b/samples/microkernel/benchmark/app_kernel/testcase.ini new file mode 100644 index 00000000000..4a7ab9ed0cb --- /dev/null +++ b/samples/microkernel/benchmark/app_kernel/testcase.ini @@ -0,0 +1,6 @@ +[test] +tags = benchmark +arch_whitelist = x86 +# On my machine, takes about 110 to run, 180 to be safe +timeout = 180 + diff --git a/samples/microkernel/benchmark/boot_time/testcase.ini b/samples/microkernel/benchmark/boot_time/testcase.ini new file mode 100644 index 00000000000..3609d30092d --- /dev/null +++ b/samples/microkernel/benchmark/boot_time/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = benchmark +arch_whitelist = x86 + diff --git a/samples/microkernel/benchmark/footprint/testcase.ini b/samples/microkernel/benchmark/footprint/testcase.ini new file mode 100644 index 00000000000..ed1318c1480 --- /dev/null +++ b/samples/microkernel/benchmark/footprint/testcase.ini @@ -0,0 +1,17 @@ +[footprint-min] +tags = footprint +extra_args = TEST=min +build_only = true + +[footprint-reg] +tags = footprint +extra_args = TEST=reg +build_only = true +arch_whitelist = x86 + +[footprint-max] +tags = footprint +extra_args = TEST=max +build_only = true +arch_whitelist = x86 + diff --git a/samples/microkernel/benchmark/latency_measure/testcase.ini b/samples/microkernel/benchmark/latency_measure/testcase.ini new file mode 100644 index 00000000000..3609d30092d --- /dev/null +++ b/samples/microkernel/benchmark/latency_measure/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = benchmark +arch_whitelist = x86 + diff --git a/samples/microkernel/benchmark/sys_kernel/testcase.ini b/samples/microkernel/benchmark/sys_kernel/testcase.ini new file mode 100644 index 00000000000..3609d30092d --- /dev/null +++ b/samples/microkernel/benchmark/sys_kernel/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = benchmark +arch_whitelist = x86 + diff --git a/samples/microkernel/test/test_bluetooth/testcase.ini b/samples/microkernel/test/test_bluetooth/testcase.ini new file mode 100644 index 00000000000..ee3f7b9a092 --- /dev/null +++ b/samples/microkernel/test/test_bluetooth/testcase.ini @@ -0,0 +1,3 @@ +[test-bluetooth] +tags = bluetooth + diff --git a/samples/microkernel/test/test_critical/testcase.ini b/samples/microkernel/test/test_critical/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_critical/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_events/testcase.ini b/samples/microkernel/test/test_events/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_events/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_fifo/testcase.ini b/samples/microkernel/test/test_fifo/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_fifo/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_libs/testcase.ini b/samples/microkernel/test/test_libs/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_libs/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_mail/testcase.ini b/samples/microkernel/test/test_mail/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_mail/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_map/testcase.ini b/samples/microkernel/test/test_map/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_map/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_mutex/testcase.ini b/samples/microkernel/test/test_mutex/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_mutex/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_pipe/testcase.ini b/samples/microkernel/test/test_pipe/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_pipe/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_pool/testcase.ini b/samples/microkernel/test/test_pool/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_pool/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_rand32/testcase.ini b/samples/microkernel/test/test_rand32/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_rand32/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_sema/testcase.ini b/samples/microkernel/test/test_sema/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_sema/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_sprintf/testcase.ini b/samples/microkernel/test/test_sprintf/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_sprintf/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_stackprot/testcase.ini b/samples/microkernel/test/test_stackprot/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_stackprot/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_static_idt/testcase.ini b/samples/microkernel/test/test_static_idt/testcase.ini new file mode 100644 index 00000000000..6356d9194b9 --- /dev/null +++ b/samples/microkernel/test/test_static_idt/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = core +config_whitelist = CONFIG_ISA_IA32 + diff --git a/samples/microkernel/test/test_task/testcase.ini b/samples/microkernel/test/test_task/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_task/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_task_irq/testcase.ini b/samples/microkernel/test/test_task_irq/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_task_irq/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_tickless/testcase.ini b/samples/microkernel/test/test_tickless/testcase.ini new file mode 100644 index 00000000000..3592865c517 --- /dev/null +++ b/samples/microkernel/test/test_tickless/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = core +config_whitelist = !CONFIG_PLATFORM_TI_LM3S6965_QEMU + diff --git a/samples/microkernel/test/test_timer/testcase.ini b/samples/microkernel/test/test_timer/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_timer/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/microkernel/test/test_xip/testcase.ini b/samples/microkernel/test/test_xip/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/microkernel/test/test_xip/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/apps/bluetooth/init/tescase.ini b/samples/nanokernel/apps/bluetooth/init/tescase.ini new file mode 100644 index 00000000000..7a400359cb8 --- /dev/null +++ b/samples/nanokernel/apps/bluetooth/init/tescase.ini @@ -0,0 +1,4 @@ +[test] +tags = bluetooth apps +build_only = true + diff --git a/samples/nanokernel/apps/bluetooth/shell/tescase.ini b/samples/nanokernel/apps/bluetooth/shell/tescase.ini new file mode 100644 index 00000000000..7a400359cb8 --- /dev/null +++ b/samples/nanokernel/apps/bluetooth/shell/tescase.ini @@ -0,0 +1,4 @@ +[test] +tags = bluetooth apps +build_only = true + diff --git a/samples/nanokernel/apps/hello_world/testcase.ini b/samples/nanokernel/apps/hello_world/testcase.ini new file mode 100644 index 00000000000..8ee37ed2f85 --- /dev/null +++ b/samples/nanokernel/apps/hello_world/testcase.ini @@ -0,0 +1,4 @@ +[test] +build_only = true +tags = apps + diff --git a/samples/nanokernel/apps/philosophers/testcase.ini b/samples/nanokernel/apps/philosophers/testcase.ini new file mode 100644 index 00000000000..8ee37ed2f85 --- /dev/null +++ b/samples/nanokernel/apps/philosophers/testcase.ini @@ -0,0 +1,4 @@ +[test] +build_only = true +tags = apps + diff --git a/samples/nanokernel/benchmark/boot_time/testcase.ini b/samples/nanokernel/benchmark/boot_time/testcase.ini new file mode 100644 index 00000000000..3609d30092d --- /dev/null +++ b/samples/nanokernel/benchmark/boot_time/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = benchmark +arch_whitelist = x86 + diff --git a/samples/nanokernel/benchmark/footprint/testcase.ini b/samples/nanokernel/benchmark/footprint/testcase.ini new file mode 100644 index 00000000000..ed1318c1480 --- /dev/null +++ b/samples/nanokernel/benchmark/footprint/testcase.ini @@ -0,0 +1,17 @@ +[footprint-min] +tags = footprint +extra_args = TEST=min +build_only = true + +[footprint-reg] +tags = footprint +extra_args = TEST=reg +build_only = true +arch_whitelist = x86 + +[footprint-max] +tags = footprint +extra_args = TEST=max +build_only = true +arch_whitelist = x86 + diff --git a/samples/nanokernel/benchmark/latency_measure/testcase.ini b/samples/nanokernel/benchmark/latency_measure/testcase.ini new file mode 100644 index 00000000000..3609d30092d --- /dev/null +++ b/samples/nanokernel/benchmark/latency_measure/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = benchmark +arch_whitelist = x86 + diff --git a/samples/nanokernel/benchmark/sys_kernel/testcase.ini b/samples/nanokernel/benchmark/sys_kernel/testcase.ini new file mode 100644 index 00000000000..3609d30092d --- /dev/null +++ b/samples/nanokernel/benchmark/sys_kernel/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = benchmark +arch_whitelist = x86 + diff --git a/samples/nanokernel/test/test_arm_m3_irq_vector_table/testcase.ini b/samples/nanokernel/test/test_arm_m3_irq_vector_table/testcase.ini new file mode 100644 index 00000000000..d7b14e17287 --- /dev/null +++ b/samples/nanokernel/test/test_arm_m3_irq_vector_table/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = core +config_whitelist = CONFIG_CPU_CORTEX_M3_M4 + diff --git a/samples/nanokernel/test/test_bluetooth/testcase.ini b/samples/nanokernel/test/test_bluetooth/testcase.ini new file mode 100644 index 00000000000..ee3f7b9a092 --- /dev/null +++ b/samples/nanokernel/test/test_bluetooth/testcase.ini @@ -0,0 +1,3 @@ +[test-bluetooth] +tags = bluetooth + diff --git a/samples/nanokernel/test/test_context/testcase.ini b/samples/nanokernel/test/test_context/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_context/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/test/test_fifo/testcase.ini b/samples/nanokernel/test/test_fifo/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_fifo/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/test/test_lifo/testcase.ini b/samples/nanokernel/test/test_lifo/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_lifo/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/test/test_sema/testcase.ini b/samples/nanokernel/test/test_sema/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_sema/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/test/test_stack/testcase.ini b/samples/nanokernel/test/test_stack/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_stack/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/test/test_stackprot/testcase.ini b/samples/nanokernel/test/test_stackprot/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_stackprot/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/test/test_static_idt/testcase.ini b/samples/nanokernel/test/test_static_idt/testcase.ini new file mode 100644 index 00000000000..6356d9194b9 --- /dev/null +++ b/samples/nanokernel/test/test_static_idt/testcase.ini @@ -0,0 +1,4 @@ +[test] +tags = core +config_whitelist = CONFIG_ISA_IA32 + diff --git a/samples/nanokernel/test/test_timer/testcase.ini b/samples/nanokernel/test/test_timer/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_timer/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/samples/nanokernel/test/test_xip/testcase.ini b/samples/nanokernel/test/test_xip/testcase.ini new file mode 100644 index 00000000000..2e4e8851815 --- /dev/null +++ b/samples/nanokernel/test/test_xip/testcase.ini @@ -0,0 +1,3 @@ +[test] +tags = core + diff --git a/scripts/sanity_chk/.gitignore b/scripts/sanity_chk/.gitignore new file mode 100644 index 00000000000..ff5caacdbe5 --- /dev/null +++ b/scripts/sanity_chk/.gitignore @@ -0,0 +1,2 @@ +last_sanity.csv + diff --git a/scripts/sanity_chk/arches/arm.ini b/scripts/sanity_chk/arches/arm.ini new file mode 100644 index 00000000000..ed67930fa61 --- /dev/null +++ b/scripts/sanity_chk/arches/arm.ini @@ -0,0 +1,9 @@ +[arch] +name = arm +platforms = basic_cortex_m3 fsl_frdm_k64f + +[basic_cortex_m3] +qemu_support = true + +[fsl_frdm_k64f] + diff --git a/scripts/sanity_chk/arches/x86.ini b/scripts/sanity_chk/arches/x86.ini new file mode 100644 index 00000000000..392bd4160b6 --- /dev/null +++ b/scripts/sanity_chk/arches/x86.ini @@ -0,0 +1,12 @@ +[arch] +name = x86 +platforms = basic_minuteia basic_atom galileo + +[basic_atom] +qemu_support = true + +[basic_minuteia] +qemu_support = true + +[galileo] + diff --git a/scripts/sanity_chk/sanity_last_release.csv b/scripts/sanity_chk/sanity_last_release.csv new file mode 100644 index 00000000000..1697b6a0c05 --- /dev/null +++ b/scripts/sanity_chk/sanity_last_release.csv @@ -0,0 +1,229 @@ +test,arch,platform,passed,status,extra_args,qemu,qemu_time,ram_size,rom_size +nanokernel/test/test_stackprot/test,x86,basic_atom,True,,,True,0.3900439739227295,11707,7491 +nanokernel/test/test_fifo/test,x86,basic_atom,True,,,True,1.578080177307129,45810,17638 +microkernel/test/test_events/test,x86,basic_minuteia,True,,,True,0.6130318641662598,25960,16376 +nanokernel/test/test_fifo/test,arm,fsl_frdm_k64f,True,,,False,,28688,16181 +bluetooth/beacon/test_arm,arm,basic_cortex_m3,True,,,True,,15160,33191 +nanokernel/test/test_sema/test,x86,galileo,True,,,False,,47044,17476 +microkernel/test/test_task_irq/test,arm,fsl_frdm_k64f,True,,,False,,11020,14872 +microkernel/benchmark/footprint/footprint-min,arm,basic_cortex_m3,True,,TEST=min,True,,1796,6316 +bluetooth/init/test,x86,basic_minuteia,True,,,True,,51880,38948 +nanokernel/test/test_lifo/test,x86,galileo,True,,,False,,48012,18260 +microkernel/test/test_mutex/test,x86,basic_atom,True,,,True,1.4354000091552734,33175,14355 +nanokernel/benchmark/latency_measure/test,x86,basic_minuteia,True,,,True,0.4863758087158203,32955,17631 +nanokernel/test/test_static_idt/test,x86,basic_atom,True,,,True,0.43580198287963867,10856,7156 +nanokernel/benchmark/sys_kernel/test,x86,basic_minuteia,True,,,True,0.5170221328735352,24626,16982 +nanokernel/benchmark/boot_time/test,x86,basic_minuteia,True,,,True,0.39171504974365234,9717,6041 +nanokernel/test/test_bluetooth/test-bluetooth,x86,galileo,True,,,False,,44816,34924 +microkernel/test/test_pool/test,arm,basic_cortex_m3,True,,,True,3.669074058532715,27020,15462 +nanokernel/test/test_sema/test,x86,basic_atom,True,,,True,1.8950130939483643,44676,15372 +microkernel/test/test_timer/test,x86,basic_atom,True,,,True,2.8563740253448486,53070,25238 +microkernel/test/test_tickless/test,x86,galileo,True,,,False,,23518,14650 +bluetooth/shell/test_arm,arm,basic_cortex_m3,True,,,True,,19068,40203 +microkernel/benchmark/boot_time/test,x86,basic_minuteia,True,,,True,0.38483095169067383,17389,11853 +nanokernel/test/test_lifo/test,x86,basic_minuteia,True,,,True,7.8495118618011475,45016,15572 +microkernel/test/test_map/test,x86,galileo,True,,,False,,28939,18023 +microkernel/test/test_sema/test,arm,basic_cortex_m3,True,,,True,12.122911930084229,19228,16717 +microkernel/test/test_static_idt/test,x86,basic_atom,True,,,True,0.38988304138183594,21112,12528 +nanokernel/test/test_stackprot/test,x86,galileo,True,,,False,,14339,9859 +nanokernel/test/test_fifo/test,arm,basic_cortex_m3,True,,,True,7.265285968780518,28672,14811 +microkernel/test/test_static_idt/test,x86,galileo,True,,,False,,23348,14500 +microkernel/test/test_libs/test,arm,basic_cortex_m3,True,,,True,0.00885319709777832,10762,13360 +microkernel/apps/hello_world/test,x86,galileo,True,,,False,,27317,18453 +nanokernel/apps/hello_world/test,x86,galileo,True,,,False,,13899,8431 +microkernel/apps/philosophers/test,arm,basic_cortex_m3,True,,,True,,14180,11295 +nanokernel/test/test_timer/test,arm,basic_cortex_m3,True,,,True,29.05277395248413,7580,8289 +microkernel/test/test_task/test,arm,fsl_frdm_k64f,True,,,False,,10880,15069 +nanokernel/benchmark/boot_time/test,x86,basic_atom,True,,,True,0.40399789810180664,10121,6405 +microkernel/test/test_libs/test,x86,basic_atom,True,,,True,0.4623570442199707,23773,15157 +microkernel/test/test_bluetooth/test-bluetooth,x86,basic_minuteia,True,,,True,0.45185089111328125,50388,37464 +microkernel/test/test_xip/test,x86,basic_minuteia,True,,,True,0.9812290668487549,9356,10556 +microkernel/test/test_pipe/test,x86,basic_atom,True,,,True,1.9525740146636963,39160,27600 +microkernel/benchmark/footprint/footprint-reg,x86,galileo,True,,TEST=reg,False,,18707,15711 +nanokernel/test/test_timer/test,x86,basic_minuteia,True,,,True,30.641869068145752,16675,9399 +microkernel/test/test_rand32/test,x86,galileo,True,,,False,,20064,13272 +nanokernel/test/test_timer/test,x86,basic_atom,True,,,True,5.522861957550049,17075,9755 +microkernel/test/test_sprintf/test,x86,basic_minuteia,True,,,True,0.4806699752807617,27705,21229 +microkernel/test/test_map/test,arm,basic_cortex_m3,True,,,True,0.1430048942565918,12804,14181 +microkernel/test/test_stackprot/test,x86,galileo,True,,,False,,25270,16418 +nanokernel/test/test_stack/test,x86,galileo,True,,,False,,22460,12740 +microkernel/test/test_mutex/test,arm,basic_cortex_m3,True,,,True,6.846920967102051,21656,12791 +microkernel/test/test_libs/test,x86,basic_minuteia,True,,,True,0.46005892753601074,23081,14517 +microkernel/apps/philosophers/test,arm,fsl_frdm_k64f,True,,,False,,14196,12517 +microkernel/test/test_events/test,arm,fsl_frdm_k64f,True,,,False,,11800,15588 +microkernel/test/test_fifo/test,x86,basic_atom,True,,,True,0.4695100784301758,26785,18153 +nanokernel/benchmark/latency_measure/test,x86,basic_atom,True,,,True,0.4914400577545166,33375,17975 +microkernel/test/test_tickless/test,x86,basic_atom,True,,,True,1.0034639835357666,21326,12734 +nanokernel/test/test_bluetooth/test-bluetooth,x86,basic_minuteia,True,,,True,0.4623138904571533,42048,32464 +nanokernel/test/test_context/test,x86,galileo,True,,,False,,36164,14604 +microkernel/test/test_stackprot/test,arm,fsl_frdm_k64f,True,,,False,,10728,14365 +nanokernel/test/test_stackprot/test,arm,fsl_frdm_k64f,True,,,False,,4492,7952 +microkernel/apps/philosophers/test,x86,galileo,True,,,False,,26573,14657 +nanokernel/benchmark/footprint/footprint-min,x86,basic_atom,True,,TEST=min,True,,2208,1852 +nanokernel/apps/philosophers/test,x86,basic_atom,True,,,True,,15953,6537 +microkernel/test/test_sema/test,x86,galileo,True,,,False,,37651,21471 +microkernel/test/test_libs/test,x86,galileo,True,,,False,,25865,16985 +microkernel/test/test_task_irq/test,x86,basic_minuteia,True,,,True,0.4734950065612793,27018,18458 +microkernel/test/test_tickless/test,x86,basic_minuteia,True,,,True,4.3573009967803955,20738,12194 +microkernel/test/test_map/test,x86,basic_atom,True,,,True,0.4864978790283203,26739,16087 +microkernel/test/test_fifo/test,arm,basic_cortex_m3,True,,,True,0.15673398971557617,10788,15897 +nanokernel/test/test_bluetooth/test-bluetooth,arm,basic_cortex_m3,True,,,True,0.00786900520324707,9992,26561 +nanokernel/test/test_xip/test,arm,fsl_frdm_k64f,True,,,False,,3472,6061 +nanokernel/test/test_stack/test,arm,basic_cortex_m3,True,,,True,1.1097431182861328,9724,8524 +bluetooth/central/test,x86,basic_minuteia,True,,,True,,52204,39272 +nanokernel/test/test_lifo/test,arm,basic_cortex_m3,True,,,True,7.264161109924316,30092,13720 +nanokernel/benchmark/footprint/footprint-reg,x86,basic_atom,True,,TEST=reg,True,,6514,4866 +microkernel/benchmark/boot_time/test,x86,galileo,True,,,False,,19893,14053 +microkernel/benchmark/app_kernel/test,x86,basic_atom,True,,,True,18.793134927749634,76772,44772 +microkernel/apps/hello_world/test,arm,basic_cortex_m3,True,,,True,,10716,11116 +nanokernel/test/test_timer/test,x86,galileo,True,,,False,,19543,11959 +microkernel/test/test_pool/test,x86,basic_atom,True,,,True,1.0459659099578857,41669,17509 +bluetooth/peripheral/test_x86,x86,basic_minuteia,True,,,True,,56617,43625 +nanokernel/test/test_stackprot/test,x86,basic_minuteia,True,,,True,0.4640929698944092,11027,6855 +microkernel/test/test_static_idt/test,x86,basic_minuteia,True,,,True,0.509680986404419,20476,11944 +microkernel/test/test_stackprot/test,x86,basic_atom,True,,,True,0.49658799171447754,22902,14314 +nanokernel/test/test_lifo/test,arm,fsl_frdm_k64f,True,,,False,,30108,15090 +microkernel/test/test_mail/test,x86,basic_minuteia,True,,,True,0.6057891845703125,35433,24749 +nanokernel/benchmark/sys_kernel/test,x86,galileo,True,,,False,,27122,19170 +nanokernel/benchmark/footprint/footprint-max,x86,basic_atom,True,,TEST=max,True,,15893,11673 +microkernel/benchmark/footprint/footprint-min,x86,basic_atom,True,,TEST=min,True,,5800,5076 +bluetooth/beacon/test_x86,x86,basic_minuteia,True,,,True,,52576,39644 +nanokernel/benchmark/boot_time/test,x86,galileo,True,,,False,,12329,8349 +nanokernel/test/test_static_idt/test,x86,basic_minuteia,True,,,True,0.4573800563812256,10460,6804 +microkernel/test/test_map/test,x86,basic_minuteia,True,,,True,0.4831821918487549,26063,15463 +nanokernel/test/test_stackprot/test,arm,basic_cortex_m3,True,,,True,0.010055065155029297,4476,6692 +microkernel/test/test_libs/test,arm,fsl_frdm_k64f,True,,,False,,10778,14514 +nanokernel/test/test_xip/test,arm,basic_cortex_m3,True,,,True,0.006029844284057617,3456,4831 +microkernel/test/test_xip/test,x86,basic_atom,True,,,True,0.8974871635437012,9616,11116 +microkernel/test/test_critical/test,x86,basic_minuteia,True,,,True,15.503901958465576,19631,13123 +microkernel/test/test_mail/test,arm,basic_cortex_m3,True,,,True,0.27579283714294434,13112,22403 +microkernel/test/test_events/test,x86,galileo,True,,,False,,28780,18880 +microkernel/test/test_task/test,arm,basic_cortex_m3,True,,,True,3.028717041015625,10864,13843 +microkernel/test/test_bluetooth/test-bluetooth,x86,galileo,True,,,False,,53164,39924 +nanokernel/test/test_xip/test,x86,basic_minuteia,True,,,True,1.2589969635009766,5296,5349 +microkernel/test/test_sprintf/test,x86,basic_atom,True,,,True,0.4770331382751465,31429,24901 +nanokernel/apps/hello_world/test,x86,basic_minuteia,True,,,True,,11031,5871 +microkernel/apps/nfc_hello/test,x86,basic_minuteia,True,,,True,,14018,10626 +microkernel/test/test_timer/test,arm,fsl_frdm_k64f,True,,,False,,30100,24391 +microkernel/benchmark/footprint/footprint-reg,x86,basic_atom,True,,TEST=reg,True,,16391,13659 +microkernel/test/test_pipe/test,x86,galileo,True,,,False,,40856,29072 +microkernel/benchmark/app_kernel/test,x86,galileo,True,,,False,,77672,45756 +nanokernel/apps/philosophers/test,x86,basic_minuteia,True,,,True,,15541,6185 +nanokernel/test/test_sema/test,arm,basic_cortex_m3,True,,,True,9.185144901275635,29844,12642 +nanokernel/test/test_xip/test,x86,basic_atom,True,,,True,1.2263851165771484,5340,5705 +microkernel/test/test_mutex/test,x86,galileo,True,,,False,,35427,16343 +nanokernel/benchmark/footprint/footprint-min,x86,basic_minuteia,True,,TEST=min,True,,2024,1720 +nanokernel/test/test_bluetooth/test-bluetooth,x86,basic_atom,True,,,True,0.44973301887512207,42776,33004 +microkernel/test/test_timer/test,arm,basic_cortex_m3,True,,,True,15.362816095352173,30084,23013 +nanokernel/test/test_context/test,x86,basic_minuteia,True,,,True,6.956724166870117,33292,12040 +microkernel/benchmark/sys_kernel/test,x86,galileo,True,,,False,,52082,26450 +nanokernel/test/test_arm_m3_irq_vector_table/test,arm,fsl_frdm_k64f,True,,,False,,3492,6421 +microkernel/test/test_mail/test,x86,basic_atom,True,,,True,0.5486729145050049,36045,25309 +bluetooth/shell/test_arm,arm,fsl_frdm_k64f,True,,,False,,19072,40973 +bluetooth/init/test,x86,basic_atom,True,,,True,,52840,39740 +microkernel/test/test_mutex/test,x86,basic_minuteia,True,,,True,7.501186847686768,32543,13775 +microkernel/apps/nfc_hello/test,x86,basic_atom,True,,,True,,14474,11010 +microkernel/test/test_task_irq/test,x86,galileo,True,,,False,,29906,21030 +microkernel/test/test_mutex/test,arm,fsl_frdm_k64f,True,,,False,,21672,14013 +microkernel/test/test_pool/test,x86,galileo,True,,,False,,43545,19269 +bluetooth/shell/test_x86,x86,basic_minuteia,True,,,True,,57193,42473 +microkernel/test/test_pipe/test,x86,basic_minuteia,True,,,True,10.45518708229065,38072,26604 +nanokernel/benchmark/footprint/footprint-reg,x86,basic_minuteia,True,,TEST=reg,True,,6122,4518 +nanokernel/test/test_arm_m3_irq_vector_table/test,arm,basic_cortex_m3,True,,,True,0.010872840881347656,3476,5191 +microkernel/test/test_sema/test,x86,basic_atom,True,,,True,2.331605911254883,35511,19587 +nanokernel/test/test_stack/test,arm,fsl_frdm_k64f,True,,,False,,9740,9746 +microkernel/test/test_stackprot/test,x86,basic_minuteia,True,,,True,0.42488694190979004,21942,13406 +microkernel/test/test_tickless/test,arm,fsl_frdm_k64f,True,,,False,,10720,12548 +microkernel/test/test_pool/test,arm,fsl_frdm_k64f,True,,,False,,27036,16688 +bluetooth/beacon/test_x86,x86,basic_atom,True,,,True,,53544,40444 +nanokernel/test/test_timer/test,arm,fsl_frdm_k64f,True,,,False,,7596,9511 +nanokernel/apps/philosophers/test,arm,basic_cortex_m3,True,,,True,,9660,5619 +microkernel/benchmark/footprint/footprint-max,x86,basic_atom,True,,TEST=max,True,,46366,37714 +microkernel/test/test_xip/test,arm,fsl_frdm_k64f,True,,,False,,7608,11056 +microkernel/test/test_stackprot/test,arm,basic_cortex_m3,True,,,True,0.011138916015625,10712,13109 +bluetooth/central/test,x86,basic_atom,True,,,True,,53168,40068 +nanokernel/test/test_fifo/test,x86,galileo,True,,,False,,48134,19746 +microkernel/test/test_critical/test,arm,fsl_frdm_k64f,True,,,False,,8968,13386 +microkernel/test/test_bluetooth/test-bluetooth,arm,fsl_frdm_k64f,True,,,False,,15168,32535 +microkernel/apps/hello_world/test,x86,basic_minuteia,True,,,True,,24441,15893 +microkernel/benchmark/app_kernel/test,x86,basic_minuteia,True,,,True,1.7877001762390137,75300,43692 +microkernel/apps/philosophers/test,x86,basic_minuteia,True,,,True,,23701,12101 +microkernel/benchmark/sys_kernel/test,x86,basic_minuteia,True,,,True,0.6504800319671631,49586,24262 +nanokernel/apps/hello_world/test,arm,fsl_frdm_k64f,True,,,False,,5480,6498 +microkernel/test/test_pool/test,x86,basic_minuteia,True,,,True,4.2199461460113525,40669,16709 +microkernel/apps/hello_world/test,x86,basic_atom,True,,,True,,25137,16537 +microkernel/benchmark/footprint/footprint-min,arm,fsl_frdm_k64f,True,,TEST=min,False,,1796,7408 +microkernel/test/test_pipe/test,arm,basic_cortex_m3,True,,,True,9.724919080734253,14848,24094 +nanokernel/test/test_context/test,arm,fsl_frdm_k64f,True,,,False,,21660,10989 +bluetooth/peripheral/test_x86,x86,basic_atom,True,,,True,,57685,44521 +microkernel/test/test_mail/test,x86,galileo,True,,,False,,38209,27209 +microkernel/test/test_task_irq/test,x86,basic_atom,True,,,True,0.4650437831878662,27790,19178 +nanokernel/test/test_sema/test,x86,basic_minuteia,True,,,True,9.942577123641968,44232,14972 +bluetooth/beacon/test_arm,arm,fsl_frdm_k64f,True,,,False,,15164,33805 +microkernel/benchmark/boot_time/test,x86,basic_atom,True,,,True,0.45662379264831543,18017,12329 +microkernel/test/test_events/test,x86,basic_atom,True,,,True,0.5145089626312256,26568,16932 +microkernel/test/test_sprintf/test,arm,basic_cortex_m3,True,,,True,0.012573957443237305,8604,17885 +microkernel/apps/hello_world/test,arm,fsl_frdm_k64f,True,,,False,,10732,12338 +microkernel/test/test_xip/test,x86,galileo,True,,,False,,9656,13112 +nanokernel/benchmark/latency_measure/test,x86,galileo,True,,,False,,35395,19763 +microkernel/test/test_xip/test,arm,basic_cortex_m3,True,,,True,0.0038878917694091797,7592,9834 +microkernel/benchmark/footprint/footprint-max,x86,galileo,True,,TEST=max,False,,48142,39226 +microkernel/benchmark/latency_measure/test,x86,basic_minuteia,True,,,True,0.57499098777771,48543,28171 +microkernel/benchmark/footprint/footprint-min,x86,basic_minuteia,True,,TEST=min,True,,5588,4956 +nanokernel/test/test_lifo/test,x86,basic_atom,True,,,True,1.595146894454956,45652,16156 +nanokernel/benchmark/sys_kernel/test,x86,basic_atom,True,,,True,0.5303559303283691,25098,17310 +nanokernel/benchmark/footprint/footprint-min,arm,basic_cortex_m3,True,,TEST=min,True,,600,2054 +nanokernel/test/test_bluetooth/test-bluetooth,arm,fsl_frdm_k64f,True,,,False,,10008,27715 +microkernel/test/test_task_irq/test,arm,basic_cortex_m3,True,,,True,0.020154953002929688,11004,13662 +nanokernel/test/test_stack/test,x86,basic_minuteia,True,,,True,1.5868020057678223,19648,10236 +microkernel/test/test_sprintf/test,x86,galileo,True,,,False,,30585,23793 +microkernel/test/test_rand32/test,arm,basic_cortex_m3,True,,,True,0.01945209503173828,8608,10078 +microkernel/test/test_sema/test,x86,basic_minuteia,True,,,True,13.0003981590271,34827,18963 +microkernel/test/test_timer/test,x86,basic_minuteia,True,,,True,16.345667839050293,52238,24482 +nanokernel/test/test_static_idt/test,x86,galileo,True,,,False,,13340,9376 +microkernel/test/test_task/test,x86,basic_minuteia,True,,,True,3.4601569175720215,24416,15848 +nanokernel/test/test_context/test,x86,basic_atom,True,,,True,1.449923038482666,33780,12484 +microkernel/benchmark/footprint/footprint-max,x86,basic_minuteia,True,,TEST=max,True,,45162,36570 +microkernel/benchmark/footprint/footprint-reg,x86,basic_minuteia,True,,TEST=reg,True,,15831,13151 +nanokernel/apps/hello_world/test,x86,basic_atom,True,,,True,,11427,6223 +bluetooth/peripheral/test_arm,arm,fsl_frdm_k64f,True,,,False,,16256,37390 +nanokernel/apps/philosophers/test,arm,fsl_frdm_k64f,True,,,False,,9676,6841 +nanokernel/test/test_sema/test,arm,fsl_frdm_k64f,True,,,False,,29860,13864 +microkernel/test/test_bluetooth/test-bluetooth,arm,basic_cortex_m3,True,,,True,0.009977102279663086,15152,31381 +microkernel/test/test_rand32/test,x86,basic_minuteia,True,,,True,0.5146400928497314,17192,10716 +microkernel/benchmark/sys_kernel/test,x86,basic_atom,True,,,True,2.2390949726104736,50254,24802 +nanokernel/benchmark/footprint/footprint-min,arm,fsl_frdm_k64f,True,,TEST=min,False,,600,3154 +nanokernel/test/test_context/test,arm,basic_cortex_m3,True,,,True,6.3456408977508545,21644,9763 +microkernel/test/test_fifo/test,x86,basic_minuteia,True,,,True,0.4909679889678955,26113,17533 +nanokernel/apps/hello_world/test,arm,basic_cortex_m3,True,,,True,,5464,5276 +nanokernel/benchmark/footprint/footprint-min,x86,galileo,True,,TEST=min,False,,2144,1856 +nanokernel/apps/philosophers/test,x86,galileo,True,,,False,,18413,8749 +microkernel/test/test_fifo/test,arm,fsl_frdm_k64f,True,,,False,,10804,17127 +microkernel/benchmark/latency_measure/test,x86,galileo,True,,,False,,50975,30295 +nanokernel/test/test_fifo/test,x86,basic_minuteia,True,,,True,7.889343023300171,45138,17058 +microkernel/test/test_fifo/test,x86,galileo,True,,,False,,28997,20101 +microkernel/test/test_map/test,arm,fsl_frdm_k64f,True,,,False,,12820,15403 +microkernel/test/test_task/test,x86,galileo,True,,,False,,27232,18348 +microkernel/test/test_rand32/test,arm,fsl_frdm_k64f,True,,,False,,8624,11300 +nanokernel/benchmark/footprint/footprint-max,x86,basic_minuteia,True,,TEST=max,True,,15045,10881 +microkernel/test/test_sema/test,arm,fsl_frdm_k64f,True,,,False,,19244,17939 +microkernel/test/test_sprintf/test,arm,fsl_frdm_k64f,True,,,False,,8620,19101 +bluetooth/shell/test_x86,x86,basic_atom,True,,,True,,58301,43281 +microkernel/test/test_task/test,x86,basic_atom,True,,,True,1.00689697265625,25056,16436 +microkernel/benchmark/latency_measure/test,x86,basic_atom,True,,,True,2.2400739192962646,49127,28671 +nanokernel/benchmark/footprint/footprint-max,x86,galileo,True,,TEST=max,False,,17857,13385 +microkernel/test/test_critical/test,x86,basic_atom,True,,,True,3.27829909324646,20247,13687 +microkernel/test/test_pipe/test,arm,fsl_frdm_k64f,True,,,False,,14864,25262 +nanokernel/benchmark/footprint/footprint-reg,x86,galileo,True,,TEST=reg,False,,8990,7078 +nanokernel/test/test_xip/test,x86,galileo,True,,,False,,5588,7913 +microkernel/test/test_critical/test,x86,galileo,True,,,False,,22507,15683 +microkernel/test/test_timer/test,x86,galileo,True,,,False,,55238,27174 +microkernel/test/test_rand32/test,x86,basic_atom,True,,,True,0.49033498764038086,17804,11276 +microkernel/benchmark/footprint/footprint-min,x86,galileo,True,,TEST=min,False,,5696,5080 +microkernel/test/test_mail/test,arm,fsl_frdm_k64f,True,,,False,,13128,23559 +nanokernel/test/test_stack/test,x86,basic_atom,True,,,True,0.6568129062652588,20188,10576 +microkernel/test/test_events/test,arm,basic_cortex_m3,True,,,True,0.3083319664001465,11784,14366 +microkernel/test/test_bluetooth/test-bluetooth,x86,basic_atom,True,,,True,0.36398792266845703,51348,38248 +microkernel/test/test_critical/test,arm,basic_cortex_m3,True,,,True,14.526043891906738,8952,12164 +bluetooth/peripheral/test_arm,arm,basic_cortex_m3,True,,,True,,16252,36774 +microkernel/apps/philosophers/test,x86,basic_atom,True,,,True,,24329,12677 diff --git a/scripts/sanitycheck b/scripts/sanitycheck new file mode 100755 index 00000000000..1c415686be5 --- /dev/null +++ b/scripts/sanitycheck @@ -0,0 +1,1509 @@ +#!/usr/bin/env python +"""Zephyr Sanity Tests + +This script scans for the set of unit test applications in the git +repository and attempts to execute them. By default, it tries to +build each test case on one platform per architecture, using a precedence +list defined in an archtecture configuration file, and if possible +run the tests in the QEMU emulator. + +Test cases are detected by the presence of a 'testcase.ini' file in +the application's project directory. This file may contain one or +more blocks, each identifying a test scenario. The title of the block +is a name for the test case, which only needs to be unique for the +test cases specified in that testcase.ini file. The full canonical +name for each test case is /. + +Each testcase.ini block can define the following key/value pairs: + + tags = (required) + A set of string tags for the testcase. Usually pertains to + functional domains but can be anything. Command line invocations + of this script can filter the set of tests to run based on tag. + + extra_args = + Extra arguments to pass to Make when building or running the + test case. + + build_only = + If true, don't try to run the test under QEMU even if the + selected platform supports it. + + timeout = + Length of time to run test in QEMU before automatically killing it. + Default to 60 seconds. + + arch_whitelist = + Set of architectures that this test case should only be run for. + + platform_whitelist = + Set of platforms that this test case should only be run for for. + + config_whitelist = + Config options can either be config names like CONFIG_FOO which + match if the configuration is defined to any value, or key/value + pairs like CONFIG_FOO=bar which match if it is set to a specific + value. May prepend a '!' to invert the match. + +Architectures and platforms are defined in an archtecture configuration +file which are stored by default in scripts/sanity_chk/arches/. These +each define an [arch] block with the following key/value pairs: + + name = + The name of the arch. Example: x86 + + platforms = + List of supported platforms for this arch. The ordering here + is used to select a default platform to build for that arch. + +For every platform defined, there must be a corresponding block for it +in the arch configuration file. This block can be empty if there are +no special definitions for that arch. Options are: + + qemu_support = (default False) + Indicates whether binaries for this platform can run under QEMU + + microkernel_support = (default True) + Indicates whether this platform supports microkernel or just nanokernel + +The set of test cases that actually run depends on directives in the +testcase and archtecture .ini file and options passed in on the command +line. If there is every any confusion, running with -v or --discard-report +can help show why particular test cases were skipped. + +Metrics (such as pass/fail state and binary size) for the last code +release are stored in scripts/sanity_chk/sanity_last_release.csv. +To update this, pass the --all --release options. + +Most everyday users will run with no arguments. +""" + +import argparse +import os +import sys +import ConfigParser +import re +import tempfile +import subprocess +import multiprocessing +import select +import shutil +import signal +import threading +import time +import csv + +if "ZEPHYR_BASE" not in os.environ: + sys.stderr.write("$ZEPHYR_BASE environment variable undefined") + exit(1) +ZEPHYR_BASE = os.environ["ZEPHYR_BASE"] +VERBOSE = 0 +LAST_SANITY = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk", + "last_sanity.csv") +RELEASE_DATA = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk", + "sanity_last_release.csv") +PARALLEL = multiprocessing.cpu_count() * 2 + +if os.isatty(sys.stdout.fileno()): + TERMINAL = True + COLOR_NORMAL = '\033[0m' + COLOR_RED = '\033[91m' + COLOR_GREEN = '\033[92m' + COLOR_YELLOW = '\033[93m' +else: + TERMINAL = False + COLOR_NORMAL = "" + COLOR_RED = "" + COLOR_GREEN = "" + COLOR_YELLOW = "" + +class SanityCheckException(Exception): + pass + +class SanityRuntimeError(SanityCheckException): + pass + +class ConfigurationError(SanityCheckException): + def __init__(self, cfile, message): + self.cfile = cfile + self.message = message + + def __str__(self): + return repr(self.cfile + ": " + self.message) + +class MakeError(SanityCheckException): + pass + +class BuildError(MakeError): + pass + +class ExecutionError(MakeError): + pass + +# Debug Functions + +def debug(what): + if VERBOSE >= 1: + print what + +def error(what): + sys.stderr.write(COLOR_RED + what + COLOR_NORMAL + "\n") + +def verbose(what): + if VERBOSE >= 2: + print what + +def info(what): + sys.stdout.write(what + "\n") + +# Utility functions +class QEMUHandler: + """Spawns a thread to monitor QEMU output from pipes + + We pass QEMU_PIPE to 'make qemu' and monitor the pipes for output. + We need to do this as once qemu starts, it runs forever until killed. + Test cases emit special messages to the console as they run, we check + for these to collect whether the test passed or failed. + """ + RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL" + RUN_FAILED = "PROJECT EXECUTION FAILED" + + @staticmethod + def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results): + fifo_in = fifo_fn + ".in" + fifo_out = fifo_fn + ".out" + + # These in/out nodes are named from QEMU's perspective, not ours + if os.path.exists(fifo_in): + os.unlink(fifo_in) + os.mkfifo(fifo_in) + if os.path.exists(fifo_out): + os.unlink(fifo_out) + os.mkfifo(fifo_out) + + # We don't do anything with out_fp but we need to open it for + # writing so that QEMU doesn't block, due to the way pipes work + out_fp = open(fifo_in, "wb") + # Disable internal buffering, we don't + # want read() or poll() to ever block if there is data in there + in_fp = open(fifo_out, "rb", buffering=0) + log_out_fp = open(logfile, "w") + + start_time = time.time() + timeout_time = start_time + timeout + p = select.poll() + p.register(in_fp, select.POLLIN) + + metrics = {} + line = "" + while True: + this_timeout = int((timeout_time - time.time()) * 1000) + if this_timeout < 0 or not p.poll(this_timeout): + out_state = "timeout" + break + + c = in_fp.read(1) + if c == "": + # EOF, this shouldn't happen unless QEMU crashes + out_state = "unexpected eof" + break + line = line + c + if c != "\n": + continue + + # If we get here, line contains a full line of data output from QEMU + log_out_fp.write(line) + log_out_fp.flush() + line = line.strip() + verbose("QEMU: %s" % line) + + if line == QEMUHandler.RUN_PASSED: + out_state = "passed" + break + + if line == QEMUHandler.RUN_FAILED: + out_state = "failed" + break + + # TODO: Add support for getting numerical performance data + # from test cases. Will involve extending test case reporting + # APIs. Add whatever gets reported to the metrics dictionary + line = "" + + metrics["qemu_time"] = time.time() - start_time + verbose("QEMU complete (%s) after %f seconds" % + (out_state, metrics["qemu_time"])) + handler.set_state(out_state, metrics) + + log_out_fp.close() + out_fp.close() + in_fp.close() + + pid = int(open(pid_fn).read()) + os.unlink(pid_fn) + os.kill(pid, signal.SIGTERM) + os.unlink(fifo_in) + os.unlink(fifo_out) + + + def __init__(self, name, outdir, log_fn, timeout): + """Constructor + + @param name Arbitrary name of the created thread + @param outdir Working directory, shoudl be where qemu.pid gets created + by kbuild + @param log_fn Absolute path to write out QEMU's log data + @param timeout Kill the QEMU process if it doesn't finish up within + the given number of seconds + """ + # Create pipe to get QEMU's serial output + self.results = {} + self.state = "waiting" + self.lock = threading.Lock() + + # We pass this to QEMU which looks for fifos with .in and .out + # suffixes. + self.fifo_fn = os.path.join(outdir, "qemu-fifo") + + self.pid_fn = os.path.join(outdir, "qemu.pid") + if os.path.exists(self.pid_fn): + os.unlink(self.pid_fn) + + self.log_fn = log_fn + self.thread = threading.Thread(name=name, target=QEMUHandler._thread, + args=(self, timeout, outdir, self.log_fn, + self.fifo_fn, self.pid_fn, + self.results)) + self.thread.daemon = True + verbose("Spawning QEMU process for %s" % name) + self.thread.start() + + def set_state(self, state, metrics): + self.lock.acquire() + self.state = state + self.metrics = metrics + self.lock.release() + + def get_state(self): + self.lock.acquire() + ret = (self.state, self.metrics) + self.lock.release() + return ret + + def get_fifo(self): + return self.fifo_fn + + +class SizeCalculator: + def __init__(self, filename_stem): + """Constructor + + @param filename_stem Path to the output binary, minus file extension. + The .elf is parsed by objdump. Either .elf or .bin + is sized for the ROM size depending on whether XIP is supported + """ + elf_filename = filename_stem + ".elf" + + # Make sure this is an ELF binary + with open(elf_filename, "rb") as f: + magic = f.read(4) + + if (magic != "\x7fELF"): + raise SanityRuntimeError("%s is not an ELF binary" % elf_filename) + + # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK. + # GREP can not be used as it returns an error if the symbol is not found. + is_xip_command = "nm " + elf_filename + " | awk '/CONFIG_XIP/ { print $3 }'" + is_xip_output = subprocess.check_output(is_xip_command, shell=True) + self.is_xip = (len(is_xip_output) != 0) + + self.elf_filename = elf_filename + self.sections = {} + self.xip_rom_size = 0 + self.xip_ram_size = 0 + self.ram_size = 0 + + self._calculate_sizes() + + def get_ram_size(self): + """Get the amount of RAM the application will use up on the device + + @return amount of RAM, in bytes + """ + if self.is_xip: + return self.xip_ram_size + else: + return self.ram_size + + def get_rom_size(self): + """Get the size of the data that this application uses on device's flash + + @return amount of ROM, in bytes + """ + return self.xip_rom_size + + def unrecognized_sections(self): + """Get a list of sections inside the binary that weren't recognized + + @return list of unrecogized section names + """ + slist = [] + for k, v in self.sections.iteritems(): + if not v["recognized"]: + slist.append(k) + return slist + + def _calculate_sizes(self): + """ Calculate RAM and ROM usage by section """ + objdump_command = "objdump -h " + self.elf_filename + objdump_output = subprocess.check_output(objdump_command, + shell=True).splitlines() + + for line in objdump_output: + words = line.split() + + if (len(words) == 0): # Skip lines that are too short + continue + + index = words[0] + if (not index[0].isdigit()): # Skip lines that do not start + continue # with a digit + + name = words[1] # Skip lines with section names + if (name[0] == '.'): # starting with '.' + continue + + size = int(words[2], 16) + phys_addr = int(words[4], 16) + + # Add section to memory use totals (for both non-XIP and XIP scenarios) + # + # In an XIP image, the following sections are placed into ROM: + # text, ctors, rodata and datas + # In an XIP image, the following sections are placed into RAM: + # datas, bss and noinit + # In a non-XIP image, the following sections are placed into RAM + # text, ctors, rodata, datas, bss and noinit + # Unrecognized section names are not included in the calculations. + + self.ram_size += size + recognized = True + + if ((name == "text") or (name == "ctors") or (name == "rodata")): + self.xip_rom_size += size + elif (name == "datas"): + self.xip_rom_size += size + self.xip_ram_size += size + elif ((name == "bss") or (name == "noinit")): + self.xip_ram_size += size + else: + recognized = False + self.ram_size -= size # Undo the calculation + + self.sections[name] = {"phys_addr" : phys_addr, "size" : size, + "recognized" : recognized} + + +class MakeGoal: + """Metadata class representing one of the sub-makes called by MakeGenerator + + MakeGenerator returns a dictionary of these which can then be associdated + with TestInstances to get a complete picture of what happened during a test. + MakeGenerator is used for tasks outside of building tests (such as + defconfigs) which is why MakeGoal is a separate class from TestInstance. + """ + def __init__(self, name, text, qemu, make_log, build_log, run_log, + qemu_log): + self.name = name + self.text = text + self.qemu = qemu + self.make_log = make_log + self.build_log = build_log + self.run_log = run_log + self.qemu_log = qemu_log + self.make_state = "waiting" + self.failed = False + self.finished = False + self.reason = None + self.metrics = {} + + def get_error_log(self): + if self.make_state == "waiting": + # Shouldn't ever see this; breakage in the main Makefile itself. + return self.make_log + elif self.make_state == "building": + # Failure when calling the sub-make to build the code + return self.build_log + elif self.make_state == "running": + # Failure in sub-make for "make qemu", qemu probably failed to start + return self.run_log + elif self.make_state == "finished": + # QEMU finished, but timed out or otherwise wasn't successful + return self.qemu_log + + def fail(self, reason): + self.failed = True + self.finished = True + self.reason = reason + + def success(self): + self.finished = True + + def __str__(self): + if self.finished: + if self.failed: + return "[%s] failed (%s: see %s)" % (self.name, self.reason, + self.get_error_log()) + else: + return "[%s] passed" % self.name + else: + return "[%s] in progress (%s)" % (self.name, self.make_state) + + +class MakeGenerator: + """Generates a Makefile which just calls a bunch of sub-make sessions + + In any given test suite we may need to build dozens if not hundreds of + test cases. The cleanest way to parallelize this is to just let Make + do the parallelization, sharing the jobserver among all the different + sub-make targets. + """ + + GOAL_HEADER_TMPL = """.PHONY: {goal} +{goal}: +""" + + MAKE_RULE_TMPL = """\t@echo sanity_test_{phase} {goal} >&2 +\t$(MAKE) -C {directory} O={outdir} V={verb} {args} >{logfile} 2>&1 +""" + + GOAL_FOOTER_TMPL = "\t@echo sanity_test_finished {goal} >&2\n\n" + + re_make = re.compile("sanity_test_([A-Za-z0-9]+) (.+)|$|make[:] \*\*\* [[](.+)[]] Error.+$") + + def __init__(self, base_outdir): + """MakeGenerator constructor + + @param base_outdir Intended to be the base out directory. A make.log + file will be created here which contains the output of the + top-level Make session, as well as the dynamic control Makefile + @param verbose If true, pass V=1 to all the sub-makes which greatly + increases their verbosity + """ + self.goals = {} + if not os.path.exists(base_outdir): + os.makedirs(base_outdir) + self.logfile = os.path.join(base_outdir, "make.log") + self.makefile = os.path.join(base_outdir, "Makefile") + + def _get_rule_header(self, name): + return MakeGenerator.GOAL_HEADER_TMPL.format(goal=name) + + def _get_sub_make(self, name, phase, workdir, outdir, logfile, args): + verb = "1" if VERBOSE else "0" + args = " ".join(args) + return MakeGenerator.MAKE_RULE_TMPL.format(phase=phase, goal=name, + outdir=outdir, + directory=workdir, verb=verb, + args=args, logfile=logfile) + + def _get_rule_footer(self, name): + return MakeGenerator.GOAL_FOOTER_TMPL.format(goal=name) + + def _add_goal(self, outdir): + if not os.path.exists(outdir): + os.makedirs(outdir) + + def add_build_goal(self, name, directory, outdir, args): + """Add a goal to invoke a Kbuild session + + @param name A unique string name for this build goal. The results + dictionary returned by execute() will be keyed by this name. + @param directory Absolute path to working directory, will be passed + to make -C + @param outdir Absolute path to output directory, will be passed to + Kbuild via -O= + @param args Extra command line arguments to pass to 'make', typically + environment variables or specific Make goals + """ + self._add_goal(outdir) + build_logfile = os.path.join(outdir, "build.log") + text = (self._get_rule_header(name) + + self._get_sub_make(name, "building", directory, + outdir, build_logfile, args) + + self._get_rule_footer(name)) + self.goals[name] = MakeGoal(name, text, None, self.logfile, build_logfile, + None, None) + + def add_qemu_goal(self, name, directory, outdir, args, timeout=30): + """Add a goal to build a Zephyr project and then run it under QEMU + + The generated make goal invokes Make twice, the first time it will + build the default goal, and the second will invoke the 'qemu' goal. + The output of the QEMU session will be monitored, and terminated + either upon pass/fail result of the test program, or the timeout + is reached. + + @param name A unique string name for this build goal. The results + dictionary returned by execute() will be keyed by this name. + @param directory Absolute path to working directory, will be passed + to make -C + @param outdir Absolute path to output directory, will be passed to + Kbuild via -O= + @param args Extra command line arguments to pass to 'make', typically + environment variables. Do not pass specific Make goals here. + @param timeout Maximum length of time QEMU session should be allowed + to run before automatically killing it. Default is 30 seconds. + """ + + self._add_goal(outdir) + build_logfile = os.path.join(outdir, "build.log") + run_logfile = os.path.join(outdir, "run.log") + qemu_logfile = os.path.join(outdir, "qemu.log") + + q = QEMUHandler(name, outdir, qemu_logfile, timeout) + args.append("QEMU_PIPE=%s" % q.get_fifo()) + text = (self._get_rule_header(name) + + self._get_sub_make(name, "building", directory, + outdir, build_logfile, args) + + self._get_sub_make(name, "running", directory, + outdir, run_logfile, + args + ["qemu"]) + + self._get_rule_footer(name)) + self.goals[name] = MakeGoal(name, text, q, self.logfile, build_logfile, + run_logfile, qemu_logfile) + + + def add_test_instance(self, ti, build_only=False): + """Add a goal to build/test a TestInstance object + + @param ti TestInstance object to build. The status dictionary returned + by execute() will be keyed by its .name field. + """ + args = ti.test.extra_args[:] + args.extend(["ARCH=%s" % ti.platform.arch.name, + "PLATFORM_CONFIG=%s" % ti.platform.name]) + if ti.platform.qemu_support and not ti.build_only and not build_only: + self.add_qemu_goal(ti.name, ti.test.code_location, ti.outdir, + args, ti.test.timeout) + else: + self.add_build_goal(ti.name, ti.test.code_location, ti.outdir, args) + + def execute(self, callback_fn=None, context=None): + """Execute all the registered build goals + + @param callback_fn If not None, a callback function will be called + as individual goals transition between states. This function + should accept two parameters: a string state and an arbitrary + context object, supplied here + @param context Context object to pass to the callback function. + Type and semantics are specific to that callback function. + @return A dictionary mapping goal names to final status. + """ + + with open(self.makefile, "w") as tf, \ + open(os.devnull, "wb") as devnull, \ + open(self.logfile, "w") as make_log: + # Create our dynamic Makefile and execute it. + # Watch stderr output which is where we will keep + # track of build state + for name, goal in self.goals.iteritems(): + tf.write(goal.text) + tf.write("all: %s\n" % (" ".join(self.goals.keys()))) + tf.flush() + + # os.environ["CC"] = "ccache gcc" FIXME doesn't work + + cmd = ["make", "-k", "-j", str(PARALLEL), "-f", tf.name, "all"] + p = subprocess.Popen(cmd, stderr=subprocess.PIPE, + stdout=devnull) + + for line in iter(p.stderr.readline, b''): + make_log.write(line) + verbose("MAKE: " + repr(line.strip())) + m = MakeGenerator.re_make.match(line) + if not m: + continue + + state, name, error = m.groups() + if error: + goal = self.goals[error] + else: + goal = self.goals[name] + goal.make_state = state + + + if error: + goal.fail("build_error") + else: + if state == "finished": + if goal.qemu: + thread_status, metrics = goal.qemu.get_state() + goal.metrics.update(metrics) + if thread_status == "passed": + goal.success() + else: + goal.fail(thread_status) + else: + goal.success() + + if callback_fn: + callback_fn(context, self.goals, goal) + + p.wait() + return self.goals + + +# "list" - List of strings +# "list:" - List of +# "set" - Set of unordered, unique strings +# "set:" - Set of +# "float" - Floating point +# "int" - Integer +# "bool" - Boolean +# "str" - String + +# XXX Be sure to update __doc__ if you change any of this!! + +arch_valid_keys = {"name" : {"type" : "str", "required" : True}, + "platforms" : {"type" : "list", "required" : True}} + +platform_valid_keys = {"qemu_support" : {"type" : "bool", "default" : False}, + "microkernel_support" : {"type" : "bool", + "default" : True}} + +testcase_valid_keys = {"tags" : {"type" : "set", "required" : True}, + "extra_args" : {"type" : "list"}, + "build_only" : {"type" : "bool", "default" : False}, + "timeout" : {"type" : "int", "default" : 60}, + "arch_whitelist" : {"type" : "set"}, + "platform_whitelist" : {"type" : "set"}, + "config_whitelist" : {"type" : "set"}} + + +class SanityConfigParser: + """Class to read architecture and test case .ini files with semantic checking + """ + def __init__(self, filename): + """Instantiate a new SanityConfigParser object + + @param filename Source .ini file to read + """ + cp = ConfigParser.SafeConfigParser() + cp.readfp(open(filename)) + self.filename = filename + self.cp = cp + + def _cast_value(self, value, typestr): + v = value.strip() + if typestr == "str": + return v + + elif typestr == "float": + return float(v) + + elif typestr == "int": + return int(v) + + elif typestr == "bool": + v = v.lower() + if v == "true" or v == "1": + return True + elif v == "" or v == "false" or v == "0": + return False + raise ConfigurationError(self.filename, + "bad value for boolean: '%s'" % value) + + elif typestr.startswith("list"): + vs = v.split() + if len(typestr) > 4 and typestr[4] == ":": + return [self._cast_value(vsi, typestr[5:]) for vsi in vs] + else: + return vs + + elif typestr.startswith("set"): + vs = v.split() + if len(typestr) > 3 and typestr[3] == ":": + return set([self._cast_value(vsi, typestr[4:]) for vsi in vs]) + else: + return set(vs) + + else: + raise ConfigurationError(self.filename, "unknown type '%s'" % value) + + + def sections(self): + """Get the set of sections within the .ini file + + @return a list of string section names""" + return self.cp.sections() + + def get_section(self, section, valid_keys): + """Get a dictionary representing the keys/values within a section + + @param section The section in the .ini file to retrieve data from + @param valid_keys A dictionary representing the intended semantics + for this section. Each key in this dictionary is a key that could + be specified, if a key is given in the .ini file which isn't in + here, it will generate an error. Each value in this dictionary + is another dictionary containing metadata: + + "default" - Default value if not given + "type" - Data type to convert the text value to. Simple types + supported are "str", "float", "int", "bool" which will get + converted to respective Python data types. "set" and "list" + may also be specified which will split the value by + whitespace (but keep the elements as strings). finally, + "list:" and "set:" may be given which will + perform a type conversion after splitting the value up. + "required" - If true, raise an error if not defined. If false + and "default" isn't specified, a type conversion will be + done on an empty string + @return A dictionary containing the section key-value pairs with + type conversion and default values filled in per valid_keys + """ + + d = {} + cp = self.cp + + if not cp.has_section(section): + raise ConfigurationError(self.filename, "Missing section '%s'" % section) + + for k, v in cp.items(section): + if k not in valid_keys: + raise ConfigurationError(self.filename, + "Unknown config key '%s' in defintiion for '%s'" + % (k, section)) + d[k] = v + + for k, kinfo in valid_keys.iteritems(): + if k not in d: + if "required" in kinfo: + required = kinfo["required"] + else: + required = False + + if required: + raise ConfigurationError(self.filename, + "missing required value for '%s' in section '%s'" + % (k, section)) + else: + if "default" in kinfo: + default = kinfo["default"] + else: + default = self._cast_value("", kinfo["type"]) + d[k] = default + else: + try: + d[k] = self._cast_value(d[k], kinfo["type"]) + except ValueError, ve: + raise ConfigurationError(self.filename, + "bad %s value '%s' for key '%s' in section '%s'" + % (kinfo["type"], d[k], k, section)) + + return d + + +class Platform: + """Class representing metadata for a particular platform + + Maps directly to PLATFORM_CONFIG when building""" + def __init__(self, arch, name, plat_dict): + """Constructor. + + @param arch Architecture object for this platform + @param name String name for this platform, same as PLATFORM_CONFIG + @param plat_dict SanityConfigParser output on the relevant section + in the architecture configuration file which has lots of metadata. + See the Architecture class. + """ + self.name = name + self.qemu_support = plat_dict["qemu_support"] + self.microkernel_support = plat_dict["microkernel_support"] + self.arch = arch + # Gets populated in a separate step + self.defconfig = {"micro" : None, "nano" : None} + pass + + def set_defconfig(self, ktype, defconfig): + """Set defconfig information for a particular kernel type. + + We do this in another step because all the defconfigs are generated + at once from a sub-make, see TestSuite constructor + + @param ktype Kernel type, either "micro" or "nano" + @param defconfig Dictionary containing defconfig information + """ + self.defconfig[ktype] = defconfig + + def get_defconfig(self, ktype): + """Return a dictionary representing the key/value pairs expressed + in the kernel defconfig used for this arch/platform. Used to identify + platform features. + + @param ktype Kernel type, either "micro" or "nano" + @return dictionary corresponding to the defconfig contents. unset + values will not be defined + """ + + if ktype == "micro" and not self.microkernel_support: + raise SanityRuntimeError("Invalid kernel type queried") + + return self.defconfig[ktype] + + def __repr__(self): + return "<%s on %s>" % (self.name, self.arch.name) + + +class Architecture: + """Class representing metadata for a particular architecture + """ + def __init__(self, cfile): + """Architecture constructor + + @param cfile Path to Architecture configuration file, which gives + info about the arch and all the platforms for it + """ + cp = SanityConfigParser(cfile) + self.platforms = [] + + arch = cp.get_section("arch", arch_valid_keys) + + self.name = arch["name"] + + for plat_name in arch["platforms"]: + verbose("Platform: %s" % plat_name) + plat_dict = cp.get_section(plat_name, platform_valid_keys) + self.platforms.append(Platform(self, plat_name, plat_dict)) + + def __repr__(self): + return "" % self.name + + +class TestCase: + """Class representing a test application + """ + makefile_re = re.compile("\s*KERNEL_TYPE\s*[?=]+\s*(micro|nano)\s*") + + def __init__(self, testcase_root, workdir, name, tc_dict): + """TestCase constructor. + + This gets called by TestSuite as it finds and reads testcase.ini files. + Multiple TestCase instances may be generated from a single testcase.ini, + each one corresponds to a section within that file. + + Reads the Makefile inside the testcase directory to figure out the + kernel type for purposes of configuration filtering + + We need to have a unique name for every single test case. Since + a testcase.ini can define multiple tests, the canonical name for + the test case is /. + + @param testcase_root Absolute path to the root directory where + all the test cases live + @param workdir Relative path to the project directory for this + test application from the test_case root. + @param name Name of this test case, corresponding to the section name + in the test case configuration file. For many test cases that just + define one test, can be anything and is usually "test". This is + really only used to distinguish between different cases when + the testcase.ini defines multiple tests + @param tc_dict Dictionary with section values for this test case + from the testcase.ini file + """ + self.code_location = os.path.join(testcase_root, workdir) + self.tags = tc_dict["tags"] + self.extra_args = tc_dict["extra_args"] + self.arch_whitelist = tc_dict["arch_whitelist"] + self.platform_whitelist = tc_dict["platform_whitelist"] + self.config_whitelist = tc_dict["config_whitelist"] + self.timeout = tc_dict["timeout"] + self.build_only = tc_dict["build_only"] + self.path = os.path.join(workdir, name) + self.name = self.path # for now + self.ktype = None + + with open(os.path.join(testcase_root, workdir, "Makefile")) as makefile: + for line in makefile.readlines(): + m = TestCase.makefile_re.match(line) + if m: + self.ktype = m.group(1) + break + if not self.ktype: + raise ConfigurationError(os.path.join(workdir, "Makefile"), + "KERNEL_TYPE not found") + + def __repr__(self): + return self.name + + + +class TestInstance: + """Class representing the execution of a particular TestCase on a platform + + @param test The TestCase object we want to build/execute + @param platform Platform object that we want to build and run against + @param base_outdir Base directory for all test results. The actual + out directory used is // + """ + def __init__(self, test, platform, base_outdir, build_only=False): + self.test = test + self.platform = platform + self.name = os.path.join(platform.name, test.path) + self.outdir = os.path.join(base_outdir, platform.name, test.path) + self.build_only = build_only or test.build_only + + def calculate_sizes(self): + """Get the RAM/ROM sizes of a test case. + + This can only be run after the instance has been executed by + MakeGenerator, otherwise there won't be any binaries to measure. + + @return A SizeCalculator object + """ + if self.test.ktype == "micro": + binary_stem = os.path.join(self.outdir, "microkernel") + else: + binary_stem = os.path.join(self.outdir, "nanokernel") + + return SizeCalculator(binary_stem) + + def __repr__(self): + return "" % (self.test.name, self.platform.name) + + + +class TestSuite: + config_re = re.compile('(CONFIG_[A-Z0-9_]+)[=](.+)$') + + def __init__(self, arch_root, testcase_root, outdir): + # Keep track of which test cases we've filtered out and why + discards = {} + self.arches = {} + self.testcases = {} + self.platforms = [] + self.outdir = outdir + self.instances = {} + self.goals = None + self.discards = None + + arch_root = os.path.abspath(arch_root) + testcase_root = os.path.abspath(testcase_root) + outdir = os.path.abspath(outdir) + + debug("Reading test case configuration files under %s..." % testcase_root) + for dirpath, dirnames, filenames in os.walk(testcase_root, + topdown=True): + if "testcase.ini" in filenames: + verbose("Found test case in " + dirpath) + dirnames[:] = [] + cp = SanityConfigParser(os.path.join(dirpath, "testcase.ini")) + workdir = os.path.relpath(dirpath, testcase_root) + + for section in cp.sections(): + tc_dict = cp.get_section(section, testcase_valid_keys) + tc = TestCase(testcase_root, workdir, section, tc_dict) + self.testcases[tc.name] = tc + + debug("Reading architecture configuration files under %s..." % arch_root) + for dirpath, dirnames, filenames in os.walk(arch_root): + for filename in filenames: + if filename.endswith(".ini"): + fn = os.path.join(dirpath, filename) + verbose("Found arch configuration " + fn) + arch = Architecture(fn) + self.arches[arch.name] = arch + self.platforms.extend(arch.platforms) + + + # Now that we know the full set of arches/platforms, get the defconfig + # information from them by calling Make + info("Building platform defconfigs...") + dlist = {} + config_outdir = os.path.join(outdir, "configs") + mg = MakeGenerator(config_outdir) + + for plat in self.platforms: + ktypes = ["nano"] + if plat.microkernel_support: + ktypes.append("micro") + + for ktype in ktypes: + stem = ktype + "_" + plat.name + + in_defconfig = stem + "_defconfig" + out_config = os.path.join(config_outdir, stem + "_config") + dlist[plat, ktype] = out_config + + args = ["ARCH=" + plat.arch.name, + "KBUILD_DEFCONFIG=" + in_defconfig, + "KCONFIG_CONFIG=" + out_config, "defconfig"] + # FIXME would be nice to use a common outdir for this so that + # conf, gen_idt, etc aren't rebuilt for every plat/ktype combo, + # need a way to avoid different Make processe from clobbering + # each other since they all try to build them simultaneously + mg.add_build_goal(stem, ZEPHYR_BASE, os.path.join(config_outdir, + plat.name, + ktype), args) + + results = mg.execute(None) + + for k, out_config in dlist.iteritems(): + plat, ktype = k + defconfig = {} + with open(out_config, "r") as fp: + for line in fp.readlines(): + m = TestSuite.config_re.match(line) + if not m: + continue + defconfig[m.group(1)] = m.group(2).strip() + plat.set_defconfig(ktype, defconfig) + + self.instances = {} + + def get_last_failed(self): + if not os.path.exists(LAST_SANITY): + return [] + result = [] + with open(LAST_SANITY, "r") as fp: + cr = csv.DictReader(fp) + for row in cr: + if row["passed"] == "True": + continue + test = row["test"] + platform = row["platform"] + result.append((test, platform)) + return result + + def apply_filters(self, platform_filter, arch_filter, tag_filter, + config_filter, testcase_filter, last_failed): + instances = [] + discards = {} + verbose("platform filter: " + str(platform_filter)) + verbose(" arch_filter: " + str(arch_filter)) + verbose(" tag_filter: " + str(tag_filter)) + verbose(" config_filter: " + str(config_filter)) + + if last_failed: + failed_tests = self.get_last_failed() + + if not platform_filter or "default" in platform_filter: + info("Selecting default platforms per test case") + default_platforms = True + platform_filter = [] + else: + default_platforms = False + + if "all" in platform_filter: + info("Selecting all possible platforms per test case") + platform_filter = [] + + for tc_name, tc in self.testcases.iteritems(): + for arch_name, arch in self.arches.iteritems(): + instance_list = [] + for plat in arch.platforms: + instance = TestInstance(tc, plat, self.outdir) + + if tag_filter and not tc.tags.intersection(tag_filter): + discards[instance] = "Command line testcase tag filter" + continue + + if testcase_filter and tc_name not in testcase_filter: + discards[instance] = "Testcase name filter" + continue + + if last_failed and (tc.name, plat.name) not in failed_tests: + discards[instance] = "Passed or skipped during last run" + continue + + if arch_filter and arch_name not in arch_filter: + discards[instance] = "Command line testcase arch filter" + continue + + if tc.arch_whitelist and arch.name not in tc.arch_whitelist: + discards[instance] = "Not in test case arch whitelist" + continue + + if platform_filter and plat.name not in platform_filter: + discards[instance] = "Command line platform filter" + continue + + if tc.platform_whitelist and plat.name not in tc.platform_whitelist: + discards[instance] = "Not in testcase platform whitelist" + continue + + if not plat.microkernel_support and tc.ktype == "micro": + discards[instance] = "No microkernel support for platform" + continue + + defconfig = plat.get_defconfig(tc.ktype) + config_pass = True + # FIXME this is kind of gross clean it up + for cw in tc.config_whitelist: + invert = (cw[0] == "!") + if invert: + cw = cw[1:] + + if "=" in cw: + k, v = cw.split("=") + testval = k not in defconfig or defconfig[k] != v + if invert: + testval = not testval + if testval: + discards[instance] = "%s%s in platform defconfig" % ( + cw, " not" if not invert else "") + config_pass = False + break + else: + testval = cw not in defconfig + if invert: + testval = not testval + if testval: + discards[instance] = "%s%s set in platform defconfig" % ( + cw, " not" if not invert else "") + config_pass = False + break + + if not config_pass: + continue + + instance_list.append(instance) + + if not instance_list: + # Every platform in this arch was rejected already + continue + + if default_platforms: + self.add_instance(instance_list[0]) + for instance in instance_list[1:]: + discards[instance] = "Not in default set for arch" + else: + for instance in instance_list: + self.add_instance(instance) + self.discards = discards + return discards + + def add_instance(self, ti): + self.instances[ti.name] = ti + + def execute(self, cb, cb_context, build_only): + mg = MakeGenerator(self.outdir) + for i in self.instances.values(): + mg.add_test_instance(i, build_only) + self.goals = mg.execute(cb, cb_context) + for name, goal in self.goals.iteritems(): + i = self.instances[name] + if goal.failed: + continue + sc = i.calculate_sizes() + goal.metrics["ram_size"] = sc.get_ram_size() + goal.metrics["rom_size"] = sc.get_rom_size() + return self.goals + + def discard_report(self, filename): + if self.discards == None: + raise SanityRuntimeException("apply_filters() hasn't been run!") + + with open(filename, "wb") as csvfile: + fieldnames = ["test", "arch", "platform", "reason"] + cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) + cw.writeheader() + for instance, reason in self.discards.iteritems(): + rowdict = {"test" : i.test.name, + "arch" : i.platform.arch.name, + "platform" : i.platform.name, + "reason" : reason} + cw.writerow(rowdict) + + def compare_metrics(self, filename): + # name, datatype, lower results better + interesting_metrics = [("ram_size", int, True), + ("rom_size", int, True)] + + if self.goals == None: + raise SanityRuntimeException("execute() hasn't been run!") + + if not os.path.exists(filename): + info("Cannot compare metrics, %s not found" % filename) + return [] + + results = [] + saved_metrics = {} + with open(filename) as fp: + cr = csv.DictReader(fp) + for row in cr: + d = {} + for m, _, _ in interesting_metrics: + d[m] = row[m] + saved_metrics[(row["test"], row["platform"])] = d + + for name, goal in self.goals.iteritems(): + i = self.instances[name] + mkey = (i.test.name, i.platform.name) + if mkey not in saved_metrics: + continue + sm = saved_metrics[mkey] + for metric, mtype, lower_better in interesting_metrics: + if metric not in goal.metrics: + continue + if sm[metric] == "": + continue + delta = goal.metrics[metric] - mtype(sm[metric]) + if ((lower_better and delta > 0) or + (not lower_better and delta < 0)): + results.append((i, metric, goal.metrics[metric], delta)) + return results + + def testcase_report(self, filename): + if self.goals == None: + raise SanityRuntimeException("execute() hasn't been run!") + + with open(filename, "wb") as csvfile: + fieldnames = ["test", "arch", "platform", "passed", "status", + "extra_args", "qemu", "qemu_time", "ram_size", + "rom_size"] + cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) + cw.writeheader() + for name, goal in self.goals.iteritems(): + i = self.instances[name] + rowdict = {"test" : i.test.name, + "arch" : i.platform.arch.name, + "platform" : i.platform.name, + "extra_args" : " ".join(i.test.extra_args), + "qemu" : i.platform.qemu_support} + if goal.failed: + rowdict["passed"] = False + rowdict["status"] = goal.reason + else: + rowdict["passed"] = True + if goal.qemu: + rowdict["qemu_time"] = goal.metrics["qemu_time"] + rowdict["ram_size"] = goal.metrics["ram_size"] + rowdict["rom_size"] = goal.metrics["rom_size"] + cw.writerow(rowdict) + + +def parse_arguments(): + + parser = argparse.ArgumentParser(description = __doc__, + formatter_class = argparse.RawDescriptionHelpFormatter) + + parser.add_argument("-p", "--platform", action="append", + help="Platform filter for testing. If unspecified, default to the " + "set of default platforms in the arch configuration files for " + "the selected arches. May also specify 'all' to match all " + "platforms for the selected arches. Multiple invocations " + "are treated as a logical 'or' relationship") + parser.add_argument("-a", "--arch", action="append", + help="Arch filter for testing. Takes precedence over --platform. " + "If unspecified, test all arches. Multiple invocations " + "are treated as a logical 'or' relationship") + parser.add_argument("-t", "--tag", action="append", + help="Specify tags to restrict which tests to run by tag value. " + "Default is to not do any tag filtering. Multiple invocations " + "are treated as a logical 'or' relationship") + parser.add_argument("-f", "--only-failed", action="store_true", + help="Run only those tests that failed the previous sanity check " + "invocation.") + parser.add_argument("-c", "--config", action="append", + help="Specify platform configuration values filtering. This can be " + "specified two ways: = or just . The " + "defconfig for all platforms, for all kernel types will be " + "checked. For the = case, only match defconfig " + "that have that value defined. For the case, match " + "defconfig that have that value assigned to any value. " + "Prepend a '!' to invert the match.") + parser.add_argument("-s", "--test", action="append", + help="Run only the specified test cases. These are named by " + "/") + parser.add_argument("-l", "--all", action="store_true", + help="Same as --platform all") + + parser.add_argument("-o", "--testcase-report", + help="Output a CSV spreadsheet containing results of the test run") + parser.add_argument("-d", "--discard-report", + help="Output a CSV spreadhseet showing tests that were skipped " + "and why") + parser.add_argument("-y", "--dry-run", action="store_true", + help="Create the filtered list of test cases, but don't actually " + "run them. Useful if you're just interested in " + "--discard-report") + + parser.add_argument("-r", "--release", action="store_true", + help="Update the benchmark database with the results of this test " + "run. Intended to be run by CI when tagging an official " + "release. This database is used as a basis for comparison " + "when looking for deltas in metrics such as footprint") + parser.add_argument("-w", "--warnings-as-errors", action="store_true", + help="Treat warning conditions as errors") + parser.add_argument("-v", "--verbose", action="count", default=0, + help="Emit debugging information, call multiple times to increase " + "verbosity") + parser.add_argument("-i", "--inline-logs", action="store_true", + help="Upon test failure, print relevant log data to stdout " + "instead of just a path to it") + parser.add_argument("-m", "--last-metrics", action="store_true", + help="Instead of comparing metrics from the last --release, " + "compare with the results of the previous sanity check " + "invocation") + parser.add_argument("-u", "--no-update", action="store_true", + help="do not update the results of the last run of the sanity " + "checks") + parser.add_argument("-b", "--build-only", action="store_true", + help="Only build the code, do not execute any of it in QEMU") + parser.add_argument("-j", "--jobs", type=int, + help="Number of cores to use when building, defaults to " + "number of CPUs * 2") + parser.add_argument("-H", "--footprint-threshold", type=float, default=5, + help="When checking test case footprint sizes, warn the user if " + "the new app size is greater then the specified percentage " + "from the last release. Default is 5. 0 to warn on any " + "increase on app size") + + parser.add_argument("-O", "--outdir", + default="%s/sanity-out" % ZEPHYR_BASE, + help="Output directory for logs and binaries.") + parser.add_argument("-C", "--clean", action="store_true", + help="Delete the outdir before building") + parser.add_argument("-T", "--testcase-root", + default="%s/samples" % ZEPHYR_BASE, + help="Base directory to recursively search for test cases. All " + "testcase.ini files under here will be processed") + parser.add_argument("-A", "--arch-root", + default="%s/scripts/sanity_chk/arches" % ZEPHYR_BASE, + help="Directory to search for arch configuration files. All .ini " + "files in the directory will be processed.") + + return parser.parse_args() + +def log_info(filename): + filename = os.path.relpath(filename) + if INLINE_LOGS: + print "{:-^100}".format(filename) + with open(filename) as fp: + sys.stdout.write(fp.read()) + print "{:-^100}".format(filename) + else: + print "\tsee: " + COLOR_YELLOW + filename + COLOR_NORMAL + +def terse_test_cb(instances, goals, goal): + total_tests = len(goals) + total_done = 0 + total_failed = 0 + + for k, g in goals.iteritems(): + if g.finished: + total_done += 1 + if g.failed: + total_failed += 1 + + if goal.failed: + i = instances[goal.name] + info("\n\n{:<25} {:<50} {}FAILED{}: {}".format(i.platform.name, + i.test.name, COLOR_RED, COLOR_NORMAL, goal.reason)) + log_info(goal.get_error_log()) + info("") + + sys.stdout.write("\rtotal complete: %s%3d/%3d%s failed: %s%3d%s" % ( + COLOR_GREEN, total_done, total_tests, COLOR_NORMAL, + COLOR_RED if total_failed > 0 else COLOR_NORMAL, + total_failed, COLOR_NORMAL)) + sys.stdout.flush() + +def chatty_test_cb(instances, goals, goal): + i = instances[goal.name] + + if VERBOSE < 2 and not goal.finished: + return + + if goal.failed: + status = COLOR_RED + "FAILED" + COLOR_NORMAL + ": " + goal.reason + elif goal.finished: + status = COLOR_GREEN + "PASSED" + COLOR_NORMAL + else: + status = goal.make_state + + info("{:<25} {:<50} {}".format(i.platform.name, i.test.name, status)) + if goal.failed: + log_info(goal.get_error_log()) + +def main(): + global VERBOSE, INLINE_LOGS, PARALLEL + args = parse_arguments() + VERBOSE += args.verbose + INLINE_LOGS = args.inline_logs + if args.jobs: + PARALLEL = args.jobs + if args.all: + args.platform = ["all"] + + if os.path.exists(args.outdir) and args.clean: + info("Cleaning output directory " + args.outdir) + shutil.rmtree(args.outdir) + + ts = TestSuite(args.arch_root, args.testcase_root, args.outdir) + discards = ts.apply_filters(args.platform, args.arch, args.tag, args.config, + args.test, args.only_failed) + + if args.discard_report: + ts.discard_report(args.discard_report) + + if VERBOSE: + for i, reason in discards.iteritems(): + debug("{:<25} {:<50} {}SKIPPED{}: {}".format(i.platform.name, + i.test.name, COLOR_YELLOW, COLOR_NORMAL, reason)) + + info("%d tests selected, %d tests discarded due to filters" % + (len(ts.instances), len(discards))) + + if args.dry_run: + return + + if VERBOSE or not TERMINAL: + goals = ts.execute(chatty_test_cb, ts.instances, args.build_only) + else: + goals = ts.execute(terse_test_cb, ts.instances, args.build_only) + print + + deltas = ts.compare_metrics(LAST_SANITY if args.last_metrics + else RELEASE_DATA) + warnings = 0 + if deltas: + for i, metric, value, delta in deltas: + percentage = (float(delta) / float(value - delta)) + if percentage < (args.footprint_threshold / 100.0): + continue + + info("{:<25} {:<50} {}WARNING{}: {} is now {} {:+.2%}".format( + i.platform.name, i.test.name, COLOR_YELLOW, COLOR_NORMAL, + metric, value, percentage)) + warnings += 1 + + if warnings: + info("Deltas based on metrics from last %s" % + ("release" if not args.last_metrics else "run")) + + failed = 0 + for name, goal in goals.iteritems(): + if goal.failed: + failed += 1 + + info("%s%d of %d%s tests passed with %s%d%s warnings" % + (COLOR_RED if failed else COLOR_GREEN, len(goals) - failed, + len(goals), COLOR_NORMAL, COLOR_YELLOW if warnings else COLOR_NORMAL, + warnings, COLOR_NORMAL)) + + if args.testcase_report: + ts.testcase_report(args.testcase_report) + if not args.no_update: + ts.testcase_report(LAST_SANITY) + if args.release: + ts.testcase_report(RELEASE_DATA) + + if failed or (warnings and args.warnings_as_errors): + sys.exit(1) + +if __name__ == "__main__": + main() +