diff --git a/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_defconfig b/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_defconfig
index 4c19444708b..e697012a690 100644
--- a/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_defconfig
+++ b/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_defconfig
@@ -1,9 +1,6 @@
 CONFIG_SOC_QEMU_CORTEX_A53=y
 CONFIG_BOARD_QEMU_CORTEX_A53=y
 CONFIG_ARM_ARCH_TIMER=y
-CONFIG_QEMU_ICOUNT=y
-CONFIG_QEMU_ICOUNT_SHIFT=4
-CONFIG_QEMU_ICOUNT_SLEEP=y
 
 # Cache management
 CONFIG_CACHE_MANAGEMENT=y
@@ -17,3 +14,8 @@ CONFIG_UART_CONSOLE=y
 
 # Enable serial port
 CONFIG_UART_INTERRUPT_DRIVEN=y
+
+# Avoid timing skew in tests
+CONFIG_QEMU_ICOUNT=y
+CONFIG_QEMU_ICOUNT_SHIFT=4
+CONFIG_QEMU_ICOUNT_SLEEP=y
diff --git a/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_smp_defconfig b/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_smp_defconfig
index 40603c131ec..496e633201a 100644
--- a/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_smp_defconfig
+++ b/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_smp_defconfig
@@ -1,7 +1,9 @@
 CONFIG_SOC_QEMU_CORTEX_A53=y
 CONFIG_BOARD_QEMU_CORTEX_A53=y
 CONFIG_ARM_ARCH_TIMER=y
-CONFIG_QEMU_ICOUNT=n
+
+# Cache management
+CONFIG_CACHE_MANAGEMENT=y
 
 # Enable UART driver
 CONFIG_SERIAL=y
@@ -10,15 +12,21 @@ CONFIG_SERIAL=y
 CONFIG_CONSOLE=y
 CONFIG_UART_CONSOLE=y
 
-# PSCI is supported
-CONFIG_PM_CPU_OPS=y
-
 # Enable serial port
 CONFIG_UART_INTERRUPT_DRIVEN=y
 
+# icount does not work well with SMP
+CONFIG_QEMU_ICOUNT=n
+
+# We have multiple QEMU-A53 boards, so let us exercise ARMV8_A_NS on this one
+# (plus it is needed for SMP)
+CONFIG_ARMV8_A_NS=y
+
+# PSCI is supported with NS
+CONFIG_PM_CPU_OPS=y
+
+# SMP-related
 CONFIG_SMP=y
 CONFIG_MP_MAX_NUM_CPUS=2
-CONFIG_CACHE_MANAGEMENT=y
 CONFIG_TIMEOUT_64BIT=y
-CONFIG_ARMV8_A_NS=y
 CONFIG_MAX_THREAD_BYTES=3
diff --git a/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_xip_defconfig b/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_xip_defconfig
index 68e7fea9754..d2dbdc1fc51 100644
--- a/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_xip_defconfig
+++ b/boards/arm64/qemu_cortex_a53/qemu_cortex_a53_xip_defconfig
@@ -1,7 +1,6 @@
 CONFIG_SOC_QEMU_CORTEX_A53=y
 CONFIG_BOARD_QEMU_CORTEX_A53=y
 CONFIG_ARM_ARCH_TIMER=y
-CONFIG_QEMU_ICOUNT=n
 
 # Cache management
 CONFIG_CACHE_MANAGEMENT=y
@@ -9,12 +8,17 @@ CONFIG_CACHE_MANAGEMENT=y
 # Enable UART driver
 CONFIG_SERIAL=y
 
-# Enable XIP
-CONFIG_XIP=y
-
 # Enable console
 CONFIG_CONSOLE=y
 CONFIG_UART_CONSOLE=y
 
 # Enable serial port
 CONFIG_UART_INTERRUPT_DRIVEN=y
+
+# Avoid timing skew in tests
+CONFIG_QEMU_ICOUNT=y
+CONFIG_QEMU_ICOUNT_SHIFT=4
+CONFIG_QEMU_ICOUNT_SLEEP=y
+
+# Enable XIP
+CONFIG_XIP=y