diff --git a/subsys/net/ip/icmpv4.c b/subsys/net/ip/icmpv4.c
index 0ea8d40ba46..308ebd21256 100644
--- a/subsys/net/ip/icmpv4.c
+++ b/subsys/net/ip/icmpv4.c
@@ -455,7 +455,12 @@ static int icmpv4_handle_echo_request(struct net_icmp_ctx *ctx,
 	    net_ipv4_is_addr_bcast(net_pkt_iface(pkt),
 				   (struct in_addr *)ip_hdr->dst)) {
 		src = net_if_ipv4_select_src_addr(net_pkt_iface(pkt),
-						  (struct in_addr *)ip_hdr->dst);
+						  (struct in_addr *)ip_hdr->src);
+
+		if (net_ipv4_is_addr_unspecified(src)) {
+			NET_DBG("DROP: No src address match");
+			goto drop;
+		}
 	} else {
 		src = (struct in_addr *)ip_hdr->dst;
 	}
diff --git a/subsys/net/ip/icmpv6.c b/subsys/net/ip/icmpv6.c
index 65c01a510c7..b380a5f4558 100644
--- a/subsys/net/ip/icmpv6.c
+++ b/subsys/net/ip/icmpv6.c
@@ -130,7 +130,12 @@ static int icmpv6_handle_echo_request(struct net_icmp_ctx *ctx,
 
 	if (net_ipv6_is_addr_mcast((struct in6_addr *)ip_hdr->dst)) {
 		src = net_if_ipv6_select_src_addr(net_pkt_iface(pkt),
-						  (struct in6_addr *)ip_hdr->dst);
+						  (struct in6_addr *)ip_hdr->src);
+
+		if (net_ipv6_is_addr_unspecified(src)) {
+			NET_DBG("DROP: No src address match");
+			goto drop;
+		}
 	} else {
 		src = (struct in6_addr *)ip_hdr->dst;
 	}