diff --git a/include/zephyr/bluetooth/audio/audio.h b/include/zephyr/bluetooth/audio/audio.h
index d3ae8bfdc63..8b0c313ee6c 100644
--- a/include/zephyr/bluetooth/audio/audio.h
+++ b/include/zephyr/bluetooth/audio/audio.h
@@ -2032,11 +2032,10 @@ int bt_audio_stream_stop(struct bt_audio_stream *stream);
  *  bt_audio_broadcast_source_delete().
  *
  *  @param stream Stream object
- *  @param cache True to cache the codec configuration or false to forget it
  *
  *  @return 0 in case of success or negative value in case of error.
  */
-int bt_audio_stream_release(struct bt_audio_stream *stream, bool cache);
+int bt_audio_stream_release(struct bt_audio_stream *stream);
 
 /** @brief Send data to Audio stream
  *
diff --git a/subsys/bluetooth/audio/ascs.c b/subsys/bluetooth/audio/ascs.c
index a307528ab67..d7a70f025ce 100644
--- a/subsys/bluetooth/audio/ascs.c
+++ b/subsys/bluetooth/audio/ascs.c
@@ -437,7 +437,7 @@ static void ascs_iso_disconnected(struct bt_iso_chan *chan, uint8_t reason)
 
 	ep = stream->ep;
 	if (ep->status.state == BT_AUDIO_EP_STATE_RELEASING) {
-		ascs_ep_set_state(ep, BT_AUDIO_EP_STATE_CODEC_CONFIGURED);
+		ascs_ep_set_state(ep, BT_AUDIO_EP_STATE_IDLE);
 	} else {
 		int err;
 
@@ -589,12 +589,17 @@ static void ascs_cp_rsp_success(uint8_t id, uint8_t op)
 	ascs_cp_rsp_add(id, op, BT_ASCS_RSP_SUCCESS, BT_ASCS_REASON_NONE);
 }
 
-static void ase_release(struct bt_ascs_ase *ase, bool cache)
+static void ase_release(struct bt_ascs_ase *ase)
 {
 	int err;
 
 	BT_DBG("ase %p", ase);
 
+	if (ase->ep.status.state == BT_AUDIO_EP_STATE_RELEASING) {
+		/* already releasing */
+		return;
+	}
+
 	if (unicast_server_cb != NULL && unicast_server_cb->release != NULL) {
 		err = unicast_server_cb->release(ase->ep.stream);
 	} else {
@@ -607,11 +612,7 @@ static void ase_release(struct bt_ascs_ase *ase, bool cache)
 		return;
 	}
 
-	if (cache) {
-		ascs_ep_set_state(&ase->ep, BT_AUDIO_EP_STATE_CODEC_CONFIGURED);
-	} else {
-		ascs_ep_set_state(&ase->ep, BT_AUDIO_EP_STATE_RELEASING);
-	}
+	ascs_ep_set_state(&ase->ep, BT_AUDIO_EP_STATE_RELEASING);
 
 	ascs_cp_rsp_success(ASE_ID(ase), BT_ASCS_RELEASE_OP);
 }
@@ -631,11 +632,8 @@ static void ascs_clear(struct bt_ascs *ascs)
 			continue;
 		}
 
-		if (ase->ep.status.state != BT_AUDIO_EP_STATE_RELEASING) {
-			ase_release(ase, false);
-		}
-
-		ascs_ep_set_state(&ase->ep, BT_AUDIO_EP_STATE_IDLE);
+		/* ase_process will handle the final state transition into idle state */
+		ase_release(ase);
 	}
 }
 
@@ -689,29 +687,6 @@ static void ase_disable(struct bt_ascs_ase *ase)
 	ascs_cp_rsp_success(ASE_ID(ase), BT_ASCS_DISABLE_OP);
 }
 
-static void ascs_detach(struct bt_ascs *ascs)
-{
-	int i;
-
-	BT_DBG("ascs %p conn %p", ascs, ascs->conn);
-
-	/* Update address in case it has changed */
-	ascs->id = ascs->conn->id;
-	bt_addr_le_copy(&ascs->peer, &ascs->conn->le.dst);
-
-	/* TODO: Store the ASES in the settings? */
-
-	for (i = 0; i < ASE_COUNT; i++) {
-		struct bt_ascs_ase *ase = &ascs->ases[i];
-
-		if (ase->ep.status.state != BT_AUDIO_EP_STATE_IDLE &&
-		    ase->ep.status.state != BT_AUDIO_EP_STATE_RELEASING) {
-			/* Cache if disconnected with codec configured */
-			ase_release(ase, true);
-		}
-	}
-}
-
 static void disconnected(struct bt_conn *conn, uint8_t reason)
 {
 	int i;
@@ -726,11 +701,7 @@ static void disconnected(struct bt_conn *conn, uint8_t reason)
 		}
 
 		/* Release existing ASEs if not bonded */
-		if (!bt_addr_le_is_bonded(conn->id, &conn->le.dst)) {
-			ascs_clear(ascs);
-		} else {
-			ascs_detach(ascs);
-		}
+		ascs_clear(ascs);
 
 		bt_conn_unref(ascs->conn);
 		ascs->conn = NULL;
@@ -898,7 +869,10 @@ static void ase_process(struct k_work *work)
 
 	ascs_ep_get_status(&ase->ep, &ase_buf);
 
-	bt_gatt_notify(ase->ascs->conn, ase->ep.server.attr, ase_buf.data, ase_buf.len);
+	if (ase->ascs->conn != NULL) {
+		bt_gatt_notify(ase->ascs->conn, ase->ep.server.attr,
+			       ase_buf.data, ase_buf.len);
+	}
 
 	if (ase->ep.status.state == BT_AUDIO_EP_STATE_RELEASING) {
 		__ASSERT(ase->ep.stream, "stream is NULL");
@@ -2231,7 +2205,7 @@ static ssize_t ascs_release(struct bt_ascs *ascs, struct net_buf_simple *buf)
 			continue;
 		}
 
-		ase_release(ase, false);
+		ase_release(ase);
 	}
 
 	return buf->size;
diff --git a/subsys/bluetooth/audio/stream.c b/subsys/bluetooth/audio/stream.c
index c6054b1a95f..1efed06e2dd 100644
--- a/subsys/bluetooth/audio/stream.c
+++ b/subsys/bluetooth/audio/stream.c
@@ -1185,13 +1185,13 @@ int bt_audio_stream_disable(struct bt_audio_stream *stream)
 	return 0;
 }
 
-int bt_audio_stream_release(struct bt_audio_stream *stream, bool cache)
+int bt_audio_stream_release(struct bt_audio_stream *stream)
 {
 	uint8_t state;
 	uint8_t role;
 	int err;
 
-	BT_DBG("stream %p cache %s", stream, cache ? "true" : "false");
+	BT_DBG("stream %p", stream);
 
 	CHECKIF(stream == NULL || stream->ep == NULL || stream->conn == NULL) {
 		BT_DBG("Invalid stream");
@@ -1222,7 +1222,7 @@ int bt_audio_stream_release(struct bt_audio_stream *stream, bool cache)
 		err = bt_unicast_client_release(stream);
 	} else if (IS_ENABLED(CONFIG_BT_AUDIO_UNICAST_SERVER) &&
 		   role == BT_HCI_ROLE_PERIPHERAL) {
-		err = bt_unicast_server_release(stream, cache);
+		err = bt_unicast_server_release(stream);
 	} else {
 		err = -EOPNOTSUPP;
 	}
diff --git a/subsys/bluetooth/audio/unicast_server.c b/subsys/bluetooth/audio/unicast_server.c
index 421a3c0f8b1..ee01b989eae 100644
--- a/subsys/bluetooth/audio/unicast_server.c
+++ b/subsys/bluetooth/audio/unicast_server.c
@@ -158,7 +158,7 @@ int bt_unicast_server_disable(struct bt_audio_stream *stream)
 	return 0;
 }
 
-int bt_unicast_server_release(struct bt_audio_stream *stream, bool cache)
+int bt_unicast_server_release(struct bt_audio_stream *stream)
 {
 	int err;
 
@@ -172,15 +172,10 @@ int bt_unicast_server_release(struct bt_audio_stream *stream, bool cache)
 		return err;
 	}
 
-	if (cache) {
-		ascs_ep_set_state(stream->ep,
-				  BT_AUDIO_EP_STATE_CODEC_CONFIGURED);
-	} else {
-		/* ase_process will set the state to IDLE after sending the
-		 * notification, finalizing the release
-		 */
-		ascs_ep_set_state(stream->ep, BT_AUDIO_EP_STATE_RELEASING);
-	}
+	/* ase_process will set the state to IDLE after sending the
+	 * notification, finalizing the release
+	 */
+	ascs_ep_set_state(stream->ep, BT_AUDIO_EP_STATE_RELEASING);
 
 	return 0;
 }
diff --git a/subsys/bluetooth/audio/unicast_server.h b/subsys/bluetooth/audio/unicast_server.h
index 57874e2d6e4..acc3071581a 100644
--- a/subsys/bluetooth/audio/unicast_server.h
+++ b/subsys/bluetooth/audio/unicast_server.h
@@ -17,4 +17,4 @@ int bt_unicast_server_metadata(struct bt_audio_stream *stream,
 			       struct bt_codec_data meta[],
 			       size_t meta_count);
 int bt_unicast_server_disable(struct bt_audio_stream *stream);
-int bt_unicast_server_release(struct bt_audio_stream *stream, bool cache);
+int bt_unicast_server_release(struct bt_audio_stream *stream);
diff --git a/subsys/bluetooth/shell/audio.c b/subsys/bluetooth/shell/audio.c
index dcceef0d7ac..4878edaefcb 100644
--- a/subsys/bluetooth/shell/audio.c
+++ b/subsys/bluetooth/shell/audio.c
@@ -767,7 +767,7 @@ static int cmd_release(const struct shell *sh, size_t argc, char *argv[])
 		return -ENOEXEC;
 	}
 
-	err = bt_audio_stream_release(default_stream, false);
+	err = bt_audio_stream_release(default_stream);
 	if (err) {
 		shell_error(sh, "Unable to release Channel");
 		return -ENOEXEC;
diff --git a/tests/bluetooth/bsim_bt/bsim_test_audio/src/unicast_client_test.c b/tests/bluetooth/bsim_bt/bsim_test_audio/src/unicast_client_test.c
index 1fcc1aa55f2..1e0b73a2683 100644
--- a/tests/bluetooth/bsim_bt/bsim_test_audio/src/unicast_client_test.c
+++ b/tests/bluetooth/bsim_bt/bsim_test_audio/src/unicast_client_test.c
@@ -323,7 +323,7 @@ static size_t release_streams(size_t stream_cnt)
 
 		UNSET_FLAG(flag_stream_released);
 
-		err = bt_audio_stream_release(&g_streams[i], false);
+		err = bt_audio_stream_release(&g_streams[i]);
 		if (err != 0) {
 			FAIL("Unable to release stream[%zu]: %d", i, err);
 			return 0;