shell: rtt: Add detection of host presence

If host is not reading RTT data (because there is no PC connection
or RTT reading application is not running on the host), thread will
stuck continuously trying to write to RTT. All threads with equal or
lower priority are blocked then. Adding detection of that case and
if host is not reading data for configurable period then data is
dropped until host accepts new data.

Similar solution is using in RTT logging backend.

Signed-off-by: Krzysztof Chruściński <krzysztof.chruscinski@nordicsemi.no>
diff --git a/subsys/shell/backends/Kconfig.backends b/subsys/shell/backends/Kconfig.backends
index 0baf777..a78c924 100644
--- a/subsys/shell/backends/Kconfig.backends
+++ b/subsys/shell/backends/Kconfig.backends
@@ -185,6 +185,7 @@
 	bool "RTT backend"
 	select CONSOLE
 	select RTT_CONSOLE
+	select SEGGER_RTT_CUSTOM_LOCKING
 	depends on USE_SEGGER_RTT
 	help
 	  Enable RTT backend.
@@ -206,6 +207,23 @@
 	  Select index of up-buffer used for shell output, by default it uses
 	  terminal up-buffer and its settings.
 
+config SHELL_BACKEND_RTT_RETRY_CNT
+	int "Number of retries"
+	default 4
+	help
+	  Number of TX retries before dropping the data and assuming that
+	  RTT session is inactive.
+
+config SHELL_BACKEND_RTT_RETRY_DELAY_MS
+	int "Delay between TX retries in milliseconds"
+	default 5
+	help
+	  Sleep period between TX retry attempts. During RTT session, host pulls
+	  data periodically. Period starts from 1-2 milliseconds and can be
+	  increased if traffic on RTT increases (also from host to device). In
+	  case of heavy traffic data can be lost and it may be necessary to
+	  increase delay or number of retries.
+
 config SHELL_RTT_RX_POLL_PERIOD
 	int "RX polling period (in milliseconds)"
 	default 10
diff --git a/subsys/shell/backends/shell_rtt.c b/subsys/shell/backends/shell_rtt.c
index 6490975..4a874ba 100644
--- a/subsys/shell/backends/shell_rtt.c
+++ b/subsys/shell/backends/shell_rtt.c
@@ -9,6 +9,12 @@
 #include <SEGGER_RTT.h>
 #include <zephyr/logging/log.h>
 
+#define RTT_LOCK() \
+	COND_CODE_0(CONFIG_SHELL_BACKEND_RTT_BUFFER, (SEGGER_RTT_LOCK()), ())
+
+#define RTT_UNLOCK() \
+	COND_CODE_0(CONFIG_SHELL_BACKEND_RTT_BUFFER, (SEGGER_RTT_UNLOCK()), ())
+
 #if IS_ENABLED(CONFIG_LOG_BACKEND_RTT)
 BUILD_ASSERT(!(CONFIG_SHELL_BACKEND_RTT_BUFFER == CONFIG_LOG_BACKEND_RTT_BUFFER),
 	     "Conflicting log RTT backend enabled on the same channel");
@@ -25,7 +31,8 @@
 
 LOG_MODULE_REGISTER(shell_rtt, CONFIG_SHELL_RTT_LOG_LEVEL);
 
-static bool rtt_blocking;
+static bool panic_mode;
+static bool host_present;
 
 static void timer_handler(struct k_timer *timer)
 {
@@ -78,29 +85,81 @@
 	struct shell_rtt *sh_rtt = (struct shell_rtt *)transport->ctx;
 
 	if (blocking) {
-		rtt_blocking = true;
 		k_timer_stop(&sh_rtt->timer);
 	}
 
 	return 0;
 }
 
+static inline bool is_panic_mode(void)
+{
+	return panic_mode;
+}
+
+static inline bool is_sync_mode(void)
+{
+	return (IS_ENABLED(CONFIG_LOG_MODE_IMMEDIATE) && IS_ENABLED(CONFIG_SHELL_LOG_BACKEND)) ||
+		is_panic_mode();
+}
+
+static void on_failed_write(int retry_cnt)
+{
+	if (retry_cnt == 0) {
+		host_present = false;
+	} else if (is_sync_mode()) {
+		k_busy_wait(USEC_PER_MSEC *
+				CONFIG_SHELL_BACKEND_RTT_RETRY_DELAY_MS);
+	} else {
+		k_msleep(CONFIG_SHELL_BACKEND_RTT_RETRY_DELAY_MS);
+	}
+}
+
+static void on_write(int retry_cnt)
+{
+	host_present = true;
+	if (is_panic_mode()) {
+		/* In panic mode block on each write until host reads it. This
+		 * way it is ensured that if system resets all messages are read
+		 * by the host. While pending on data being read by the host we
+		 * must also detect situation where host is disconnected.
+		 */
+		while (SEGGER_RTT_HasDataUp(CONFIG_SHELL_BACKEND_RTT_BUFFER) &&
+			host_present) {
+			on_failed_write(retry_cnt--);
+		}
+	}
+
+}
+
 static int write(const struct shell_transport *transport,
 		 const void *data, size_t length, size_t *cnt)
 {
 	struct shell_rtt *sh_rtt = (struct shell_rtt *)transport->ctx;
-	const uint8_t *data8 = (const uint8_t *)data;
+	int ret = 0;
+	int retry_cnt = CONFIG_SHELL_BACKEND_RTT_RETRY_CNT;
 
-	if (rtt_blocking) {
-		*cnt = SEGGER_RTT_WriteNoLock(CONFIG_SHELL_BACKEND_RTT_BUFFER, data8, length);
-		while (SEGGER_RTT_HasDataUp(CONFIG_SHELL_BACKEND_RTT_BUFFER)) {
-			/* empty */
+	do {
+		if (!is_sync_mode()) {
+			RTT_LOCK();
+			ret = SEGGER_RTT_WriteSkipNoLock(CONFIG_SHELL_BACKEND_RTT_BUFFER,
+							 data, length);
+			RTT_UNLOCK();
+		} else {
+			ret = SEGGER_RTT_WriteSkipNoLock(CONFIG_SHELL_BACKEND_RTT_BUFFER,
+							 data, length);
 		}
-	} else {
-		*cnt = SEGGER_RTT_Write(CONFIG_SHELL_BACKEND_RTT_BUFFER, data8, length);
-	}
+
+		if (ret) {
+			on_write(retry_cnt);
+		} else if (host_present) {
+			retry_cnt--;
+			on_failed_write(retry_cnt);
+		} else {
+		}
+	} while ((ret == 0) && host_present);
 
 	sh_rtt->handler(SHELL_TRANSPORT_EVT_TX_RDY, sh_rtt->context);
+	*cnt = length;
 
 	return 0;
 }