diff --git a/src/libvncserver/main.c b/src/libvncserver/main.c
index 32519f14..d505c52e 100644
--- a/src/libvncserver/main.c
+++ b/src/libvncserver/main.c
@@ -610,6 +610,10 @@ clientInput(void *data)
 
     rfbClientConnectionGone(cl);
 
+    /* We can't reliably join this thread, so detach it here
+       so the associated memory will be freed by the OS. */
+    THREAD_DETACH(cl->client_thread);
+
     return THREAD_ROUTINE_RETURN_VALUE;
 }
 
@@ -1204,10 +1208,7 @@ void rfbShutdownServer(rfbScreenInfoPtr screen,rfbBool disconnectClients) {
       }
 
 #ifdef LIBVNCSERVER_HAVE_LIBPTHREAD
-    if(currentCl->screen->backgroundLoop) {
-      /* Wait for threads to finish. The thread has already been pipe-notified by rfbCloseClient() */
-      pthread_join(currentCl->client_thread, NULL);
-    } else {
+    if(!currentCl->screen->backgroundLoop) {
       /*
 	In threaded mode, rfbClientConnectionGone() is called by the client-to-server thread.
 	Only need to call this here for non-threaded mode.