diff --git a/src/connector.c b/src/connector.c
index 2990d997..95c3d27c 100644
--- a/src/connector.c
+++ b/src/connector.c
@@ -450,8 +450,8 @@ retry:
 		if (interrupted())
 			goto retry;
 		LOGERR("Failed to accept on connector socket, retrying in 5s");
-		sleep(5);
-		goto retry;
+		ret = 1;
+		goto out;
 	}
 
 	buf = recv_unix_msg(sockd);