From 0f08e9518f633c1dfda789b86b572b7f73e86572 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Feb 2015 11:42:35 +1100 Subject: [PATCH 1/2] Add helpers for rw timed locks --- src/libckpool.c | 38 ++++++++++++++++++++++++++++++++++++-- src/libckpool.h | 4 +++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index bed7a816..655e79c2 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -123,6 +123,23 @@ bool ck_completion_timeout(void *fn, void *fnarg, int timeout) #define GUNLOCK(_lock, _file, _func, _line) #define INITLOCK(_typ, _lock, _file, _func, _line) +int _mutex_timedlock(pthread_mutex_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line) +{ + tv_t now; + ts_t abs; + int ret; + + tv_time(&now); + tv_to_ts(&abs, &now); + abs.tv_sec += timeout; + + TRYLOCK(lock, file, func, line); + ret = pthread_mutex_timedlock(lock, &abs); + DIDLOCK(ret, lock, file, func, line); + + return ret; +} + void _mutex_lock(pthread_mutex_t *lock, const char *file, const char *func, const int line) { GETLOCK(lock, file, func, line); @@ -149,7 +166,7 @@ int _mutex_trylock(pthread_mutex_t *lock, __maybe_unused const char *file, __may return ret; } -int _mutex_timedlock(pthread_mutex_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line) +int _wr_timedlock(pthread_rwlock_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line) { tv_t now; ts_t abs; @@ -160,7 +177,7 @@ int _mutex_timedlock(pthread_mutex_t *lock, int timeout, __maybe_unused const ch abs.tv_sec += timeout; TRYLOCK(lock, file, func, line); - ret = pthread_mutex_timedlock(lock, &abs); + ret = pthread_rwlock_timedwrlock(lock, &abs); DIDLOCK(ret, lock, file, func, line); return ret; @@ -182,6 +199,23 @@ int _wr_trylock(pthread_rwlock_t *lock, __maybe_unused const char *file, __maybe return ret; } +int _rd_timedlock(pthread_rwlock_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line) +{ + tv_t now; + ts_t abs; + int ret; + + tv_time(&now); + tv_to_ts(&abs, &now); + abs.tv_sec += timeout; + + TRYLOCK(lock, file, func, line); + ret = pthread_rwlock_timedrdlock(lock, &abs); + DIDLOCK(ret, lock, file, func, line); + + return ret; +} + void _rd_lock(pthread_rwlock_t *lock, const char *file, const char *func, const int line) { GETLOCK(lock, file, func, line); diff --git a/src/libckpool.h b/src/libckpool.h index cb7c1d01..870e1f0b 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -386,13 +386,15 @@ void create_pthread(pthread_t *thread, void *(*start_routine)(void *), void *arg void join_pthread(pthread_t thread); bool ck_completion_timeout(void *fn, void *fnarg, int timeout); +int _mutex_timedlock(pthread_mutex_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line); void _mutex_lock(pthread_mutex_t *lock, const char *file, const char *func, const int line); void _mutex_unlock_noyield(pthread_mutex_t *lock, const char *file, const char *func, const int line); void _mutex_unlock(pthread_mutex_t *lock, const char *file, const char *func, const int line); int _mutex_trylock(pthread_mutex_t *lock, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line); -int _mutex_timedlock(pthread_mutex_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line); +int _wr_timedlock(pthread_rwlock_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line); void _wr_lock(pthread_rwlock_t *lock, const char *file, const char *func, const int line); int _wr_trylock(pthread_rwlock_t *lock, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line); +int _rd_timedlock(pthread_rwlock_t *lock, int timeout, __maybe_unused const char *file, __maybe_unused const char *func, __maybe_unused const int line); void _rd_lock(pthread_rwlock_t *lock, const char *file, const char *func, const int line); void _rw_unlock(pthread_rwlock_t *lock, const char *file, const char *func, const int line); void _rd_unlock_noyield(pthread_rwlock_t *lock, const char *file, const char *func, const int line); From c572d580eb6e2b70f9389a7307d2e49b38e74c51 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Feb 2015 12:02:07 +1100 Subject: [PATCH 2/2] Add lock contention testing warning at 10 seconds to all lock grabbing and fail if a lock is unable to be grabbed for more than a minute --- src/libckpool.c | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index 655e79c2..350ae09e 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -140,11 +140,24 @@ int _mutex_timedlock(pthread_mutex_t *lock, int timeout, __maybe_unused const ch return ret; } +/* Make every locking attempt warn if we're unable to get the lock for more + * than 10 seconds and fail if we can't get it for longer than a minute. */ void _mutex_lock(pthread_mutex_t *lock, const char *file, const char *func, const int line) { + int ret, retries = 0; + GETLOCK(lock, file, func, line); - if (unlikely(pthread_mutex_lock(lock))) +retry: + ret = _mutex_timedlock(lock, 10, file, func, line); + if (unlikely(ret)) { + if (likely(ret == ETIMEDOUT)) { + LOGERR("WARNING: Prolonged mutex lock contention from %s %s:%d", file, func, line); + if (++retries < 6) + goto retry; + quitfrom(1, file, func, line, "FAILED TO GRAB MUTEX!"); + } quitfrom(1, file, func, line, "WTF MUTEX ERROR ON LOCK!"); + } GOTLOCK(lock, file, func, line); } @@ -185,9 +198,20 @@ int _wr_timedlock(pthread_rwlock_t *lock, int timeout, __maybe_unused const char void _wr_lock(pthread_rwlock_t *lock, const char *file, const char *func, const int line) { + int ret, retries = 0; + GETLOCK(lock, file, func, line); - if (unlikely(pthread_rwlock_wrlock(lock))) - quitfrom(1, file, func, line, "WTF WRLOCK ERROR ON LOCK!"); +retry: + ret = _wr_timedlock(lock, 10, file, func, line); + if (unlikely(ret)) { + if (likely(ret == ETIMEDOUT)) { + LOGERR("WARNING: Prolonged write lock contention from %s %s:%d", file, func, line); + if (++retries < 6) + goto retry; + quitfrom(1, file, func, line, "FAILED TO GRAB WRITE LOCK!"); + } + quitfrom(1, file, func, line, "WTF ERROR ON WRITE LOCK!"); + } GOTLOCK(lock, file, func, line); } @@ -218,9 +242,20 @@ int _rd_timedlock(pthread_rwlock_t *lock, int timeout, __maybe_unused const char void _rd_lock(pthread_rwlock_t *lock, const char *file, const char *func, const int line) { + int ret, retries = 0; + GETLOCK(lock, file, func, line); - if (unlikely(pthread_rwlock_rdlock(lock))) - quitfrom(1, file, func, line, "WTF RDLOCK ERROR ON LOCK!"); +retry: + ret = _rd_timedlock(lock, 10, file, func, line); + if (unlikely(ret)) { + if (likely(ret == ETIMEDOUT)) { + LOGERR("WARNING: Prolonged read lock contention from %s %s:%d", file, func, line); + if (++retries < 6) + goto retry; + quitfrom(1, file, func, line, "FAILED TO GRAB READ LOCK!"); + } + quitfrom(1, file, func, line, "WTF ERROR ON READ LOCK!"); + } GOTLOCK(lock, file, func, line); }