From 910bb8bc566c9381494f2248148930b9ff7e20c9 Mon Sep 17 00:00:00 2001 From: Mark Vejvoda Date: Fri, 17 May 2013 00:08:26 +0000 Subject: [PATCH] - attempt to have more clean shutdown of threads on game exit --- source/glest_game/main/main.cpp | 21 ++- .../include/platform/common/base_thread.h | 12 +- .../shared_lib/include/platform/sdl/thread.h | 11 +- .../sources/platform/common/base_thread.cpp | 18 ++- .../platform/common/simple_threads.cpp | 4 +- .../sources/platform/posix/ircclient.cpp | 29 +++- .../sources/platform/sdl/thread.cpp | 124 ++++++++++++++++-- 7 files changed, 196 insertions(+), 23 deletions(-) diff --git a/source/glest_game/main/main.cpp b/source/glest_game/main/main.cpp index b1560274..ba000cc9 100644 --- a/source/glest_game/main/main.cpp +++ b/source/glest_game/main/main.cpp @@ -207,6 +207,7 @@ static void cleanupProcessObjects() { //printf("Closing IRC CLient %d\n",__LINE__); ircClient->disconnect(); + ircClient->signalQuit(); ircClient = NULL; /* @@ -248,21 +249,39 @@ static void cleanupProcessObjects() { //printf("Closing IRC CLient %d\n",__LINE__); + Thread::shutdownThreads(); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("start running threads = " MG_SIZE_T_SPECIFIER "\n",Thread::getThreadList().size()); time_t elapsed = time(NULL); int lastLazyThreadDump = 0; for(;Thread::getThreadList().size() > 0 && difftime((long int)time(NULL),elapsed) <= 10;) { //sleep(0); + if(difftime((long int)time(NULL),elapsed) > 1) { if(lastLazyThreadDump != (int)difftime((long int)time(NULL),elapsed)) { lastLazyThreadDump = difftime((long int)time(NULL),elapsed); printf("Waiting for the following threads to exit [" MG_SIZE_T_SPECIFIER "]:\n",Thread::getThreadList().size()); + //std::auto_ptr baseThreadTest(new FileCRCPreCacheThread()); + for(int i = 0; i < Thread::getThreadList().size(); ++i) { + //Thread *thr = Thread::getThreadList()[i]; + //printf("#1 Lagging thread typeid: %d [%s]\n,",typeid(thr),typeid(thr).name()); + + //BaseThread *baseThread = dynamic_cast(Thread::getThreadList()[i]); BaseThread *baseThread = dynamic_cast(Thread::getThreadList()[i]); - printf("Thread index: %d isBaseThread: %d, Name: [%s]\n",i,(baseThread != NULL),(baseThread != NULL ? baseThread->getUniqueID().c_str() : "")); + + printf("Thread index: %d ptr [%p] isBaseThread: %d, Name: [%s]\n",i,baseThread,(baseThread != NULL),(baseThread != NULL ? baseThread->getUniqueID().c_str() : "")); + //printf("#2 Lagging thread typeid: %d [%s]\n,",typeid(baseThread),typeid(baseThread).name()); + + //if(baseThread != NULL && baseThread->getRunningStatus() == false) { + // baseThread->kill(); + //} + //BaseThread *baseThread2 = dynamic_cast(baseThreadTest.get()); + //printf("#3 Thread index: %d isBaseThread: %d, Name: [%s]\n",i,(baseThread2 != NULL),(baseThread2 != NULL ? baseThread2->getUniqueID().c_str() : "")); + //printf("#3 Lagging thread typeid: %d [%s]\n,",typeid(baseThread2),typeid(baseThread2).name()); } } } diff --git a/source/shared_lib/include/platform/common/base_thread.h b/source/shared_lib/include/platform/common/base_thread.h index f54db28b..54019600 100644 --- a/source/shared_lib/include/platform/common/base_thread.h +++ b/source/shared_lib/include/platform/common/base_thread.h @@ -91,10 +91,14 @@ public: Mutex * getMutexThreadObjectAccessor() { return &mutexThreadObjectAccessor; } - template - T * getGenericData() { return genericData; } - template - void setGenericData(T *value) { genericData = value; } + template + T * getGenericData() { + return genericData; + } + template + void setGenericData(T *value) { + genericData = value; + } static bool isThreadDeleted(void *ptr); }; diff --git a/source/shared_lib/include/platform/sdl/thread.h b/source/shared_lib/include/platform/sdl/thread.h index c364c6e8..a9cebb20 100644 --- a/source/shared_lib/include/platform/sdl/thread.h +++ b/source/shared_lib/include/platform/sdl/thread.h @@ -58,21 +58,30 @@ public: private: SDL_Thread* thread; - + bool deleteAfterExecute; static Mutex mutexthreadList; static vector threadList; +protected: + void addThreadToList(); + void removeThreadFromList(); + public: Thread(); virtual ~Thread(); static std::vector getThreadList(); + static void shutdownThreads(); + + void setDeleteAfterExecute(bool value) { deleteAfterExecute = value; } + bool getDeleteAfterExecute() const { return deleteAfterExecute; } void start(); virtual void execute()=0; void setPriority(Thread::Priority threadPriority); void suspend(); void resume(); + void kill(); private: static int beginExecution(void *param); diff --git a/source/shared_lib/sources/platform/common/base_thread.cpp b/source/shared_lib/sources/platform/common/base_thread.cpp index d6233614..7bfc3a84 100644 --- a/source/shared_lib/sources/platform/common/base_thread.cpp +++ b/source/shared_lib/sources/platform/common/base_thread.cpp @@ -24,7 +24,7 @@ namespace Shared { namespace PlatformCommon { Mutex BaseThread::mutexMasterThreadList; std::map BaseThread::masterThreadList; -BaseThread::BaseThread() : Thread(), ptr(NULL) { +BaseThread::BaseThread() : Thread(), ptr(NULL), genericData(NULL) { if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__); ptr = this; @@ -45,14 +45,23 @@ BaseThread::BaseThread() : Thread(), ptr(NULL) { } BaseThread::~BaseThread() { + + //printf("In ~BaseThread Line: %d uniqueID [%s]\n",__LINE__,uniqueID.c_str()); + if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s]\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str()); bool ret = shutdownAndWait(); if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s] ret [%d] END\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str(),ret); + //printf("In ~BaseThread Line: %d uniqueID [%s]\n",__LINE__,uniqueID.c_str()); + MutexSafeWrapper safeMutexMasterList(&mutexMasterThreadList); + //printf("In ~BaseThread Line: %d uniqueID [%s]\n",__LINE__,uniqueID.c_str()); + if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s] ret [%d] END\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str(),ret); + //printf("In ~BaseThread Line: %d uniqueID [%s]\n",__LINE__,uniqueID.c_str()); + if(masterThreadList.find(this) == masterThreadList.end()) { if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s] ret [%d] END\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str(),ret); @@ -62,14 +71,21 @@ BaseThread::~BaseThread() { } if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s] ret [%d] END\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str(),ret); + //printf("In ~BaseThread Line: %d uniqueID [%s]\n",__LINE__,uniqueID.c_str()); + masterThreadList[this]--; if(masterThreadList[this] <= 0) { if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s] ret [%d] END\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str(),ret); masterThreadList.erase(this); } + + //printf("In ~BaseThread Line: %d uniqueID [%s]\n",__LINE__,uniqueID.c_str()); + if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s] ret [%d] END\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str(),ret); safeMutexMasterList.ReleaseLock(); if(SystemFlags::getSystemSettingType(SystemFlags::debugSystem).enabled) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] uniqueID [%s] ret [%d] END\n",__FILE__,__FUNCTION__,__LINE__,uniqueID.c_str(),ret); + + //printf("In ~BaseThread Line: %d uniqueID [%s] [%p]\n",__LINE__,uniqueID.c_str(),this); } bool BaseThread::isThreadDeleted(void *ptr) { diff --git a/source/shared_lib/sources/platform/common/simple_threads.cpp b/source/shared_lib/sources/platform/common/simple_threads.cpp index 84297cae..6c251b8b 100644 --- a/source/shared_lib/sources/platform/common/simple_threads.cpp +++ b/source/shared_lib/sources/platform/common/simple_threads.cpp @@ -343,7 +343,9 @@ vector FileCRCPreCacheThread::getPendingTextureList(int maxTextures SimpleTaskThread::SimpleTaskThread( SimpleTaskCallbackInterface *simpleTaskInterface, unsigned int executionCount, unsigned int millisecsBetweenExecutions, - bool needTaskSignal) : BaseThread() { + bool needTaskSignal) : BaseThread(), + simpleTaskInterface(NULL), + overrideShutdownTask(NULL) { this->simpleTaskInterface = simpleTaskInterface; this->executionCount = executionCount; this->millisecsBetweenExecutions = millisecsBetweenExecutions; diff --git a/source/shared_lib/sources/platform/posix/ircclient.cpp b/source/shared_lib/sources/platform/posix/ircclient.cpp index fcf34bdc..24a8f5c5 100644 --- a/source/shared_lib/sources/platform/posix/ircclient.cpp +++ b/source/shared_lib/sources/platform/posix/ircclient.cpp @@ -567,10 +567,10 @@ void IRCThread::signalQuit() { MutexSafeWrapper safeMutex1(&mutexIRCSession,string(__FILE__) + "_" + intToStr(__LINE__)); irc_cmd_quit(ircSession, "MG Bot is closing!"); safeMutex1.ReleaseLock(); - - BaseThread::signalQuit(); - hasJoinedChannel = false; + hasJoinedChannel = false; } + BaseThread::signalQuit(); + #else BaseThread::signalQuit(); #endif @@ -802,22 +802,29 @@ void IRCThread::execute() { return; } + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); + for(int iAttempts=1; this->getQuitStatus() == false && iAttempts <= 7; ++iAttempts) { //if(irc_run(ircSession)) { int run_result = irc_run_session(ircSession); if(run_result && this->getQuitStatus() == false) { + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); + if(SystemFlags::VERBOSE_MODE_ENABLED || IRCThread::debugEnabled) printf ("===> IRC Could not run the session: %s run_result = %d\n", irc_strerror (irc_errno(ircSession)), run_result); printf ("===> IRC Could not run the session: %s run_result = %d\n", irc_strerror (irc_errno(ircSession)), run_result); } } + + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); #else for(;this->getQuitStatus() == false;) { sleep(50); } #endif if(SystemFlags::VERBOSE_MODE_ENABLED || IRCThread::debugEnabled) printf ("===> IRC exiting IRC CLient!\n"); + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); } catch(const exception &ex) { SystemFlags::OutputDebug(SystemFlags::debugError,"In [%s::%s Line: %d] Error [%s]\n",__FILE__,__FUNCTION__,__LINE__,ex.what()); @@ -829,19 +836,27 @@ void IRCThread::execute() { } if(SystemFlags::getSystemSettingType(SystemFlags::debugNetwork).enabled) SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s Line: %d] IRC thread is exiting\n",__FILE__,__FUNCTION__,__LINE__); + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); } + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); // Delete ourself when the thread is done (no other actions can happen after this // such as the mutex which modifies the running status of this method MutexSafeWrapper safeMutex(&mutexIRCCB,string(__FILE__) + "_" + intToStr(__LINE__)); IRCCallbackInterface *cb = getCallbackObj(false); if(cb != NULL) { + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); cb->IRC_CallbackEvent(IRC_evt_exitThread, NULL, NULL, 0); } safeMutex.ReleaseLock(); + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In IRCThread() calling delete ...\n"); - delete this; + + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); + //delete this; + setDeleteAfterExecute(true); } int IRCThread::irc_run_session(irc_session_t * session) { @@ -899,12 +914,18 @@ int IRCThread::irc_run_session(irc_session_t * session) { } IRCThread::~IRCThread() { + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); + if(SystemFlags::VERBOSE_MODE_ENABLED) printf("In ~IRCThread() ...\n"); if(IRCThread::globalCacheContainerName != NULL) { + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); + IRCThread * &ircClient = CacheManager::getCachedItem< IRCThread * >(IRCThread::globalCacheContainerName); ircClient = NULL; } + + //printf("In ~IRCThread Line: %d [%p]\n",__LINE__,this); } void normalizeNick(char *nick) { diff --git a/source/shared_lib/sources/platform/sdl/thread.cpp b/source/shared_lib/sources/platform/sdl/thread.cpp index e3b822ba..f6cbc7fa 100644 --- a/source/shared_lib/sources/platform/sdl/thread.cpp +++ b/source/shared_lib/sources/platform/sdl/thread.cpp @@ -16,6 +16,8 @@ #include #include "platform_util.h" #include "platform_common.h" +#include "base_thread.h" +#include "time.h" #include using namespace std; @@ -28,31 +30,112 @@ vector Thread::threadList; auto_ptr Mutex::mutexMutexList(new Mutex(CODE_AT_LINE)); vector Mutex::mutexList; +class ThreadAutoCleanup : public BaseThread +{ +protected: + Mutex mutexPendingCleanupList; + vector pendingCleanupList; + + bool cleanupPendingThreads() { + MutexSafeWrapper safeMutex(&mutexPendingCleanupList); + if(pendingCleanupList.empty() == false) { + for(unsigned int index = 0; index < pendingCleanupList.size(); ++index) { + delete pendingCleanupList[index]; + } + pendingCleanupList.clear(); + return true; + } + return false; + } +public: + ThreadAutoCleanup() : BaseThread() { + removeThreadFromList(); + } + virtual ~ThreadAutoCleanup() { + } + virtual void execute() { + RunningStatusSafeWrapper runningStatus(this); + for(;getQuitStatus() == false;) { + if(cleanupPendingThreads() == false) { + if(getQuitStatus() == false) { + sleep(200); + } + } + } + cleanupPendingThreads(); + } + void addThread(Thread *thread) { + MutexSafeWrapper safeMutex(&mutexPendingCleanupList); + pendingCleanupList.push_back(thread); + safeMutex.ReleaseLock(); + } +}; + +static auto_ptr cleanupThread; // ===================================== // Threads // ===================================== -Thread::Thread() { +Thread::Thread() : thread(NULL), deleteAfterExecute(false) { + addThreadToList(); +} + +void Thread::addThreadToList() { MutexSafeWrapper safeMutex(&Thread::mutexthreadList); Thread::threadList.push_back(this); safeMutex.ReleaseLock(); - thread = NULL; +} +void Thread::removeThreadFromList() { + MutexSafeWrapper safeMutex(&Thread::mutexthreadList); + std::vector::iterator iterFind = std::find(Thread::threadList.begin(),Thread::threadList.end(),this); + if(iterFind == Thread::threadList.end()) { + if(this != cleanupThread.get()) { + char szBuf[8096]=""; + snprintf(szBuf,8095,"In [%s::%s Line: %d] iterFind == Thread::threadList.end()",extractFileFromDirectoryPath(__FILE__).c_str(),__FUNCTION__,__LINE__); + throw megaglest_runtime_error(szBuf); + } + } + else { + Thread::threadList.erase(iterFind); + } + safeMutex.ReleaseLock(); +} + +void Thread::shutdownThreads() { + MutexSafeWrapper safeMutex(&Thread::mutexthreadList); + for(unsigned int index = 0; index < Thread::threadList.size(); ++index) { + BaseThread *thread = dynamic_cast(Thread::threadList[index]); + if(thread && thread->getRunningStatus() == true) { + thread->signalQuit(); + } + } + safeMutex.ReleaseLock(); + + if(cleanupThread.get() != 0) { + sleep(0); + cleanupThread->signalQuit(); + + time_t elapsed = time(NULL); + for(;cleanupThread->getRunningStatus() == true && + difftime((long int)time(NULL),elapsed) <= 5;) { + sleep(100); + } + cleanupThread.reset(0); + } } Thread::~Thread() { + //printf("In ~Thread Line: %d [%p] thread = %p\n",__LINE__,this,thread); + if(thread != NULL) { SDL_WaitThread(thread, NULL); thread = NULL; } - MutexSafeWrapper safeMutex(&Thread::mutexthreadList); - std::vector::iterator iterFind = std::find(Thread::threadList.begin(),Thread::threadList.end(),this); - if(iterFind == Thread::threadList.end()) { - char szBuf[8096]=""; - snprintf(szBuf,8095,"In [%s::%s Line: %d] iterFind == Thread::threadList.end()",extractFileFromDirectoryPath(__FILE__).c_str(),__FUNCTION__,__LINE__); - throw megaglest_runtime_error(szBuf); - } - Thread::threadList.erase(iterFind); - safeMutex.ReleaseLock(); + //printf("In ~Thread Line: %d [%p] thread = %p\n",__LINE__,this,thread); + + removeThreadFromList(); + + //printf("In ~Thread Line: %d [%p] thread = %p\n",__LINE__,this,thread); } std::vector Thread::getThreadList() { @@ -71,6 +154,8 @@ void Thread::start() { snprintf(szBuf,8095,"In [%s::%s Line: %d] thread == NULL",extractFileFromDirectoryPath(__FILE__).c_str(),__FUNCTION__,__LINE__); throw megaglest_runtime_error(szBuf); } + + //printf("In Thread::start Line: %d [%p] thread = %p\n",__LINE__,this,thread); } void Thread::setPriority(Thread::Priority threadPriority) { @@ -85,10 +170,27 @@ int Thread::beginExecution(void* data) { snprintf(szBuf,8095,"In [%s::%s Line: %d] thread == NULL",extractFileFromDirectoryPath(__FILE__).c_str(),__FUNCTION__,__LINE__); throw megaglest_runtime_error(szBuf); } + //printf("In Thread::execute Line: %d thread = %p\n",__LINE__,thread); + thread->execute(); + + //printf("In Thread::execute Line: %d thread = %p\n",__LINE__,thread); + + if(thread->deleteAfterExecute == true) { + if(cleanupThread.get() == NULL) { + cleanupThread.reset(new ThreadAutoCleanup()); + cleanupThread->start(); + } + cleanupThread->addThread(thread); + } return 0; } +void Thread::kill() { + SDL_KillThread(thread); + thread = NULL; +} + void Thread::suspend() { NOIMPL; }