multithreading windows and embedding python

[freesteel]

I have posted about this problem before. SInce then I found a much
better article to help with embedding python in a multithreaded
application:

http://www.linuxjournal.com/article/3641

I found this article very good and it clarified for me what needs
doing. Now I have an example application that almost works, but it is
far from reliable. If I run this several times I get crashes telling me
that the heap is modified after deallocation. Can anybody else
reproduce this?

At the bottom of this file I left a debug dump, and a stack dump.

Here is my application, compile in windows using a standard windows
application project.




#include <afxmt.h>
#include <afxwin.h>
#include <stdio.h>
#include <Python.h>
#include <Windows.h>
#include <process.h>

static int threadnum = 0;




UINT MyThread(LPVOID lpParam)
{
ASSERT(Py_IsIni tialized());
threadnum++;


PyThreadState* mainThreadState = (PyThreadState *)lpParam;

// get the global lock
PyEval_AcquireL ock();
// get a reference to the PyInterpreterSt ate
PyInterpreterSt ate * mainInterpreter State = mainThreadState->interp;
PyThreadState_S wap(mainThreadS tate);

// create a thread state object for this thread
PyThreadState * myThreadState =
PyThreadState_N ew(mainInterpre terState);
// free the lock
PyEval_ReleaseL ock();


// lock - swap in thread state - swap out thread state - unlock
PyEval_AcquireL ock();
PyThreadState_S wap(myThreadSta te);

int num = 0;
int ret = 0;
ret = PyRun_SimpleStr ing("x = []");
ret = PyRun_SimpleStr ing("for i in range(10):\n x.append(i)");
char cmd[100];
sprintf(cmd, "f = open('c:/windows/temp/test%d.txt', 'w')",
threadnum);
ret = PyRun_SimpleStr ing(cmd);
ret = PyRun_SimpleStr ing("f.write('% s\\n' % x.__str__())");
sprintf(cmd, "f.write('0x%d\ \n')", &myThreadState) ;
ret = PyRun_SimpleStr ing(cmd);
ret = PyRun_SimpleStr ing("f.close()" );

PyThreadState_S wap(NULL);
PyEval_ReleaseL ock();


// clean up
// grab the lock
PyEval_AcquireL ock();
// swap my thread state out of the interpreter
PyThreadState_S wap(NULL);
// clear out any cruft from thread state object
PyThreadState_C lear(myThreadSt ate);
// delete my thread state object
PyThreadState_D elete(myThreadS tate);
// release the lock
PyEval_ReleaseL ock();

return 0;
}

class CMyWinApp : public CWinApp
{
public:
CMyWinApp() { }
BOOL InitInstance()
{
Py_Initialize() ;
PyEval_InitThre ads();

// save a pointer to the main PyThreadState object
PyThreadState * mainThreadState = PyThreadState_G et();
// release the lock
PyEval_ReleaseL ock();

const int nhandles = 100;
HANDLE hnd[nhandles];
CWinThread* pThread[nhandles];
for (int ih = 0; ih < nhandles; ih++)
{
pThread[ih] = AfxBeginThread( MyThread, mainThreadState ,
THREAD_PRIORITY _NORMAL, CREATE_SUSPENDE D);
pThread[ih]->m_bAutoDelet e = false;
pThread[ih]->ResumeThread() ;

hnd[ih] = pThread[ih]->m_hThread;
}

int nwaits, nfails;
do
{
nwaits = 0;
nfails = 0;
for (int ih = 0; ih < nhandles; ih++)
{
DWORD ret = WaitForSingleOb ject(hnd[ih], INFINITE);
switch (ret)
{
case WAIT_OBJECT_0:
printf("WAIT_OB JECT_0\n");
break;

case WAIT_TIMEOUT:
++nwaits;
printf("WAIT_TI MEOUT\n");
break;

case WAIT_FAILED:
++nfails;
printf("WAIT_FA ILED\n");
break;
}
}
}
while (nwaits > 0);
ASSERT(nfails == 0);

// delete all windows threads
for (int ih = 0; ih < nhandles; ++ih)
delete pThread[ih];

PyEval_AcquireL ock();
PyThreadState_S wap(mainThreadS tate);
Py_Finalize();

return TRUE;
};
};

CMyWinApp app;




Debug dump:

'pyembed_test.e xe': Loaded
'C:\mdunschen\p yembed_test\Deb ug\pyembed_test .exe', Symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\ntdll.dll ', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\mscoree.d ll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\kernel32. dll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\advapi32. dll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\rpcrt4.dl l', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\user32.dl l', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\gdi32.dll ', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\comctl32. dll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\shlwapi.d ll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\msvcrt.dl l', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\oleacc.dl l', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\msvcp60.d ll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\ole32.dll ', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\oleaut32. dll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\winspool. drv', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\comdlg32. dll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\shell32.d ll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\python24. dll', No
symbols loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\msvcr71.d ll', Symbols
loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\Win SxS\x86_Microso ft.Windows.Comm on-Controls_6595b6 4144ccf1df_6.0. 2600.2180_x-ww_a84f1ff9\com ctl32.dll',
No symbols loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\Mic rosoft.NET\Fram ework\v1.1.4322 \mscorwks.dll', No symbols
loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\Mic rosoft.NET\Fram ework\v1.1.4322 \fusion.dll', No symbols
loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\Mic rosoft.NET\Fram ework\v1.1.4322 \mscorlib.dll', No symbols
loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\ass embly\NativeIma ges1_v1.1.4322\ mscorlib\1.0.50 00.0__b77a5c561 934e089_ca105f6 f\mscorlib.dll' ,
No symbols loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\Mic rosoft.NET\Fram ework\v1.1.4322 \diasymreader.d ll', No
symbols loaded.
'DefaultDomain' : Loaded
'c:\windows\mic rosoft.net\fram ework\v1.1.4322 \mscorlib.dll', No symbols
loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\Mic rosoft.NET\Fram ework\v1.1.4322 \mscorsn.dll', No symbols
loaded.
'pyembed_test': Loaded
'c:\mdunschen\p yembed_test\Deb ug\pyembed_test .exe', Symbols loaded.
'pyembed_test.e xe': Loaded
'C:\WINDOWS\Mic rosoft.NET\Fram ework\v1.1.4322 \mscorjit.dll', No symbols
loaded.
'pyembed_test.e xe': Loaded 'C:\WINDOWS\sys tem32\uxtheme.d ll', No
symbols loaded.
The thread '_threadstartex ' (0x500) has exited with code 0 (0x0).
The thread '_threadstartex ' (0xadc) has exited with code 0 (0x0).
The thread '<No Name>' (0xadc) has exited with code 0 (0x0).
The thread '<No Name>' (0x500) has exited with code 0 (0x0).
The thread '_threadstartex ' (0x928) has exited with code 0 (0x0).
The thread '<No Name>' (0x928) has exited with code 0 (0x0).
The thread '_threadstartex ' (0xe48) has exited with code 0 (0x0).
The thread '<No Name>' (0xe48) has exited with code 0 (0x0).
The thread '_threadstartex ' (0x984) has exited with code 0 (0x0).
The thread '<No Name>' (0x984) has exited with code 0 (0x0).
The thread '_threadstartex ' (0x108) has exited with code 0 (0x0).
HEAP[pyembed_test.ex e]: HEAP: Free Heap block eb5bd8 modified at eb5c24
after it was freed
Unhandled exception at 0x7c901230 in pyembed_test.ex e: User breakpoint.




Stack dump:[color=blue]
> pyembed_test.ex e!_heap_alloc_d bg(unsigned int nSize=15400960, int nBlockUse=10737 41920, const char * szFileName=0x00 000064, int nLine=1242368) Line 359 + 0x1e C[/color]
pyembed_test.ex e!_heap_alloc_b ase(unsigned int size=100) Line 212 C
pyembed_test.ex e!_heap_alloc_d bg(unsigned int nSize=64, int
nBlockUse=12582 916, const char * szFileName=0x00 4c4dd4, int nLine=311)
Line 397 + 0x9 C
pyembed_test.ex e!_nh_malloc_db g(unsigned int nSize=64, int nhFlag=0,
int nBlockUse=12582 916, const char * szFileName=0x00 4c4dd4, int
nLine=311) Line 260 + 0x15 C
pyembed_test.ex e!_malloc_dbg(u nsigned int nSize=64, int
nBlockUse=12582 916, const char * szFileName=0x00 4c4dd4, int nLine=311)
Line 176 + 0x1b C
pyembed_test.ex e!operator new(unsigned int nSize=64, int
nType=12582916, const char * lpszFileName=0x 004c4dd4, int nLine=311)
Line 403 + 0x15 C++
pyembed_test.ex e!CObject::oper ator new(unsigned int nSize=64, const
char * lpszFileName=0x 004c4dd4, int nLine=311) Line 93 + 0x16 C++
pyembed_test.ex e!AfxBeginThrea d(unsigned int (void *)*
pfnThreadProc=0 x00401b80, void * pParam=0x00df7b 10, int nPriority=0,
unsigned int nStackSize=4, unsigned long dwCreateFlags=0 ,
_SECURITY_ATTRI BUTES * lpSecurityAttrs =0x00000000) Line 311 + 0x11 C++
00aaa43d()
pyembed_test.ex e!CMyWinApp::In itInstance() Line 87 + 0x26 bytes C++
pyembed_test.ex e!AfxWinMain(HI NSTANCE__ * hInstance=0x004 00000,
HINSTANCE__ * hPrevInstance=0 x00000000, char * lpCmdLine=0x001 41f12,
int nCmdShow=5) Line 39 + 0xb C++
pyembed_test.ex e!WinMain(HINST ANCE__ * hInstance=0x004 00000,
HINSTANCE__ * hPrevInstance=0 x00000000, char * lpCmdLine=0x001 41f12,
int nCmdShow=5) Line 25 C++
pyembed_test.ex e!WinMainCRTSta rtup() Line 251 + 0x30 C

[freesteel]

Re: multithreading windows and embedding python


freesteel wrote:
....

pThread[ih] = AfxBeginThread( MyThread, mainThreadState ,
THREAD_PRIORITY _NORMAL, CREATE_SUSPENDE D);

....

Here the call to AfxBeginThread is wrong, there is one argument
missing, it should be:

pThread[ih] = AfxBeginThread( MyThread, mainThreadState ,
THREAD_PRIORITY _NORMAL, 0, CREATE_SUSPENDE D);

Because there are so many default arguments of similar types the
compiler did not notice that I passed 'CREATE_SUSPEND ED' as a stack
size, and use the default 'creation' state of 'start right away' for
the thread.

Don't you love default args and types like 'void*' ?

Martin