This is the third question in a series.
I am writing an application that uses Distributed Objects for communications
among several processes. I developed and tested on OSX 10.4.11 for several
months, and all seemed well. A few weeks back I finally upgraded to 10.5.5, and
now I'm having a variety of problems. The app is complex and getting more so,
so I'm writing simple programs to try to demonstrate the problems I encounter
and rule out errors in my own code obscured by all the complexity.
A server process vends an object (as the root object of an NSConnection). Then
it runs several threads to service incoming invocations from clients. That is,
each thread adds its NSRunLoop to the NSConnection, and then the thread runs its
runloop. Thus the vended object may be invoked on any of these threads, and is
often invoked several times concurrently, if several clients are sending
invocations. In my simple example, several instances of a single-threaded
client connect to the vended object (get a proxy NSDistantObject to it) and then
hammer on it with rapid method invocations.
On OSX 10.5.5, this setup will crash the server with as few as two clients (or
perhaps even just one, though I haven't been able to reproduce that recently).
The more clients, the sooner the server will crash. I've observed many
different call stacks leading to the crashes, but many suggest memory has been
changed after being freed, and diagnostic messages emitted by the runtime to the
console also indicate this.
On OSX 10.4, it's harder to crash the server, but it does crash. In my testing,
it may crash once 4 clients connect to the server (saturating my 4 CPU cores).
Any idea what I'm doing wrong? Anybody else seeing such problems? Is nobody
else using multi-threaded servers? NSConnection's public methods -addRunLoop:
and -runInNewThread seem to indicate that it's intended to support such.
Here's code, and following that some representative output and crash dumps.
--------------------------------------------------------------------------------
// server.mm
#import "unistd.h"
#import "shared.h"
#import "log.h"
// server class
@interface CServer : NSObject <IServer> {}
- (void) dealloc;
@end
// server factory class
@interface CServerFactory : NSObject <IServerFactory> {}
@end
@implementation CServer
- (int) getInt {
return 42;
}
- (void) dealloc {
[super dealloc];
}
@end
@implementation CServerFactory
- (int) getServer: (out byref id<IServer>*) server {
*server = [[CServer alloc] init];
[*server autorelease];
return 1;
}
@end
@interface CServerThread : NSObject {}
+ (void) run: (id) object;
@end
int main (int argc, char * const argv[]) {
log("server...\n");
NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
NSMachPort* port = nil;
NSConnection* connection = nil;
CServerFactory* factory = nil;
do {
// vend server object...
// port
port = [[NSMachPort alloc] init];
if (port == nil) {
log("server: error: port is nil\n");
break;
}
[port autorelease];
// connection
connection = [NSConnection connectionWithReceivePort:port sendPort:nil];
if (connection == nil) {
log("server: error: connection is nil\n");
break;
}
[connection enableMultipleThreads];
// create factory object to vend
factory = [[CServerFactory alloc] init];
if (factory == nil) {
log("server: error: factory is nil\n");
break;
}
[factory autorelease];
// vend object
[connection setRootObject: factory];
// publish name
NSMachBootstrapServer* namesrv = [NSMachBootstrapServer sharedInstance];
if ([namesrv registerPort: port name: @SERVER_NAME] == NO) {
log("server: error: port not registered\n");
break;
}
// detach new threads to run runloops for the connection
for (unsigned n = 0; n < 4; n++) {
[ NSThread detachNewThreadSelector: @selector(run:)
toTarget: [CServerThread class]
withObject: connection ];
}
// run the server until terminated
while (true) {
sleep(10);
}
} while (false);
[port invalidate];
[pool release];
return 0;
}
@implementation CServerThread
+ (void) run: (id) object {
log("server: +[CServerThread run:]\n");
NSRunLoop* runLoop = [NSRunLoop currentRunLoop];
NSConnection* connection = (NSConnection*)object;
[connection addRunLoop: runLoop];
while (true) {
NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
// let exceptions crash
//@try {
[ runLoop runMode: NSDefaultRunLoopMode
beforeDate: [NSDate distantFuture] ];
//} @catch (id x) {
// log("server: caught exception\n");
//}
[pool release];
}
}
@end
--------------------------------------------------------------------------------
// client.mm
#import <unistd.h>
#import "shared.h"
#import "log.h"
int main (int argc, char * const argv[]) {
log("client...\n");
bool extraRelease = false;
if (argc > 1 && argv[1][0] == 'x') {
extraRelease = true;
}
NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
id<IServerFactory> factory = nil;
do {
// lookup intermediate factory object...
NSConnection* connection
= [ NSConnection connectionWithRegisteredName: @SERVER_NAME
host: nil ];
if (connection == nil) {
log("client: error: connection is nil (server not found?)\n");
break;
}
[connection enableMultipleThreads];
id proxy = [connection rootProxy];
if (proxy == nil) {
log("client: error: proxy is nil (server not found?)\n");
break;
}
factory = (id<IServerFactory>)proxy;
id<IServer> server;
char* activity;
// call server in loop until terminated
while (true) {
server = nil;
NSAutoreleasePool* innerPool = [[NSAutoreleasePool alloc] init];
@try {
// get a server object from factory
activity = "getServer";
int factoryResult = [factory getServer: &server];
if (factoryResult != 1) {
log("client: getServer returned %d\n", factoryResult);
break;
}
if (server == nil) {
log("client: getServer returned nil server\n");
break;
}
// call server object
activity = "getInt";
int serverResult = [server getInt];
if (serverResult != 42) {
log("client: getInt returned %d\n", serverResult);
break;
}
} @catch (NSException* x) {
log( "client: exception thrown from %s: %s\n",
activity,
[[x reason] UTF8String] );
break;
}
[innerPool release];
if (extraRelease) {
// if out-param proxies are considered owned by the caller,
// then this would be appropriate:
[server release];
}
}
} while (false);
[pool release];
return 0;
}
--------------------------------------------------------------------------------
// shared.h
#ifndef __SHARED_H__
#define __SHARED_H__
#import <Foundation/NSObject.h>
#import <Foundation/NSAutoreleasePool.h>
#import <Foundation/NSPort.h>
#import <Foundation/NSConnection.h>
#import <Foundation/NSPortNameServer.h>
#import <Foundation/NSRunLoop.h>
#import <Foundation/NSException.h>
#import <Foundation/NSString.h>
#import <Foundation/NSThread.h>
#define SERVER_NAME "MultiThreadServer"
// server interface
@protocol IServer <NSObject>
- (int) getInt;
@end
// server factory interface
@protocol IServerFactory <NSObject>
- (int) getServer: (out byref id<IServer>*) server;
@end
#endif // ndef'd __SHARED_H__
--------------------------------------------------------------------------------
Test runs on OSX 10.5.5
<run 'server' under debugger>
20081111 17:12:19 [ 9486] <0xa065dfa0> server...
20081111 17:12:19 [ 9486] <0xb0103000> server: +[CServerThread run:]
20081111 17:12:19 [ 9486] <0xb0207000> server: +[CServerThread run:]
20081111 17:12:19 [ 9486] <0xb0081000> server: +[CServerThread run:]
20081111 17:12:19 [ 9486] <0xb0185000> server: +[CServerThread run:]
[Switching to process 9486 thread 0x1703]
Program received signal: “EXC_BAD_ACCESS�.
#0 0x926895a4 in CFRelease
#1 0x92659dbb in CFDictionaryRemoveValue
#2 0x95335c2f in removeConversation
#3 0x952fa14b in -[NSConnection handleRequest:sequence:]
#4 0x952f994d in -[NSConnection handlePortCoder:]
#5 0x952f947e in -[NSConcretePortCoder dispatch]
#6 0x952f8be3 in __NSFireMachPort
#7 0x92663635 in __CFMachPortPerform
#8 0x92687908 in CFRunLoopRunSpecific
#9 0x92687cf8 in CFRunLoopRunInMode
#10 0x952f3135 in -[NSRunLoop(NSRunLoop) runMode:beforeDate:]
#11 0x00002989 in +[CServerThread run:] at server.mm:148
#12 0x952bebad in -[NSThread main]
#13 0x952be754 in __NSThread__main__
#14 0x9569b6f5 in _pthread_start
#15 0x9569b5b2 in thread_start
CFRelease:
0x92689580 <+0000> push %ebp
0x92689581 <+0001> mov %esp,%ebp
0x92689583 <+0003> push %esi
0x92689584 <+0004> push %ebx
0x92689585 <+0005> call 0x9268958a <CFRelease+10>
0x9268958a <+0010> pop %ebx
0x9268958b <+0011> sub $0x10,%esp
0x9268958e <+0014> mov 0x8(%ebp),%esi
0x92689591 <+0017> mov 0xdbfbf56(%ebx),%eax
0x92689597 <+0023> test %eax,%eax
0x92689599 <+0025> je 0x926895a4 <CFRelease+36>
0x9268959b <+0027> mov %esi,(%esp)
0x9268959e <+0030> call *%eax
0x926895a0 <+0032> test %al,%al
0x926895a2 <+0034> jne 0x92689620 <CFRelease+160>
0x926895a4 <+0036> movzwl 0x5(%esi),%edx <<<<<<<<<<<<<<<<<<<<<<<<< Crash Here
0x926895a8 <+0040> cmp 0xdbfbf66(%ebx),%edx
0x926895ae <+0046> jb 0x926895f0 <CFRelease+112>
0x926895b0 <+0048> mov $0x1,%eax
0x926895b5 <+0053> test %al,%al
0x926895b7 <+0055> je 0x92689610 <CFRelease+144>
0x926895b9 <+0057> mov 0xdc10b5e(%ebx),%ecx
0x926895bf <+0063> test %ecx,%ecx
0x926895c1 <+0065> je 0x9268968c <CFRelease+268>
0x926895c7 <+0071> mov 0xdc10b5e(%ebx),%eax
0x926895cd <+0077> mov %esi,(%esp)
0x926895d0 <+0080> mov %eax,0x4(%esp)
0x926895d4 <+0084> call *0xe3b9a9a(%ebx)
0x926895da <+0090> add $0x10,%esp
0x926895dd <+0093> pop %ebx
0x926895de <+0094> pop %esi
0x926895df <+0095> leave
0x926895e0 <+0096> ret
-------------------------------------------------------------------------------------
<run 'server' under debugger>
20081111 16:59:54 [ 9397] <0xa065dfa0> server...
20081111 16:59:54 [ 9397] <0xb0081000> server: +[CServerThread run:]
20081111 16:59:54 [ 9397] <0xb0103000> server: +[CServerThread run:]
20081111 16:59:54 [ 9397] <0xb0207000> server: +[CServerThread run:]
20081111 16:59:54 [ 9397] <0xb0185000> server: +[CServerThread run:]
(gdb) b malloc_error_break
Breakpoint 1 at 0x95751131
(gdb) continue
server(9397,0xb0103000) malloc: *** error for object 0x10f760: incorrect
checksum for freed object - object was probably modified after being freed.
*** set a breakpoint in malloc_error_break to debug
server(9397,0xb0207000) malloc: *** error for object 0x10f760: incorrect
checksum for freed object - object was probably modified after being freed.
*** set a breakpoint in malloc_error_break to debug
server(9397,0xb0185000) malloc: *** error for object 0x10d870: incorrect
checksum for freed object - object was probably modified after being freed.
*** set a breakpoint in malloc_error_break to debug
server(9397,0xb0081000) malloc: *** error for object 0x10f760: incorrect
checksum for freed object - object was probably modified after being freed.
*** set a breakpoint in malloc_error_break to debug
#0 0x95751131 in malloc_error_break
#1 0x9574c11f in szone_error
#2 0x95673e2e in tiny_free_list_add_ptr
#3 0x95670eba in szone_free
#4 0x92618c48 in __CFArrayReleaseValues
#5 0x92689788 in _CFRelease
#6 0x952f8c0a in __NSFireMachPort
#7 0x92663635 in __CFMachPortPerform
#8 0x92687908 in CFRunLoopRunSpecific
#9 0x92687cf8 in CFRunLoopRunInMode
#10 0x952f3135 in -[NSRunLoop(NSRunLoop) runMode:beforeDate:]
#11 0x00002989 in +[CServerThread run:] at server.mm:148
#12 0x952bebad in -[NSThread main]
#13 0x952be754 in __NSThread__main__
#14 0x9569b6f5 in _pthread_start
#15 0x9569b5b2 in thread_start
-------------------------------------------------------------------------------------
<run 'server' under debugger>
20081111 17:05:49 [ 9444] <0xa065dfa0> server...
20081111 17:05:49 [ 9444] <0xb0103000> server: +[CServerThread run:]
20081111 17:05:49 [ 9444] <0xb0081000> server: +[CServerThread run:]
20081111 17:05:49 [ 9444] <0xb0207000> server: +[CServerThread run:]
20081111 17:05:49 [ 9444] <0xb0185000> server: +[CServerThread run:]
(gdb) b malloc_error_break
Note: breakpoint 1 also set at pc 0x95751131.
Breakpoint 2 at 0x95751131
(gdb) continue
server(9444,0xb0081000) malloc: *** error for object 0x10ca60: double free
*** set a breakpoint in malloc_error_break to debug
#0 0x95751131 in malloc_error_break
#1 0x9574c11f in szone_error
#2 0x95670743 in szone_free
#3 0x92658094 in __CFDictionaryGrow
#4 0x9265893f in CFDictionarySetValue
#5 0x95335aed in addConversation
#6 0x952fbde7 in -[NSConnection _shouldDispatch:invocation:sequence:coder:]
#7 0x952f9f8e in -[NSConnection handleRequest:sequence:]
#8 0x952f994d in -[NSConnection handlePortCoder:]
#9 0x952f947e in -[NSConcretePortCoder dispatch]
#10 0x952f8be3 in __NSFireMachPort
#11 0x92663635 in __CFMachPortPerform
#12 0x92687908 in CFRunLoopRunSpecific
#13 0x92687cf8 in CFRunLoopRunInMode
#14 0x952f3135 in -[NSRunLoop(NSRunLoop) runMode:beforeDate:]
#15 0x00002989 in +[CServerThread run:] at server.mm:148
#16 0x952bebad in -[NSThread main]
#17 0x952be754 in __NSThread__main__
#18 0x9569b6f5 in _pthread_start
#19 0x9569b5b2 in thread_start
-------------------------------------------------------------------------------------
Test runs on OSX 10.4.11
> ./server
20081112 12:40:07 [ 492] <0xa000d000> server...
20081112 12:40:07 [ 492] <0x180ec00> server: +[CServerThread run:]
20081112 12:40:07 [ 492] <0x180ca00> server: +[CServerThread run:]
20081112 12:40:07 [ 492] <0x180ce00> server: +[CServerThread run:]
20081112 12:40:07 [ 492] <0x1814000> server: +[CServerThread run:]
server(492,0x180ca00) malloc: *** Deallocation of a pointer not malloced:
0x310003;
This could be a double free(), or free() called with the middle of an allocated
block;
Try setting environment variable MallocHelp to see tools to help debug
<excerpt from crash log>
Thread: 1
Exception: EXC_BAD_ACCESS (0x0001)
Codes: KERN_PROTECTION_FAILURE (0x0002) at 0x00000000
Thread 1 Crashed:
0 libobjc.A.dylib 0x90a584c7 objc_msgSend + 23
1 com.apple.Foundation 0x9284e2df -[NSConnection(NSOldArchivingPriv)
handleReleasedProxies:length:] + 162
2 com.apple.Foundation 0x9284e140 -[NSConnection
_cleanupAndAuthenticate:sequence:conversation:invocation:raise:] + 318
3 com.apple.Foundation 0x9284c9ee -[NSConnection handleRequest:sequence:]
+ 914
4 com.apple.Foundation 0x9284c3b2 -[NSConnection handlePortCoder:] +
922
5 com.apple.Foundation 0x9284bff3 -[NSConcretePortCoder dispatch] + 188
6 com.apple.Foundation 0x9284ba50 __NSFireMachPort + 307
7 com.apple.CoreFoundation 0x9083c2fd __CFMachPortPerform + 136
8 com.apple.CoreFoundation 0x9082c5a1 CFRunLoopRunSpecific + 2904
9 com.apple.CoreFoundation 0x9082ba42 CFRunLoopRunInMode + 61
10 com.apple.Foundation 0x9282fd6a -[NSRunLoop runMode:beforeDate:] +
182
11 server 0x00002989 +[CServerThread run:] + 231
12 com.apple.Foundation 0x927fa39c forkThreadForFunction + 123
13 libSystem.B.dylib 0x90024227 _pthread_body + 84
-------------------------------------------------------------------------------------
> ./server
20081112 12:54:41 [ 519] <0xa000d000> server...
20081112 12:54:42 [ 519] <0x180ec00> server: +[CServerThread run:]
20081112 12:54:42 [ 519] <0x180ca00> server: +[CServerThread run:]
20081112 12:54:42 [ 519] <0x180ce00> server: +[CServerThread run:]
20081112 12:54:42 [ 519] <0x1814000> server: +[CServerThread run:]
Bus error
<excerpt from crash log>
Thread: 4
Exception: EXC_BAD_ACCESS (0x0001)
Codes: KERN_PROTECTION_FAILURE (0x0002) at 0x00000000
Thread 4 Crashed:
0 libobjc.A.dylib 0x90a584c7 objc_msgSend + 23
1 com.apple.Foundation 0x9284baac __NSFireMachPort + 399
2 com.apple.CoreFoundation 0x9083c2fd __CFMachPortPerform + 136
3 com.apple.CoreFoundation 0x9082c5a1 CFRunLoopRunSpecific + 2904
4 com.apple.CoreFoundation 0x9082ba42 CFRunLoopRunInMode + 61
5 com.apple.Foundation 0x9282fd6a -[NSRunLoop runMode:beforeDate:] +
182
6 server 0x00002989 +[CServerThread run:] + 231
7 com.apple.Foundation 0x927fa39c forkThreadForFunction + 123
8 libSystem.B.dylib 0x90024227 _pthread_body + 84
Thread 4 crashed with X86 Thread State (32-bit):
eax: 0x00000000 ebx: 0x927de76e ecx: 0x90abf010 edx: 0x00000005
edi: 0x00000008 esi: 0xb02037d8 ebp: 0xb0203868 esp: 0xb02037a4
ss: 0x0000001f efl: 0x00010206 eip: 0x90a584c7 cs: 0x00000017
ds: 0x0000001f es: 0x0000001f fs: 0x00000000 gs: 0x00000037
-------------------------------------------------------------------------------------
> ./server
20081112 12:57:10 [ 530] <0xa000d000> server...
20081112 12:57:10 [ 530] <0x180ec00> server: +[CServerThread run:]
20081112 12:57:10 [ 530] <0x180ca00> server: +[CServerThread run:]
20081112 12:57:10 [ 530] <0x180ce00> server: +[CServerThread run:]
20081112 12:57:10 [ 530] <0x1814000> server: +[CServerThread run:]
objc: FREED(id): message release sent to freed object=0x317fe0
Trace/BPT trap
<excerpt from crash log>
Thread: 3
Exception: EXC_BREAKPOINT (0x0006)
Code[0]: 0x00000002
Code[1]: 0x00000000
Thread 3 Crashed:
0 libobjc.A.dylib 0x90a60b09 _objc_error + 86
1 libobjc.A.dylib 0x90a60b40 __objc_error + 45
2 libobjc.A.dylib 0x90a5f1a0 _freedHandler + 53
3 com.apple.Foundation 0x927de96f NSPopAutoreleasePool + 530
4 com.apple.Foundation 0x9284baac __NSFireMachPort + 399
5 com.apple.CoreFoundation 0x9083c2fd __CFMachPortPerform + 136
6 com.apple.CoreFoundation 0x9082c5a1 CFRunLoopRunSpecific + 2904
7 com.apple.CoreFoundation 0x9082ba42 CFRunLoopRunInMode + 61
8 com.apple.Foundation 0x9282fd6a -[NSRunLoop runMode:beforeDate:] +
182
9 server 0x00002989 +[CServerThread run:] + 231
10 com.apple.Foundation 0x927fa39c forkThreadForFunction + 123
11 libSystem.B.dylib 0x90024227 _pthread_body + 84
_______________________________________________
Cocoa-dev mailing list (Cocoa-dev@lists.apple.com)
Please do not post admin requests or moderator comments to the list.
Contact the moderators at cocoa-dev-admins(at)lists.apple.com
Help/Unsubscribe/Update your Subscription:
http://lists.apple.com/mailman/options/cocoa-dev/archive%40mail-archive.com
This email sent to [EMAIL PROTECTED]
