Hi, As I said, I think that SQLite should perform at the best, for each system architecture (single process single thread, single process multithread, multi process single thread and multi process multithread). I don't think that one architecture limitations should affect the others. So, I did one step toward proc_exclusive, and changed wal HEAPMEMORY_MODE to use memory vfs (patch pasted below). This does not add functionality yet, but it makes the code clearer. I only implemented the shared memory part of the VFS, but implementing all other functions is not complicated. In general I think that SQLite should have a in memory VFS, which is OS independent.
I am going to implement proc_exclusive now, and would love to get any directions/tips from sqlite developers (if you think its not a good idea, please let me know, or better - suggest another way :-)) Yoni. Index: main.mk =================================================================== --- main.mk +++ main.mk @@ -56,11 +56,11 @@ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ - memjournal.o \ + memjournal.o memvfs.o \ mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \ notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \ pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o select.o status.o \ table.o tokenize.o trigger.o \ @@ -107,10 +107,11 @@ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ $(TOP)/src/mem3.c \ $(TOP)/src/mem5.c \ $(TOP)/src/memjournal.c \ + $(TOP)/src/memvfs.c \ $(TOP)/src/mutex.c \ $(TOP)/src/mutex.h \ $(TOP)/src/mutex_noop.c \ $(TOP)/src/mutex_os2.c \ $(TOP)/src/mutex_unix.c \ Index: src/main.c =================================================================== --- src/main.c +++ src/main.c @@ -191,10 +191,13 @@ rc = sqlite3PcacheInitialize(); } if( rc==SQLITE_OK ){ sqlite3GlobalConfig.isPCacheInit = 1; rc = sqlite3OsInit(); + } + if( rc==SQLITE_OK ){ + rc = memvfs_init(); } if( rc==SQLITE_OK ){ sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); sqlite3GlobalConfig.isInit = 1; ADDED src/memvfs.c Index: src/memvfs.c =================================================================== --- src/memvfs.c +++ src/memvfs.c @@ -1,0 +1,161 @@ +/* +** 2010 Dec 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +******************************************************************************/ + +#include "sqliteInt.h" + +typedef struct { + void volatile *mem; + int szMem; +} memNode; + +typedef struct memFile memFile; +struct memFile { + const sqlite3_io_methods *pMethod; /*** Must be first ***/ + sqlite3_vfs *pVfs; /* The VFS used to open this file */ + memNode *pMemNode; + int szMemNode; +}; + +static int memShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + int rc = SQLITE_OK, newSize = (iRegion+1)*sizeof(memNode); + memFile *pFile = (memFile*)fd; + *pp = 0; + if( pFile->szMemNode < newSize ){ + if( !isWrite ){ + goto Exit; + } + pFile->pMemNode = sqlite3Realloc(pFile->pMemNode, newSize); + if( !pFile->pMemNode ){ + rc = SQLITE_NOMEM; + goto Exit; + } + memset(pFile->pMemNode + pFile->szMemNode, 0, newSize - pFile->szMemNode); + pFile->szMemNode = newSize; + } + if( !pFile->pMemNode[iRegion].mem ){ + pFile->pMemNode[iRegion].mem = sqlite3MallocZero(szRegion); + if( !pFile->pMemNode[iRegion].mem ){ + rc = SQLITE_NOMEM; + goto Exit; + } + pFile->pMemNode[iRegion].szMem = szRegion; + } + assert( pFile->pMemNode[iRegion].szMem == szRegion ); + *pp = pFile->pMemNode[iRegion].mem; +Exit: + return rc; +} + +static int memShmUnmap( + sqlite3_file *fd, /* Database holding shared memory */ + int deleteFlag /* Delete after closing if true */ +){ + int i; + memFile *pFile = (memFile*)fd; + for(i=0; i < pFile->szMemNode / sizeof(memNode); i++ ){ + sqlite3_free((void *)pFile->pMemNode[i].mem); + } + sqlite3_free(pFile->pMemNode); + pFile->szMemNode = 0; + return SQLITE_OK; +} + +static int memShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + UNUSED_PARAMETER(fd); + return SQLITE_OK; +} + +static void memShmBarrier( + sqlite3_file *fd /* Database holding the shared memory */ +){ + UNUSED_PARAMETER(fd); +} + +static const sqlite3_io_methods memIoMethod = { + 2, /* iVersion */ + 0, /* xClose */ + 0, /* xRead */ + 0, /* xWrite */ + 0, /* xTruncate */ + 0, /* xSync */ + 0, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + memShmMap, /* xShmMap */ + memShmLock, /* xShmLock */ + memShmBarrier, /* xShmBarrier */ + memShmUnmap /* xShmUnmap */ +}; + +static int memOpen( + sqlite3_vfs *pVfs, /* Not used */ + const char *zName, /* Not used */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + memFile *pFile = (memFile*)id; + pFile->pVfs = pVfs; + pFile->pMethod = &memIoMethod; + return SQLITE_OK; +} + +static int memDelete( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on win32 */ +){ + return SQLITE_OK; +} + +int memvfs_init(void){ + static sqlite3_vfs memVfs = { + 2, /* iVersion */ + sizeof(memFile), /* szOsFile */ + 0, /* mxPathname */ + 0, /* pNext */ + "memvfs", /* zName */ + 0, /* pAppData */ + memOpen, /* xOpen */ + memDelete, /* xDelete */ + 0, /* xAccess */ + 0, /* xFullPathname */ + 0, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ + 0, /* xRandomness */ + 0, /* xSleep */ + 0, /* xCurrentTime */ + 0, /* xGetLastError */ + 0, /* xCurrentTimeInt64 */ + }; + + sqlite3_vfs_register(&memVfs, 0); + return SQLITE_OK; +} + Index: src/wal.c =================================================================== --- src/wal.c +++ src/wal.c @@ -409,10 +409,11 @@ */ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ + sqlite3_file *pShmFd; /* File handle for shared memory */ u32 iCallback; /* Value to pass to log callback (or 0) */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ u32 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ @@ -519,18 +520,13 @@ pWal->nWiData = iPage+1; } /* Request a pointer to the required page from the VFS */ if( pWal->apWiData[iPage]==0 ){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ - pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; - }else{ - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + rc = sqlite3OsShmMap(pWal->pShmFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); - } } *ppPage = pWal->apWiData[iPage]; assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); return rc; @@ -611,11 +607,11 @@ aOut[1] = s2; } static void walShmBarrier(Wal *pWal){ if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ - sqlite3OsShmBarrier(pWal->pDbFd); + sqlite3OsShmBarrier(pWal->pShmFd); } } /* ** Write the header information in pWal->hdr into the wal-index. @@ -755,36 +751,36 @@ ** In locking_mode=EXCLUSIVE, all of these routines become no-ops. */ static int walLockShared(Wal *pWal, int lockIdx){ int rc; if( pWal->exclusiveMode ) return SQLITE_OK; - rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + rc = sqlite3OsShmLock(pWal->pShmFd, lockIdx, 1, SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, walLockName(lockIdx), rc ? "failed" : "ok")); VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) return rc; } static void walUnlockShared(Wal *pWal, int lockIdx){ if( pWal->exclusiveMode ) return; - (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + (void)sqlite3OsShmLock(pWal->pShmFd, lockIdx, 1, SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); } static int walLockExclusive(Wal *pWal, int lockIdx, int n){ int rc; if( pWal->exclusiveMode ) return SQLITE_OK; - rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + rc = sqlite3OsShmLock(pWal->pShmFd, lockIdx, n, SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, walLockName(lockIdx), n, rc ? "failed" : "ok")); VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) return rc; } static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ if( pWal->exclusiveMode ) return; - (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + (void)sqlite3OsShmLock(pWal->pShmFd, lockIdx, n, SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal, walLockName(lockIdx), n)); } @@ -1201,18 +1197,12 @@ /* ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ - int i; - for(i=0; i<pWal->nWiData; i++){ - sqlite3_free((void *)pWal->apWiData[i]); - pWal->apWiData[i] = 0; - } - }else{ - sqlite3OsShmUnmap(pWal->pDbFd, isDelete); + if ( pWal->pShmFd ){ + sqlite3OsShmUnmap(pWal->pShmFd, isDelete); } } /* ** Open a connection to the WAL file zWalName. The database file must @@ -1234,13 +1224,14 @@ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Wal *pRet; /* Object to allocate and return */ int flags; /* Flags passed to OsOpen() */ + sqlite3_vfs *pMemVfs; assert( zWalName && zWalName[0] ); assert( pDbFd ); /* In the amalgamation, the os_unix.c and os_win.c source files come before @@ -1266,16 +1257,31 @@ pRet->pWalFd = (sqlite3_file *)&pRet[1]; pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->zWalName = zWalName; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); + if( pRet->exclusiveMode && (pMemVfs = sqlite3_vfs_find("memvfs"))){ + pRet->pShmFd = sqlite3MallocZero(pMemVfs->szOsFile); + if( !pRet->pShmFd ){ + return SQLITE_NOMEM; + } + rc = sqlite3OsOpen(pMemVfs, NULL, pRet->pShmFd, 0, NULL); + if ( rc!=SQLITE_OK ){ + sqlite3_free( pRet->pShmFd ); + pRet->pShmFd = NULL; + } + }else{ + pRet->pShmFd = pDbFd; + } /* Open file handle on the write-ahead log file. */ - flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); - rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); - if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ - pRet->readOnly = 1; + if ( rc==SQLITE_OK ){ + flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ + pRet->readOnly = 1; + } } if( rc!=SQLITE_OK ){ walIndexClose(pRet, 0); sqlite3OsClose(pRet->pWalFd); _______________________________________________ sqlite-users mailing list sqlite-users@sqlite.org http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-users