Hi,
As I said, I think that SQLite should perform at the best, for each
system architecture (single process single thread, single process
multithread, multi process single thread and multi process multithread).
I don't think that one architecture limitations should affect the others.
So, I did one step toward proc_exclusive, and changed wal
HEAPMEMORY_MODE to use memory vfs (patch pasted below). This does not
add functionality yet, but it makes the code clearer.
I only implemented the shared memory part of the VFS, but implementing
all other functions is not complicated.
In general I think that SQLite should have a in memory VFS, which is OS
independent.
I am going to implement proc_exclusive now, and would love to get any
directions/tips from sqlite developers (if you think its not a good
idea, please let me know, or better - suggest another way :-))
Yoni.
Index: main.mk
===================================================================
--- main.mk
+++ main.mk
@@ -56,11 +56,11 @@
fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \
func.o global.o hash.o \
icu.o insert.o journal.o legacy.o loadext.o \
main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
- memjournal.o \
+ memjournal.o memvfs.o \
mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \
notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \
pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \
random.o resolve.o rowset.o rtree.o select.o status.o \
table.o tokenize.o trigger.o \
@@ -107,10 +107,11 @@
$(TOP)/src/mem1.c \
$(TOP)/src/mem2.c \
$(TOP)/src/mem3.c \
$(TOP)/src/mem5.c \
$(TOP)/src/memjournal.c \
+ $(TOP)/src/memvfs.c \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex.h \
$(TOP)/src/mutex_noop.c \
$(TOP)/src/mutex_os2.c \
$(TOP)/src/mutex_unix.c \
Index: src/main.c
===================================================================
--- src/main.c
+++ src/main.c
@@ -191,10 +191,13 @@
rc = sqlite3PcacheInitialize();
}
if( rc==SQLITE_OK ){
sqlite3GlobalConfig.isPCacheInit = 1;
rc = sqlite3OsInit();
+ }
+ if( rc==SQLITE_OK ){
+ rc = memvfs_init();
}
if( rc==SQLITE_OK ){
sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage,
sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage);
sqlite3GlobalConfig.isInit = 1;
ADDED src/memvfs.c
Index: src/memvfs.c
===================================================================
--- src/memvfs.c
+++ src/memvfs.c
@@ -1,0 +1,161 @@
+/*
+** 2010 Dec 12
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************/
+
+#include "sqliteInt.h"
+
+typedef struct {
+ void volatile *mem;
+ int szMem;
+} memNode;
+
+typedef struct memFile memFile;
+struct memFile {
+ const sqlite3_io_methods *pMethod; /*** Must be first ***/
+ sqlite3_vfs *pVfs; /* The VFS used to open this file */
+ memNode *pMemNode;
+ int szMemNode;
+};
+
+static int memShmMap(
+ sqlite3_file *fd, /* Handle open on database file */
+ int iRegion, /* Region to retrieve */
+ int szRegion, /* Size of regions */
+ int isWrite, /* True to extend file if necessary */
+ void volatile **pp /* OUT: Mapped memory */
+){
+ int rc = SQLITE_OK, newSize = (iRegion+1)*sizeof(memNode);
+ memFile *pFile = (memFile*)fd;
+ *pp = 0;
+ if( pFile->szMemNode < newSize ){
+ if( !isWrite ){
+ goto Exit;
+ }
+ pFile->pMemNode = sqlite3Realloc(pFile->pMemNode, newSize);
+ if( !pFile->pMemNode ){
+ rc = SQLITE_NOMEM;
+ goto Exit;
+ }
+ memset(pFile->pMemNode + pFile->szMemNode, 0, newSize -
pFile->szMemNode);
+ pFile->szMemNode = newSize;
+ }
+ if( !pFile->pMemNode[iRegion].mem ){
+ pFile->pMemNode[iRegion].mem = sqlite3MallocZero(szRegion);
+ if( !pFile->pMemNode[iRegion].mem ){
+ rc = SQLITE_NOMEM;
+ goto Exit;
+ }
+ pFile->pMemNode[iRegion].szMem = szRegion;
+ }
+ assert( pFile->pMemNode[iRegion].szMem == szRegion );
+ *pp = pFile->pMemNode[iRegion].mem;
+Exit:
+ return rc;
+}
+
+static int memShmUnmap(
+ sqlite3_file *fd, /* Database holding shared memory */
+ int deleteFlag /* Delete after closing if true */
+){
+ int i;
+ memFile *pFile = (memFile*)fd;
+ for(i=0; i < pFile->szMemNode / sizeof(memNode); i++ ){
+ sqlite3_free((void *)pFile->pMemNode[i].mem);
+ }
+ sqlite3_free(pFile->pMemNode);
+ pFile->szMemNode = 0;
+ return SQLITE_OK;
+}
+
+static int memShmLock(
+ sqlite3_file *fd, /* Database file holding the shared memory */
+ int ofst, /* First lock to acquire or release */
+ int n, /* Number of locks to acquire or release */
+ int flags /* What to do with the lock */
+){
+ UNUSED_PARAMETER(fd);
+ return SQLITE_OK;
+}
+
+static void memShmBarrier(
+ sqlite3_file *fd /* Database holding the shared memory */
+){
+ UNUSED_PARAMETER(fd);
+}
+
+static const sqlite3_io_methods memIoMethod = {
+ 2, /* iVersion */
+ 0, /* xClose */
+ 0, /* xRead */
+ 0, /* xWrite */
+ 0, /* xTruncate */
+ 0, /* xSync */
+ 0, /* xFileSize */
+ 0, /* xLock */
+ 0, /* xUnlock */
+ 0, /* xCheckReservedLock */
+ 0, /* xFileControl */
+ 0, /* xSectorSize */
+ 0, /* xDeviceCharacteristics */
+ memShmMap, /* xShmMap */
+ memShmLock, /* xShmLock */
+ memShmBarrier, /* xShmBarrier */
+ memShmUnmap /* xShmUnmap */
+};
+
+static int memOpen(
+ sqlite3_vfs *pVfs, /* Not used */
+ const char *zName, /* Not used */
+ sqlite3_file *id, /* Write the SQLite file handle here */
+ int flags, /* Open mode flags */
+ int *pOutFlags /* Status return flags */
+){
+ memFile *pFile = (memFile*)id;
+ pFile->pVfs = pVfs;
+ pFile->pMethod = &memIoMethod;
+ return SQLITE_OK;
+}
+
+static int memDelete(
+ sqlite3_vfs *pVfs, /* Not used on win32 */
+ const char *zFilename, /* Name of file to delete */
+ int syncDir /* Not used on win32 */
+){
+ return SQLITE_OK;
+}
+
+int memvfs_init(void){
+ static sqlite3_vfs memVfs = {
+ 2, /* iVersion */
+ sizeof(memFile), /* szOsFile */
+ 0, /* mxPathname */
+ 0, /* pNext */
+ "memvfs", /* zName */
+ 0, /* pAppData */
+ memOpen, /* xOpen */
+ memDelete, /* xDelete */
+ 0, /* xAccess */
+ 0, /* xFullPathname */
+ 0, /* xDlOpen */
+ 0, /* xDlError */
+ 0, /* xDlSym */
+ 0, /* xDlClose */
+ 0, /* xRandomness */
+ 0, /* xSleep */
+ 0, /* xCurrentTime */
+ 0, /* xGetLastError */
+ 0, /* xCurrentTimeInt64 */
+ };
+
+ sqlite3_vfs_register(&memVfs, 0);
+ return SQLITE_OK;
+}
+
Index: src/wal.c
===================================================================
--- src/wal.c
+++ src/wal.c
@@ -409,10 +409,11 @@
*/
struct Wal {
sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */
sqlite3_file *pDbFd; /* File handle for the database file */
sqlite3_file *pWalFd; /* File handle for WAL file */
+ sqlite3_file *pShmFd; /* File handle for shared memory */
u32 iCallback; /* Value to pass to log callback (or 0) */
int nWiData; /* Size of array apWiData */
volatile u32 **apWiData; /* Pointer to wal-index content in memory */
u32 szPage; /* Database page size */
i16 readLock; /* Which read lock is being held. -1 for
none */
@@ -519,18 +520,13 @@
pWal->nWiData = iPage+1;
}
/* Request a pointer to the required page from the VFS */
if( pWal->apWiData[iPage]==0 ){
- if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
- pWal->apWiData[iPage] = (u32 volatile
*)sqlite3MallocZero(WALINDEX_PGSZ);
- if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM;
- }else{
- rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ,
+ rc = sqlite3OsShmMap(pWal->pShmFd, iPage, WALINDEX_PGSZ,
pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
);
- }
}
*ppPage = pWal->apWiData[iPage];
assert( iPage==0 || *ppPage || rc!=SQLITE_OK );
return rc;
@@ -611,11 +607,11 @@
aOut[1] = s2;
}
static void walShmBarrier(Wal *pWal){
if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){
- sqlite3OsShmBarrier(pWal->pDbFd);
+ sqlite3OsShmBarrier(pWal->pShmFd);
}
}
/*
** Write the header information in pWal->hdr into the wal-index.
@@ -755,36 +751,36 @@
** In locking_mode=EXCLUSIVE, all of these routines become no-ops.
*/
static int walLockShared(Wal *pWal, int lockIdx){
int rc;
if( pWal->exclusiveMode ) return SQLITE_OK;
- rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1,
+ rc = sqlite3OsShmLock(pWal->pShmFd, lockIdx, 1,
SQLITE_SHM_LOCK | SQLITE_SHM_SHARED);
WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal,
walLockName(lockIdx), rc ? "failed" : "ok"));
VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); )
return rc;
}
static void walUnlockShared(Wal *pWal, int lockIdx){
if( pWal->exclusiveMode ) return;
- (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1,
+ (void)sqlite3OsShmLock(pWal->pShmFd, lockIdx, 1,
SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED);
WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx)));
}
static int walLockExclusive(Wal *pWal, int lockIdx, int n){
int rc;
if( pWal->exclusiveMode ) return SQLITE_OK;
- rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n,
+ rc = sqlite3OsShmLock(pWal->pShmFd, lockIdx, n,
SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE);
WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal,
walLockName(lockIdx), n, rc ? "failed" : "ok"));
VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); )
return rc;
}
static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){
if( pWal->exclusiveMode ) return;
- (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n,
+ (void)sqlite3OsShmLock(pWal->pShmFd, lockIdx, n,
SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE);
WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal,
walLockName(lockIdx), n));
}
@@ -1201,18 +1197,12 @@
/*
** Close an open wal-index.
*/
static void walIndexClose(Wal *pWal, int isDelete){
- if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
- int i;
- for(i=0; i<pWal->nWiData; i++){
- sqlite3_free((void *)pWal->apWiData[i]);
- pWal->apWiData[i] = 0;
- }
- }else{
- sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
+ if ( pWal->pShmFd ){
+ sqlite3OsShmUnmap(pWal->pShmFd, isDelete);
}
}
/*
** Open a connection to the WAL file zWalName. The database file must
@@ -1234,13 +1224,14 @@
sqlite3_file *pDbFd, /* The open database file */
const char *zWalName, /* Name of the WAL file */
int bNoShm, /* True to run in heap-memory mode */
Wal **ppWal /* OUT: Allocated Wal handle */
){
- int rc; /* Return Code */
+ int rc = SQLITE_OK; /* Return Code */
Wal *pRet; /* Object to allocate and return */
int flags; /* Flags passed to OsOpen() */
+ sqlite3_vfs *pMemVfs;
assert( zWalName && zWalName[0] );
assert( pDbFd );
/* In the amalgamation, the os_unix.c and os_win.c source files come
before
@@ -1266,16 +1257,31 @@
pRet->pWalFd = (sqlite3_file *)&pRet[1];
pRet->pDbFd = pDbFd;
pRet->readLock = -1;
pRet->zWalName = zWalName;
pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
+ if( pRet->exclusiveMode && (pMemVfs = sqlite3_vfs_find("memvfs"))){
+ pRet->pShmFd = sqlite3MallocZero(pMemVfs->szOsFile);
+ if( !pRet->pShmFd ){
+ return SQLITE_NOMEM;
+ }
+ rc = sqlite3OsOpen(pMemVfs, NULL, pRet->pShmFd, 0, NULL);
+ if ( rc!=SQLITE_OK ){
+ sqlite3_free( pRet->pShmFd );
+ pRet->pShmFd = NULL;
+ }
+ }else{
+ pRet->pShmFd = pDbFd;
+ }
/* Open file handle on the write-ahead log file. */
- flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
- rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags);
- if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){
- pRet->readOnly = 1;
+ if ( rc==SQLITE_OK ){
+ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
+ rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags);
+ if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){
+ pRet->readOnly = 1;
+ }
}
if( rc!=SQLITE_OK ){
walIndexClose(pRet, 0);
sqlite3OsClose(pRet->pWalFd);
_______________________________________________
sqlite-users mailing list
[email protected]
http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-users
