From ac167e4e81983218e7fdefc2790ab65615de65a1 Mon Sep 17 00:00:00 2001 From: "ideriha.takeshi" Date: Wed, 12 Sep 2018 14:55:06 +0900 Subject: [PATCH] PoC: Allocate catcache on the shared memmory Just allocated catalog cache header and pool for CatCache, CatCTup and CatCList. --- doc/src/sgml/config.sgml | 16 ++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/lmgr/lwlocknames.txt | 2 + src/backend/utils/cache/catcache.c | 244 +++++++++++++++++++++++++- src/backend/utils/cache/syscache.c | 64 +++++-- src/backend/utils/misc/guc.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/include/storage/lwlock.h | 1 + src/include/utils/catcache.h | 21 +++ 9 files changed, 349 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f11b8f7..ed1200f 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1617,6 +1617,22 @@ include_dir 'conf.d' + + shared_catcache_mem (integer) + + shared_catcache_mem configuration parameter + + + + + Specifies the amount of shared catalog cache. It defaults to 0, + which indicates the catalog cache is not shared but created per + backend. When the number of backends is enourmous, the setting + alleviates the amount of memory usage. + + + + max_stack_depth (integer) diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 0c86a58..f051e8f 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -45,6 +45,7 @@ #include "storage/sinvaladt.h" #include "storage/spin.h" #include "utils/backend_random.h" +#include "utils/catcache.h" #include "utils/snapmgr.h" @@ -150,6 +151,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port) size = add_size(size, SyncScanShmemSize()); size = add_size(size, AsyncShmemSize()); size = add_size(size, BackendRandomShmemSize()); + size = add_size(size, CatCacheShmemSize()); #ifdef EXEC_BACKEND size = add_size(size, ShmemBackendArraySize()); #endif @@ -270,6 +272,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port) SyncScanShmemInit(); AsyncShmemInit(); BackendRandomShmemInit(); + CatCacheShmemInit(); #ifdef EXEC_BACKEND diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index e6025ec..7512464 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -50,3 +50,5 @@ OldSnapshotTimeMapLock 42 BackendRandomLock 43 LogicalRepWorkerLock 44 CLogTruncationLock 45 +CatCacheHdrLock 46 +CatCacheLock 47 diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index 5ddbf6e..0e71072 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -24,6 +24,7 @@ #include "access/xact.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" +#include "lib/dshash.h" #include "miscadmin.h" #ifdef CATCACHE_STATS #include "storage/ipc.h" /* for on_proc_exit */ @@ -40,6 +41,14 @@ #include "utils/syscache.h" #include "utils/tqual.h" +/* GUC parameter: A value of 0 means catalog cache is built per backend */ +int shared_catcache_mem; + +/* Pointer to Postmaster-initilized shared memory */ +typedef void *ShmemCatCachePointer; + +/* Unit of shared_catcache_mem is megabyte */ +#define SHM_CATCACHE_SIZE ((size_t)(1024 * 1024 * shared_catcache_mem)) /* #define CACHEDEBUG */ /* turns DEBUG elogs on */ @@ -73,6 +82,21 @@ /* Cache management header --- pointer is NULL until created */ static CatCacheHeader *CacheHdr = NULL; +ShmCatCacheHeader *ShmCacheHdr = NULL; + +/* + * stuffs for shared catcaches + */ + +static char *ShmCatCPtr = NULL; + +static dsa_area *catcdsa_area = NULL; + +static CatCache *InitSharedCatCache(int id, Oid reloid, + Oid indexoid, + int nkeys, + const int *key, + int nbuckets); static inline HeapTuple SearchCatCacheInternal(CatCache *cache, int nkeys, @@ -416,8 +440,14 @@ CatCachePrintStats(int code, Datum arg) long cc_invals = 0; long cc_lsearches = 0; long cc_lhits = 0; + CatCacheHeader *CHdr = CacheHdr; + + if (shared_catcache_mem != 0) { + LWLockAcquire(CatCacheHdrLock, LW_SHARED); + CHdr = (CatCacheHeader *)ShmCacheHdr; + } - slist_foreach(iter, &CacheHdr->ch_caches) + slist_foreach(iter, &CHdr->ch_caches) { CatCache *cache = slist_container(CatCache, cc_next, iter.cur); @@ -446,7 +476,7 @@ CatCachePrintStats(int code, Datum arg) cc_lhits += cache->cc_lhits; } elog(DEBUG2, "catcache totals: %d tup, %ld srch, %ld+%ld=%ld hits, %ld+%ld=%ld loads, %ld invals, %ld lsrch, %ld lhits", - CacheHdr->ch_ntup, + CHdr->ch_ntup, cc_searches, cc_hits, cc_neg_hits, @@ -457,6 +487,9 @@ CatCachePrintStats(int code, Datum arg) cc_invals, cc_lsearches, cc_lhits); + + if (shared_catcache_mem != 0) + LWLockRelease(CatCacheHdrLock); } #endif /* CATCACHE_STATS */ @@ -759,7 +792,8 @@ CatalogCacheFlushCatalog(Oid catId) /* * InitCatCache * - * This allocates and initializes a cache for a system catalog relation. + * This allocates and initializes a cache for a system catalog relation + * either on CacheMemoryContext or on DSA. * Actually, the cache is only partially initialized to avoid opening the * relation. The relation will be opened and the rest of the cache * structure initialized on the first access. @@ -788,6 +822,11 @@ InitCatCache(int id, size_t sz; int i; + + if (shared_catcache_mem != 0) + return InitSharedCatCache(id, reloid, indexoid, nkeys, key, nbuckets); + + /* * nbuckets is the initial number of hash buckets to use in this catcache. * It will be enlarged later if it becomes too full. @@ -811,7 +850,8 @@ InitCatCache(int id, oldcxt = MemoryContextSwitchTo(CacheMemoryContext); /* - * if first time through, initialize the cache group header + * if catcache memory is not shared and first time through, + * initialize the cache group header. */ if (CacheHdr == NULL) { @@ -869,6 +909,9 @@ InitCatCache(int id, return cp; } + + + /* * Enlarge a catcache, doubling the number of buckets. */ @@ -1081,7 +1124,6 @@ InitCatCachePhase2(CatCache *cache, bool touch_index) } } - /* * IndexScanOK * @@ -1910,7 +1952,11 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments, dlist_push_head(&cache->cc_bucket[hashIndex], &ct->cache_elem); cache->cc_ntup++; - CacheHdr->ch_ntup++; + + if (shared_catcache_mem == 0) + CacheHdr->ch_ntup++; + else + ShmCacheHdr->ch_ntup++; /* * If the hash table has become too full, enlarge the buckets array. Quite @@ -2118,3 +2164,189 @@ PrintCatCacheListLeakWarning(CatCList *list) list->my_cache->cc_relname, list->my_cache->id, list, list->refcount); } + +/* + * Functions used if shared_catcache_mem is nonzero value + */ + + +/* + * CatCacheShmemInit + * + * This is called during shared-memory initialization + * Fixed-size data structure for catalog cache header is initilized. + * The other shared area is supposed to be used as DSA area + * for CatCache, CatCList, CatCTup. + * + */ +void +CatCacheShmemInit(void) +{ + bool foundhdr; + bool foundcat; + + /* do nothing if catalog cache is not shared */ + if (shared_catcache_mem == 0) + return; + + /*initialize shared version CacheHdr */ + ShmCacheHdr = (ShmCatCacheHeader *) + ShmemInitStruct("Catlog Cache Header", sizeof(ShmCatCacheHeader), &foundhdr); + + ShmCatCPtr = + ShmemInitStruct("Catalog Cache",SHM_CATCACHE_SIZE , &foundcat); + + + if (!IsUnderPostmaster) + { + Assert(!foundhdr); + Assert(!foundcat); + + /* First time through ... */ + slist_init(&ShmCacheHdr->ch_caches); + ShmCacheHdr->ch_ntup = 0; + ShmCacheHdr->ch_initilized = false; + + /* + * XXX: the lwlock tranche for dsa area might not be necessary + * because we alredy register tranche for static shared memory + */ + LWLockRegisterTranche(LWTRANCHE_CATCACHE_DSA, "catcache_dsa"); + +#ifdef CATCACHE_STATS + /* set up to dump stats at backend exit */ + on_proc_exit(CatCachePrintStats, 0); +#endif + } + else + { + Assert(foundhdr); + Assert(foundcat); + } +} + +/* + * CatCacheShmemsize + * + * Compute the size of shared memory for the catalog cache header + * and fixed-size area for CatCache, CatCList, CatCTup + */ +Size +CatCacheShmemSize(void) +{ + Size size = 0; + + /* size of cache header */ + size = add_size(size, sizeof(ShmCatCacheHeader)); + + /* size of catalog cache area */ + size = add_size(size, SHM_CATCACHE_SIZE); + return size; +} + + +/* + * InitSharedCatCache + * + * the lock handling for ShmCacheHdr is done by InitCatalogCache() + */ +static CatCache * +InitSharedCatCache(int id, + Oid reloid, + Oid indexoid, + int nkeys, + const int *key, + int nbuckets) +{ + CatCache *cp; + MemoryContext oldcxt; + size_t sz; + int i; + dsa_pointer dsa_ptr; + + Assert(shared_catcache_mem != 0); + + /* + * first switch to the cache context so our allocations do not vanish at + * the end of a transaction + */ + if (!CacheMemoryContext) + CreateCacheMemoryContext(); + + oldcxt = MemoryContextSwitchTo(CacheMemoryContext); + + /* if first time through */ + if (catcdsa_area == NULL) { + + /* + * Create a new DSA area in Postmaster-initilized shared memory space, + * whose pointer is common to the all of backends. + */ + catcdsa_area = dsa_create_in_place(ShmCatCPtr, SHM_CATCACHE_SIZE, + LWTRANCHE_CATCACHE_DSA, NULL); + CACHE1_elog(DEBUG2,"DSA area for CatCache is created"); + + dsa_set_size_limit(catcdsa_area, SHM_CATCACHE_SIZE); + CACHE2_elog(DEBUG2, "Limit size of DSA area is set to %lu", + SHM_CATCACHE_SIZE); + + /* this area should remian even if no session attaches it */ + dsa_pin(catcdsa_area); + } + /* + * Allocate a new cache structure, aligning to a cacheline boundary + * + */ + sz = sizeof(CatCache) + PG_CACHE_LINE_SIZE; + + dsa_ptr = dsa_allocate0(catcdsa_area, sz); + + cp = (CatCache *) + CACHELINEALIGN(dsa_get_address(catcdsa_area, dsa_ptr)); + + /* + * TODO: + * in the shared catcache hash table is built based on dshash + * cc_bucket become a pointer to dshash table and + */ + cp->cc_bucket = palloc0(nbuckets * sizeof(dlist_head)); + + /* + * initialize the cache's relation information for the relation + * corresponding to this cache, and initialize some of the new cache's + * other internal fields. But don't open the relation yet. + */ + cp->id = id; + cp->cc_relname = "(not known yet)"; + cp->cc_reloid = reloid; + cp->cc_indexoid = indexoid; + cp->cc_relisshared = false; /* temporary */ + cp->cc_tupdesc = (TupleDesc) NULL; + cp->cc_ntup = 0; + cp->cc_nbuckets = nbuckets; + cp->cc_nkeys = nkeys; + for (i = 0; i < nkeys; ++i) + cp->cc_keyno[i] = key[i]; + + /* + * new cache is initialized as far as we can go for now. print some + * debugging information, if appropriate. + */ + InitCatCache_DEBUG2; + + /* + * add completed cache to top of group header's list + */ + slist_push_head(&ShmCacheHdr->ch_caches, &cp->cc_next); + +#ifdef CACHEDEBUG + dsa_dump(catcdsa_area); +#endif + + /* + * back to the old context before we return... + */ + MemoryContextSwitchTo(oldcxt); + + return cp; +} diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index 2b38178..a897a75 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -73,6 +73,8 @@ #include "catalog/pg_ts_template.h" #include "catalog/pg_type.h" #include "catalog/pg_user_mapping.h" +#include "storage/lmgr.h" +#include "utils/dsa.h" #include "utils/rel.h" #include "utils/catcache.h" #include "utils/syscache.h" @@ -971,6 +973,8 @@ static const struct cachedesc cacheinfo[] = { } }; +int shared_catcache_mem; + static CatCache *SysCache[SysCacheSize]; static bool CacheInitialized = false; @@ -983,6 +987,7 @@ static int SysCacheRelationOidSize; static Oid SysCacheSupportingRelOid[SysCacheSize * 2]; static int SysCacheSupportingRelOidSize; +static void AttachCatalogCache(void); static int oid_compare(const void *a, const void *b); @@ -1005,20 +1010,33 @@ InitCatalogCache(void) "SysCacheSize does not match syscache.c's array"); Assert(!CacheInitialized); - + SysCacheRelationOidSize = SysCacheSupportingRelOidSize = 0; + if (shared_catcache_mem != 0) + LWLockAcquire(CatCacheHdrLock, LW_EXCLUSIVE); + + /* if shared CatCache is already created, just set pointer to SysCache */ + if (shared_catcache_mem != 0 && ShmCacheHdr->ch_initilized == true) + AttachCatalogCache(); + else + { + for (cacheId = 0; cacheId < SysCacheSize; cacheId++) + { + SysCache[cacheId] = InitCatCache(cacheId, + cacheinfo[cacheId].reloid, + cacheinfo[cacheId].indoid, + cacheinfo[cacheId].nkeys, + cacheinfo[cacheId].key, + cacheinfo[cacheId].nbuckets); + if (!PointerIsValid(SysCache[cacheId])) + elog(ERROR, "could not initialize cache %u (%d)", + cacheinfo[cacheId].reloid, cacheId); + } + } + for (cacheId = 0; cacheId < SysCacheSize; cacheId++) { - SysCache[cacheId] = InitCatCache(cacheId, - cacheinfo[cacheId].reloid, - cacheinfo[cacheId].indoid, - cacheinfo[cacheId].nkeys, - cacheinfo[cacheId].key, - cacheinfo[cacheId].nbuckets); - if (!PointerIsValid(SysCache[cacheId])) - elog(ERROR, "could not initialize cache %u (%d)", - cacheinfo[cacheId].reloid, cacheId); /* Accumulate data for OID lists, too */ SysCacheRelationOid[SysCacheRelationOidSize++] = cacheinfo[cacheId].reloid; @@ -1053,6 +1071,12 @@ InitCatalogCache(void) SysCacheSupportingRelOidSize = j + 1; CacheInitialized = true; + + /* if shared_catcache_mem == 0, ShmCacheHdr is not created */ + if (shared_catcache_mem != 0 && ShmCacheHdr->ch_initilized == false) + ShmCacheHdr->ch_initilized = true; + if (shared_catcache_mem != 0) + LWLockRelease(CatCacheHdrLock); } /* @@ -1529,6 +1553,26 @@ RelationSupportsSysCache(Oid relid) return false; } +/* + * attach shared CatCache + */ +static void +AttachCatalogCache(void) +{ + slist_iter iter; + int cacheId = SysCacheSize - 1 ; + + Assert(ShmCacheHdr->ch_initilized == true); + + /* list of CatCache pointer is in descending order of cacheId */ + slist_foreach(iter, &ShmCacheHdr->ch_caches) + { + SysCache[cacheId] = slist_container(CatCache, cc_next, iter.cur); + if (!PointerIsValid(SysCache[cacheId--])) + elog(ERROR, "could not attach cache %u (%d)", + cacheinfo[cacheId + 1].reloid, cacheId); + } +} /* * OID comparator for pg_qsort diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 0bec391..0c7e3c9 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -80,6 +80,7 @@ #include "tsearch/ts_cache.h" #include "utils/builtins.h" #include "utils/bytea.h" +#include "utils/catcache.h" #include "utils/guc_tables.h" #include "utils/float.h" #include "utils/memutils.h" @@ -2131,6 +2132,17 @@ static struct config_int ConfigureNamesInt[] = }, { + {"shared_catcache_mem", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared catlog cache memory."), + gettext_noop("A value of 0 disables this feature and catalog cache is built per backend. "), + GUC_UNIT_MB + }, + &shared_catcache_mem, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + + { {"temp_file_limit", PGC_SUSET, RESOURCES_DISK, gettext_noop("Limits the total size of all temporary files used by each process."), gettext_noop("-1 means no limit."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 4e61bc6..d12b9b1 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -134,6 +134,8 @@ # windows # mmap # (change requires restart) +#shared_catcache_mem = 0MB # zero disables the feature + # (change requires restart) # - Disk - diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index c21bfe2..c470b6e 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -219,6 +219,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_SHARED_TUPLESTORE, LWTRANCHE_TBM, LWTRANCHE_PARALLEL_APPEND, + LWTRANCHE_CATCACHE_DSA, LWTRANCHE_FIRST_USER_DEFINED } BuiltinTrancheIds; diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h index 7b22f9c..45c126b 100644 --- a/src/include/utils/catcache.h +++ b/src/include/utils/catcache.h @@ -186,11 +186,32 @@ typedef struct catcacheheader } CatCacheHeader; +/* + * the 1st and 2nd field is same as CatCacheHeader so that it can be + * casted into CatCachHeader + */ +typedef struct shm_catcacheheader +{ + slist_head ch_caches; /* head of list of CatCache structs */ + int ch_ntup; /* # of tuples in all caches */ + bool ch_initilized; /* CatCache structs are initilized ? */ +}ShmCatCacheHeader; + + /* this extern duplicates utils/memutils.h... */ extern PGDLLIMPORT MemoryContext CacheMemoryContext; +/* GUC parameter: A value of 0 means catalog cache is built per backend */ +extern int shared_catcache_mem; + +extern ShmCatCacheHeader *ShmCacheHdr; + extern void CreateCacheMemoryContext(void); +extern void CatCacheShmemInit(void); + +extern Size CatCacheShmemSize(void); + extern CatCache *InitCatCache(int id, Oid reloid, Oid indexoid, int nkeys, const int *key, int nbuckets); -- 1.8.3.1