The branch, master has been updated
via 76705d10c626a66cc77f3ec294f4f98bef95aeb5 (commit)
via 3d7dfc1197017c34bdb8dbc6e62460f19bd7d141 (commit)
via 8a17cd810fa6cbe7b11139ff0f6f24e7bacd318b (commit)
via ed87594e5fd3251f9cb3beaca06c8eee1dcd4ed2 (commit)
via 3edcd55bf140d09833284ba5a0f04f86b04ef7dc (commit)
via d936d1bd84e130aaff1de64cb1ecbd1f936dd9c4 (commit)
via e5a34b2533720ebb9181c0edebad6774ceeff189 (commit)
via 3e965d017d243f0a99e7838e6c92c37df270486c (commit)
via 565046891f9f7725b5d93eefbc3be5b9c62176fd (commit)
from 8cb44830e0356804e21d9973382e0070f20b15be (commit)
http://gitweb.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 76705d10c626a66cc77f3ec294f4f98bef95aeb5
Author: Volker Lendecke <v...@samba.org>
Date: Tue Jul 14 18:31:28 2009 +0200
Consolidate gencache also every 100 writes in a single process
commit 3d7dfc1197017c34bdb8dbc6e62460f19bd7d141
Author: Volker Lendecke <v...@samba.org>
Date: Tue Jul 14 11:33:04 2009 +0200
Consolidate string and data_blob routines in gencache
commit 8a17cd810fa6cbe7b11139ff0f6f24e7bacd318b
Author: Volker Lendecke <v...@samba.org>
Date: Mon Jul 13 17:04:29 2009 +0200
Make gencache more stable
This provides a compromise between stability and performance: gencache is a
persistent database these days that for performance reasons can not use tdb
transactions for all writes. This patch splits up gencache into gencache.tdb
and gencache_notrans.tdb. gencache_notrans is used with CLEAR_IF_FIRST,
writes
to it don't use transactions. By default every 5 minutes and when a program
exits, all entries from _notrans.tdb are transferred to gencache.tdb in one
transaction.
commit ed87594e5fd3251f9cb3beaca06c8eee1dcd4ed2
Author: Volker Lendecke <v...@samba.org>
Date: Mon Jul 13 17:03:52 2009 +0200
Add tdb_data_cmp
commit 3edcd55bf140d09833284ba5a0f04f86b04ef7dc
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 12:24:56 2009 +0200
Remove gencache_init/shutdown
gencache_get/set/del/iterate call gencache_init() internally anyway. And
we've
been very lazy calling gencache_shutdown, so this seems not really required.
commit d936d1bd84e130aaff1de64cb1ecbd1f936dd9c4
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 12:12:30 2009 +0200
Fix some nonempty blank lines
commit e5a34b2533720ebb9181c0edebad6774ceeff189
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 12:03:35 2009 +0200
Remove gencache_[un]lock_key
commit 3e965d017d243f0a99e7838e6c92c37df270486c
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 11:00:24 2009 +0200
TDB_CONTEXT -> "struct tdb_context"
commit 565046891f9f7725b5d93eefbc3be5b9c62176fd
Author: Volker Lendecke <v...@samba.org>
Date: Fri Jul 10 10:54:33 2009 +0200
Replace ASSERTs in gencache with "return false"
It's a bit strong to panic here I think.
-----------------------------------------------------------------------
Summary of changes:
source3/include/proto.h | 8 +-
source3/include/util_tdb.h | 2 +
source3/lib/gencache.c | 529 +++++++++++++++++++++++++--------------
source3/lib/netapi/netapi.c | 1 -
source3/lib/util_tdb.c | 19 ++
source3/libads/dns.c | 8 -
source3/libsmb/dsgetdcname.c | 33 +--
source3/libsmb/libsmb_context.c | 1 -
source3/libsmb/namecache.c | 34 ---
source3/libsmb/namequery.c | 12 -
source3/libsmb/trustdom_cache.c | 67 ++----
source3/nmbd/nmbd.c | 2 +
source3/smbd/server.c | 1 +
source3/torture/torture.c | 16 --
source3/utils/net.c | 2 +
source3/utils/net_cache.c | 27 ++-
source3/winbindd/winbindd.c | 2 +
17 files changed, 415 insertions(+), 349 deletions(-)
Changeset truncated at 500 lines:
diff --git a/source3/include/proto.h b/source3/include/proto.h
index 0dd1e98..df78155 100644
--- a/source3/include/proto.h
+++ b/source3/include/proto.h
@@ -515,17 +515,15 @@ void pull_file_id_24(char *buf, struct file_id *id);
/* The following definitions come from lib/gencache.c */
-bool gencache_init(void);
-bool gencache_shutdown(void);
bool gencache_set(const char *keystr, const char *value, time_t timeout);
bool gencache_del(const char *keystr);
bool gencache_get(const char *keystr, char **valstr, time_t *timeout);
-bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob, bool
*expired);
+bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob,
+ time_t *timeout);
+bool gencache_stabilize(void);
bool gencache_set_data_blob(const char *keystr, const DATA_BLOB *blob, time_t
timeout);
void gencache_iterate(void (*fn)(const char* key, const char *value, time_t
timeout, void* dptr),
void* data, const char* keystr_pattern);
-int gencache_lock_entry( const char *key );
-void gencache_unlock_entry( const char *key );
/* The following definitions come from lib/interface.c */
diff --git a/source3/include/util_tdb.h b/source3/include/util_tdb.h
index c794364..80b9592 100644
--- a/source3/include/util_tdb.h
+++ b/source3/include/util_tdb.h
@@ -59,4 +59,6 @@ struct tdb_wrap *tdb_wrap_open(TALLOC_CTX *mem_ctx,
NTSTATUS map_nt_error_from_tdb(enum TDB_ERROR err);
+int tdb_data_cmp(TDB_DATA t1, TDB_DATA t2);
+
#endif /* __TDBUTIL_H__ */
diff --git a/source3/lib/gencache.c b/source3/lib/gencache.c
index 7f133f2..ee1f4b7 100644
--- a/source3/lib/gencache.c
+++ b/source3/lib/gencache.c
@@ -26,12 +26,13 @@
#define DBGC_CLASS DBGC_TDB
#define TIMEOUT_LEN 12
-#define CACHE_DATA_FMT "%12u/%s"
+#define CACHE_DATA_FMT "%12u/"
#define READ_CACHE_DATA_FMT_TEMPLATE "%%12u/%%%us"
#define BLOB_TYPE "DATA_BLOB"
#define BLOB_TYPE_LEN 9
-static TDB_CONTEXT *cache;
+static struct tdb_context *cache;
+static struct tdb_context *cache_notrans;
/**
* @file gencache.c
@@ -49,9 +50,10 @@ static TDB_CONTEXT *cache;
* false on failure
**/
-bool gencache_init(void)
+static bool gencache_init(void)
{
char* cache_fname = NULL;
+ int open_flags = O_RDWR|O_CREAT;
/* skip file open if it's already opened */
if (cache) return True;
@@ -60,11 +62,12 @@ bool gencache_init(void)
DEBUG(5, ("Opening cache file at %s\n", cache_fname));
- cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT,
- O_RDWR|O_CREAT, 0644);
+ cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, open_flags, 0644);
if (!cache && (errno == EACCES)) {
- cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, O_RDONLY,
0644);
+ open_flags = O_RDONLY;
+ cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, open_flags,
+ 0644);
if (cache) {
DEBUG(5, ("gencache_init: Opening cache file %s
read-only.\n", cache_fname));
}
@@ -74,65 +77,123 @@ bool gencache_init(void)
DEBUG(5, ("Attempt to open gencache.tdb has failed.\n"));
return False;
}
- return True;
-}
+ cache_fname = lock_path("gencache_notrans.tdb");
-/**
- * Cache shutdown function. Closes opened cache tdb file.
- *
- * @return true on successful closing the cache or
- * false on failure during cache shutdown
- **/
+ DEBUG(5, ("Opening cache file at %s\n", cache_fname));
-bool gencache_shutdown(void)
-{
- int ret;
- /* tdb_close routine returns -1 on error */
- if (!cache) return False;
- DEBUG(5, ("Closing cache file\n"));
- ret = tdb_close(cache);
- cache = NULL;
- return ret != -1;
+ cache_notrans = tdb_open_log(cache_fname, 0, TDB_CLEAR_IF_FIRST,
+ open_flags, 0644);
+ if (cache_notrans == NULL) {
+ DEBUG(5, ("Opening %s failed: %s\n", cache_fname,
+ strerror(errno)));
+ tdb_close(cache);
+ return false;
+ }
+
+ return True;
}
+static TDB_DATA last_stabilize_key(void)
+{
+ TDB_DATA result;
+ result.dptr = (uint8_t *)"@LAST_STABILIZED";
+ result.dsize = 17;
+ return result;
+}
/**
* Set an entry in the cache file. If there's no such
* one, then add it.
*
* @param keystr string that represents a key of this entry
- * @param value text representation value being cached
+ * @param blob DATA_BLOB value being cached
* @param timeout time when the value is expired
*
* @retval true when entry is successfuly stored
* @retval false on failure
**/
-bool gencache_set(const char *keystr, const char *value, time_t timeout)
+bool gencache_set_data_blob(const char *keystr, const DATA_BLOB *blob,
+ time_t timeout)
{
int ret;
TDB_DATA databuf;
- char* valstr = NULL;
+ char* val;
+ time_t last_stabilize;
+ static int writecount;
+
+ if (tdb_data_cmp(string_term_tdb_data(keystr),
+ last_stabilize_key()) == 0) {
+ DEBUG(10, ("Can't store %s as a key\n", keystr));
+ return false;
+ }
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr && value);
+ if ((keystr == NULL) || (blob == NULL)) {
+ return false;
+ }
if (!gencache_init()) return False;
- if (asprintf(&valstr, CACHE_DATA_FMT, (int)timeout, value) == -1) {
+ val = talloc_asprintf(talloc_tos(), CACHE_DATA_FMT, (int)timeout);
+ if (val == NULL) {
return False;
}
+ val = talloc_realloc(NULL, val, char, talloc_array_length(val)-1);
+ if (val == NULL) {
+ return false;
+ }
+ val = (char *)talloc_append_blob(NULL, val, *blob);
+ if (val == NULL) {
+ return false;
+ }
- databuf = string_term_tdb_data(valstr);
- DEBUG(10, ("Adding cache entry with key = %s; value = %s and timeout ="
- " %s (%d seconds %s)\n", keystr, value,ctime(&timeout),
+ DEBUG(10, ("Adding cache entry with key = %s and timeout ="
+ " %s (%d seconds %s)\n", keystr, ctime(&timeout),
(int)(timeout - time(NULL)),
timeout > time(NULL) ? "ahead" : "in the past"));
- ret = tdb_store_bystring(cache, keystr, databuf, 0);
- SAFE_FREE(valstr);
+ ret = tdb_store_bystring(
+ cache_notrans, keystr,
+ make_tdb_data((uint8_t *)val, talloc_array_length(val)),
+ 0);
+ TALLOC_FREE(val);
+ if (ret != 0) {
+ return false;
+ }
+
+ /*
+ * Every 100 writes within a single process, stabilize the cache with
+ * a transaction. This is done to prevent a single transaction to
+ * become huge and chew lots of memory.
+ */
+ writecount += 1;
+ if (writecount > lp_parm_int(-1, "gencache", "stabilize_count", 100)) {
+ gencache_stabilize();
+ writecount = 0;
+ goto done;
+ }
+
+ /*
+ * Every 5 minutes, call gencache_stabilize() to not let grow
+ * gencache_notrans.tdb too large.
+ */
+
+ last_stabilize = 0;
+ databuf = tdb_fetch(cache_notrans, last_stabilize_key());
+ if ((databuf.dptr != NULL)
+ && (databuf.dptr[databuf.dsize-1] == '\0')) {
+ last_stabilize = atoi((char *)databuf.dptr);
+ SAFE_FREE(databuf.dptr);
+ }
+ if ((last_stabilize
+ + lp_parm_int(-1, "gencache", "stabilize_interval", 300))
+ < time(NULL)) {
+ gencache_stabilize();
+ }
+
+done:
return ret == 0;
}
@@ -147,26 +208,63 @@ bool gencache_set(const char *keystr, const char *value,
time_t timeout)
bool gencache_del(const char *keystr)
{
- int ret;
+ bool exists;
+ bool ret = false;
+ char *value;
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr);
+ if (keystr == NULL) {
+ return false;
+ }
if (!gencache_init()) return False;
DEBUG(10, ("Deleting cache entry (key = %s)\n", keystr));
- ret = tdb_delete_bystring(cache, keystr);
- return ret == 0;
+ if (tdb_lock_bystring(cache_notrans, keystr) == -1) {
+ DEBUG(5, ("Could not lock key for %s\n", keystr));
+ return false;
+ }
+
+ /*
+ * We delete an element by setting its timeout to 0. This way we don't
+ * have to do a transaction on gencache.tdb every time we delete an
+ * element.
+ */
+
+ exists = gencache_get(keystr, &value, NULL);
+ if (exists) {
+ SAFE_FREE(value);
+ ret = gencache_set(keystr, "", 0);
+ }
+ tdb_unlock_bystring(cache_notrans, keystr);
+ return ret;
}
+static bool gencache_pull_timeout(char *val, time_t *pres, char **pendptr)
+{
+ time_t res;
+ char *endptr;
+
+ res = strtol(val, &endptr, 10);
+
+ if ((endptr == NULL) || (*endptr != '/')) {
+ DEBUG(2, ("Invalid gencache data format: %s\n", val));
+ return false;
+ }
+ if (pres != NULL) {
+ *pres = res;
+ }
+ if (pendptr != NULL) {
+ *pendptr = endptr;
+ }
+ return true;
+}
/**
* Get existing entry from the cache file.
*
* @param keystr string that represents a key of this entry
- * @param valstr buffer that is allocated and filled with the entry value
- * buffer's disposing must be done outside
+ * @param blob DATA_BLOB that is filled with entry's blob
* @param timeout pointer to a time_t that is filled with entry's
* timeout
*
@@ -174,31 +272,40 @@ bool gencache_del(const char *keystr)
* @retval False for failure
**/
-bool gencache_get(const char *keystr, char **valstr, time_t *timeout)
+bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob,
+ time_t *timeout)
{
TDB_DATA databuf;
time_t t;
char *endptr;
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr);
+ if (keystr == NULL) {
+ return false;
+ }
+
+ if (tdb_data_cmp(string_term_tdb_data(keystr),
+ last_stabilize_key()) == 0) {
+ DEBUG(10, ("Can't get %s as a key\n", keystr));
+ return false;
+ }
if (!gencache_init()) {
return False;
}
- databuf = tdb_fetch_bystring(cache, keystr);
+ databuf = tdb_fetch_bystring(cache_notrans, keystr);
if (databuf.dptr == NULL) {
- DEBUG(10, ("Cache entry with key = %s couldn't be found\n",
+ databuf = tdb_fetch_bystring(cache, keystr);
+ }
+
+ if (databuf.dptr == NULL) {
+ DEBUG(10, ("Cache entry with key = %s couldn't be found \n",
keystr));
return False;
}
- t = strtol((const char *)databuf.dptr, &endptr, 10);
-
- if ((endptr == NULL) || (*endptr != '/')) {
- DEBUG(2, ("Invalid gencache data format: %s\n", databuf.dptr));
+ if (!gencache_pull_timeout((char *)databuf.dptr, &t, &endptr)) {
SAFE_FREE(databuf.dptr);
return False;
}
@@ -207,20 +314,33 @@ bool gencache_get(const char *keystr, char **valstr,
time_t *timeout)
"timeout = %s", t > time(NULL) ? "valid" :
"expired", keystr, endptr+1, ctime(&t)));
+ if (t == 0) {
+ /* Deleted */
+ SAFE_FREE(databuf.dptr);
+ return False;
+ }
+
if (t <= time(NULL)) {
- /* We're expired, delete the entry */
- tdb_delete_bystring(cache, keystr);
+ /*
+ * We're expired, delete the entry. We can't use gencache_del
+ * here, because that uses gencache_get_data_blob for checking
+ * the existence of a record. We know the thing exists and
+ * directly store an empty value with 0 timeout.
+ */
+ gencache_set(keystr, "", 0);
SAFE_FREE(databuf.dptr);
return False;
}
- if (valstr) {
- *valstr = SMB_STRDUP(endptr+1);
- if (*valstr == NULL) {
+ if (blob != NULL) {
+ *blob = data_blob(
+ endptr+1,
+ databuf.dsize - PTR_DIFF(endptr+1, databuf.dptr));
+ if (blob->data == NULL) {
SAFE_FREE(databuf.dptr);
- DEBUG(0, ("strdup failed\n"));
+ DEBUG(0, ("memdup failed\n"));
return False;
}
}
@@ -234,155 +354,192 @@ bool gencache_get(const char *keystr, char **valstr,
time_t *timeout)
return True;
}
+struct stabilize_state {
+ bool written;
+ bool error;
+};
+static int stabilize_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA val,
+ void *priv);
+
/**
- * Get existing entry from the cache file.
- *
- * @param keystr string that represents a key of this entry
- * @param blob DATA_BLOB that is filled with entry's blob
- * @param expired pointer to a bool that indicates whether the entry is expired
+ * Stabilize gencache
*
- * @retval true when entry is successfuly fetched
- * @retval False for failure
- **/
+ * Migrate the clear-if-first gencache data to the stable,
+ * transaction-based gencache.tdb
+ */
-bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob, bool *expired)
+bool gencache_stabilize(void)
{
- TDB_DATA databuf;
- time_t t;
- char *blob_type;
- unsigned char *buf = NULL;
- bool ret = False;
- fstring valstr;
- int buflen = 0, len = 0, blob_len = 0;
- unsigned char *blob_buf = NULL;
-
- /* fail completely if get null pointers passed */
- SMB_ASSERT(keystr);
+ struct stabilize_state state;
+ int res;
+ char *now;
if (!gencache_init()) {
- return False;
+ return false;
}
- databuf = tdb_fetch_bystring(cache, keystr);
- if (!databuf.dptr) {
- DEBUG(10,("Cache entry with key = %s couldn't be found\n",
- keystr));
- return False;
+ res = tdb_transaction_start(cache);
+ if (res == -1) {
+ DEBUG(10, ("Could not start transaction on gencache.tdb: "
+ "%s\n", tdb_errorstr(cache)));
+ return false;
+ }
+ res = tdb_transaction_start(cache_notrans);
+ if (res == -1) {
+ tdb_transaction_cancel(cache);
+ DEBUG(10, ("Could not start transaction on "
+ "gencache_notrans.tdb: %s\n",
+ tdb_errorstr(cache_notrans)));
+ return false;
}
- buf = (unsigned char *)databuf.dptr;
- buflen = databuf.dsize;
+ state.error = false;
+ state.written = false;
- len += tdb_unpack(buf+len, buflen-len, "fB",
- &valstr,
- &blob_len, &blob_buf);
- if (len == -1) {
- goto out;
+ res = tdb_traverse(cache_notrans, stabilize_fn, &state);
+ if ((res == -1) || state.error) {
+ if ((tdb_transaction_cancel(cache_notrans) == -1)
+ || (tdb_transaction_cancel(cache) == -1)) {
+ smb_panic("tdb_transaction_cancel failed\n");
+ }
+ return false;
}
- t = strtol(valstr, &blob_type, 10);
+ if (!state.written) {
+ if ((tdb_transaction_cancel(cache_notrans) == -1)
+ || (tdb_transaction_cancel(cache) == -1)) {
+ smb_panic("tdb_transaction_cancel failed\n");
+ }
+ return true;
+ }
- if (strcmp(blob_type+1, BLOB_TYPE) != 0) {
- goto out;
+ res = tdb_transaction_commit(cache);
+ if (res == -1) {
+ DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
+ "%s\n", tdb_errorstr(cache)));
--
Samba Shared Repository
