Changeset: cdcf490ccdd3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cdcf490ccdd3
Modified Files:
monetdb5/modules/mal/wlcr.c
monetdb5/modules/mal/wlcr.h
monetdb5/modules/mal/wlcr.mal
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_wlcr.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_wlcr.c
sql/backends/monet5/sql_wlcr.h
sql/backends/monet5/sql_wlcr.mal
sql/server/sql_mvc.c
Branch: wlcr
Log Message:
Sync code reshuffling
diffs (truncated from 1849 to 300 lines):
diff --git a/monetdb5/modules/mal/wlcr.c b/monetdb5/modules/mal/wlcr.c
--- a/monetdb5/modules/mal/wlcr.c
+++ b/monetdb5/modules/mal/wlcr.c
@@ -8,8 +8,9 @@
/*
* (c) 2017 Martin Kersten
- * This module collects the workload-capture-replay statements during
transaction execution.
- * It is used primarilly for replication management and workload replay
+ * This module collects the workload-capture-replay statements during
transaction execution,
+ * also called asynchronous logical replication log management.
+ * It is used primarilly for recovery and replication management.
*
* The goal is to easily clone a master database. Accidental corruption of
this
* data should ne avoided by setting ownership and access properties at the
SQL level in the replica.
@@ -22,26 +23,33 @@
*
* It creates a directory .../dbfarm/dbname/master to hold all necessary
information
* for the creation and maintenance of replicas.
+ * A configuration file is added to keep track on the status of the master.
+ * It contains the following key=value pairs:
+ * snapshot=<path to binary snapshot>
+ * archive=<path to location of the log files>
+ * start=<first batch file to be applied>
+ * last=<last batch file to be applied>
*
* Every replica should start off with a copy of binary snapshot stored
- * in .../dbfarm/dbname/master/bat or with an empty database.
- * The associated log files are stored as master/wlcr<number>.
+ * by deault in .../dbfarm/dbname/master/bat. A missing path to the snapshot
+ * is assumed to denote that we can start with an empty database.
+ * The log files are stored as master/<dbname>_<batchnumber>.
*
* Each wlcr log file contains a serial log of committed transactions.
* The log records are represented as ordinary MAL statements, which
- * are executed in serial mode.
+ * are executed in serial mode. (parallelism can be considered for large
updates)
* Each transaction job is identified by the owner of the query,
* commit/rollback status, its starting time and runtime (in ms).
* The end of the transaction is marked as exec()
*
- * Logging of queries can be limited to those that satisfy an execution
threshold.
+ * Logging of queries can be limited to those that satisfy an minimum
execution threshold.
* SET replaythreshold= <number>
* The threshold is given in milliseconds. A negative threshold leads to
ignoring all queries.
* The threshold setting is not saved because it is a client specific action.
*
* A replica is constructed in three steps.
- * 1) a snapshot copy of the BAT directory is created within a SQL
transaction.[TODO]
- * 2) a new database is created from this snapshot using monetdb[TODO]
+ * 1a) a snapshot copy of the BAT directory is created within a SQL
transaction.[TODO or
+ * 1b)) a new database is created .
* 3) The logs are replayed to bring the snapshot up to date.
*
* Step 1) and 2) can be avoided by starting with an empty database and
@@ -51,8 +59,8 @@
* Processing the log files starts in the background using the call.
* CALL clone("dbname")
* It will iterate through the log files, applying all transactions.
- * Queries are simply ignored unless needed as replacement for catalog
actions..
- * [TODO] the user might want to take a time-stamped version only, ignoring
all actions afterwards.
+ * Queries are simply ignored unless needed as replacement for catalog actions.
+ * [TODO] the user might want to indicate a time-stamped, ignoring all actions
afterwards.
*
* The alternative is to replay the complete query log
* CALL replay("dbname")
@@ -78,16 +86,16 @@
static MT_Lock wlcr_lock MT_LOCK_INITIALIZER("wlcr_lock");
-
-int wlcr_start = 0; // first batch associated with snapshot
-int wlcr_batch = 0; // last batch job identifier
-int wlcr_tid = 0; // last transaction id
-
static char *wlcr_name[]= {"","query","update","catalog"};
-static str wlcr_dbname = 0;
+static str wlcr_snapshot= 0; // The name of the snapshot against the logs work
+static str wlcr_archive = 0; // The location in the global file store for the
logs
+static str wlcr_dbname = 0; // The current database name
static stream *wlcr_fd = 0;
-static str wlcr_dir = 0;
+
+int wlcr_start = 0; // first log file associated with snapshot
+int wlcr_last = 0; // last log file identifier
+int wlcr_tid = 0; // last transaction id
/* The database snapshots are binary copies of the dbfarm/database/bat
* New snapshots are created currently using the 'monetdb snapshot <db>'
command
@@ -97,38 +105,85 @@ static str wlcr_dir = 0;
*/
int
-WLCRused(void)
+WLCused(void)
{
- return wlcr_dir != NULL;
+ return wlcr_archive != NULL;
}
-// creation of file and updating the version file should be atomic TODO!!!
-// The log files are marked with the database name. This allows for easy
recognition later on.
-static str
-WLCRloggerfile(Client cntxt)
-{
+
+/* The master configuration file is a simple key=value table */
+static
+str WLCgetConfig(void){
char path[PATHLENGTH];
FILE *fd;
- (void) cntxt;
- snprintf(path,PATHLENGTH,"%s%c%s_%06d", wlcr_dir, DIR_SEP, wlcr_dbname,
wlcr_batch);
- wlcr_fd = open_wastream(path);
- if( wlcr_fd == 0)
- throw(MAL,"wlcr.logger","Could not create %s\n",path);
+ snprintf(path,PATHLENGTH,"%s%cwlcr.config", wlcr_archive, DIR_SEP);
+ fd = fopen(path,"r");
+ if( fd == NULL)
+ throw(MAL,"wlcr.getConfig","Could not access %s\n",path);
+ while( fgets(path, PATHLENGTH, fd) ){
+ path[strlen(path)-1] = 0;
+ if( strncmp("archive=", path,8) == 0)
+ wlcr_archive = GDKstrdup(path + 8);
+ if( strncmp("snapshot=", path,9) == 0)
+ wlcr_snapshot = GDKstrdup(path + 9);
+ if( strncmp("start=", path,6) == 0)
+ wlcr_start = atoi(path+ 6);
+ if( strncmp("last=", path, 5) == 0)
+ wlcr_last = atoi(path+ 5);
+ }
+ fclose(fd);
+ return MAL_SUCCEED;
+}
- wlcr_batch++;
- wlcr_tid = 0;
- snprintf(path,PATHLENGTH,"%s%c%s_wlcr", wlcr_dir, DIR_SEP, wlcr_dbname);
-#ifdef _WLCR_DEBUG_
- mnstr_printf(cntxt->fdout,"#WLCRloggerfile %s\n", wlcr_dir);
-#endif
+static
+str WLCsetConfig(void){
+ char path[PATHLENGTH];
+ FILE *fd;
+
+ // default setting for the archive directory is the db itself
+ if( wlcr_archive == NULL){
+ snprintf(path,PATHLENGTH,"%s%c", wlcr_archive, DIR_SEP);
+ wlcr_archive = GDKstrdup(path);
+ }
+
+ snprintf(path,PATHLENGTH,"%s%cwlcr.config", wlcr_archive, DIR_SEP);
fd = fopen(path,"w");
if( fd == NULL)
- throw(MAL,"wlcr.logger","Could not access %s\n",path);
- fprintf(fd,"%d %d\n", wlcr_start, wlcr_batch );
+ throw(MAL,"wlcr.setConfig","Could not access %s\n",path);
+ if( wlcr_snapshot)
+ fprintf(fd,"snapshot=%s\n", wlcr_snapshot);
+ if( wlcr_archive)
+ fprintf(fd,"archive=%s\n", wlcr_archive);
+ fprintf(fd,"start=%d\n", wlcr_start);
+ fprintf(fd,"last=%d\n", wlcr_last );
fclose(fd);
return MAL_SUCCEED;
}
+// creation of the logger file and updating the configuration file should be
atomic !!!
+// The log files are marked with the database name. This allows for easy
recognition later on.
+static str
+WLCsetlogger(void)
+{
+ char path[PATHLENGTH];
+
+ if( wlcr_archive == NULL)
+ throw(MAL,"wlcr.setlogger","Path not initalized");
+ MT_lock_set(&wlcr_lock);
+ snprintf(path,PATHLENGTH,"%s%c%s_%012d", wlcr_archive, DIR_SEP,
wlcr_dbname, wlcr_last);
+ wlcr_fd = open_wastream(path);
+ if( wlcr_fd == 0){
+ MT_lock_unset(&wlcr_lock);
+ throw(MAL,"wlcr.logger","Could not create %s\n",path);
+ }
+
+ wlcr_last++;
+ wlcr_tid = 0;
+ WLCsetConfig();
+ MT_lock_unset(&wlcr_lock);
+ return MAL_SUCCEED;
+}
+
/*
* The existence of the master directory should be checked upon server restart.
* A new batch file should be created as a result.
@@ -136,110 +191,77 @@ WLCRloggerfile(Client cntxt)
* Upon exit we should check the log file size. If empty we need not safe it.
[TODO]
*/
str
-WLCRinit(Client cntxt)
+WLCinit(Client cntxt)
{
- str dbname= (str) GDKgetenv("gdk_dbname");
- str dir;
- str msg = MAL_SUCCEED;
+ char path[PATHLENGTH];
+ str pathname;
FILE *fd;
- char path[PATHLENGTH];
-
- if( wlcr_dir){
-#ifdef _WLCR_DEBUG_
- mnstr_printf(cntxt->fdout,"#WLCR already running\n");
+
+ if( wlcr_archive){
+#ifdef _WLC_DEBUG_
+ mnstr_printf(cntxt->fdout,"#WLC already running\n");
+#else
+ (void) cntxt;
#endif
- return MAL_SUCCEED;
+ } else{
+ // use default location for archive
+ pathname = GDKfilepath(0,0,"master",0);
+ snprintf(path, PATHLENGTH,"%s%cwlcr.config", pathname, DIR_SEP);
+
+ fd = fopen(path,"r");
+ if( fd == NULL)
+ return MAL_SUCCEED;
+ fclose(fd);
+ wlcr_dbname = GDKgetenv("gdk_dbname");
+ wlcr_archive = pathname;
+ return WLCgetConfig();
}
-
- if (dbname){
- dir = GDKfilepath(0,0,"master",0);
- snprintf(path, PATHLENGTH,"%s%c%s_wlcr",dir, DIR_SEP, dbname);
- fd = fopen(path,"r");
- if( fd){
- // database is in master tracking mode
- if( fscanf(fd,"%d %d", &wlcr_start, &wlcr_batch ) == 2){
- if( wlcr_batch < 0){
- // logging was stopped
- (void) fclose(fd);
- return MAL_SUCCEED;
- }
- wlcr_dbname = GDKstrdup(dbname);
- wlcr_dir = dir;
-#ifdef _WLCR_DEBUG_
- mnstr_printf(cntxt->fdout,"#Master control
active:%d\n", wlcr_batch);
-#endif
- (void) fclose(fd);
- msg = WLCRloggerfile(cntxt);
- } else{
-#ifdef _WLCR_DEBUG_
- mnstr_printf(cntxt->fdout,"#Inconsistent master
control:%d\n", wlcr_batch);
-#endif
- (void) fclose(fd);
- }
- }
-#ifdef _WLCR_DEBUG_
- else
- mnstr_printf(cntxt->fdout,"#Master control not
active\n");
-#endif
- }
- return msg;
+ return MAL_SUCCEED;
}
str
-WLCRexit(void)
+WLCexit(void)
{
size_t sz;
- FILE *fd;
- char path[PATHLENGTH];
- // only keep non-empty log files [TODO]
+ // only keep non-empty log files
if( wlcr_fd){
sz = getFileSize(wlcr_fd);
if (sz == 0){
- snprintf(path, PATHLENGTH,"%s%c%s_wlcr",wlcr_dir,
DIR_SEP,wlcr_dbname);
- fd = fopen(path,"w");
- if( fd == NULL)
- throw(MAL,"wlcr.exit","File can not be access");
- fprintf(fd,"%d %d\n", wlcr_start, wlcr_batch -1);
+ wlcr_last --;
+ WLCsetConfig();
}
}
return MAL_SUCCEED;
}
str
-WLCRstop (Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+WLCstop (Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
