FWIW, I like this... makes it better for me to atleast grab some stuff and copy over ***if*** it corrupts again.
thanks On Thu, Sep 1, 2011 at 3:22 PM, Marco Peereboom <sl...@peereboom.us> wrote: > Alright this diff keeps the file open and appends lines to HISTFILE. It > only rewrites HISTFILE at 125% of HISTSIZE. Does fancy locking and > deals with signals too. So unless someone finds some bugs I'll consider > this version final. > > Yes, no on moving ksh history to text? > > Other comments? > > Index: alloc.c > =================================================================== > RCS file: /cvs/src/bin/ksh/alloc.c,v > retrieving revision 1.8 > diff -u -p -r1.8 alloc.c > --- alloc.c 21 Jul 2008 17:30:08 -0000 1.8 > +++ alloc.c 30 Aug 2011 18:05:47 -0000 > @@ -62,7 +62,7 @@ alloc(size_t size, Area *ap) > { > struct link *l; > > - l = malloc(sizeof(struct link) + size); > + l = calloc(1, sizeof(struct link) + size); > if (l == NULL) > internal_errorf(1, "unable to allocate memory"); > l->next = ap->freelist; > Index: history.c > =================================================================== > RCS file: /cvs/src/bin/ksh/history.c,v > retrieving revision 1.39 > diff -u -p -r1.39 history.c > --- history.c 19 May 2010 17:36:08 -0000 1.39 > +++ history.c 1 Sep 2011 20:14:44 -0000 > @@ -11,8 +11,7 @@ > * a) the original in-memory history mechanism > * b) a more complicated mechanism done by p...@hillside.co.uk > * that more closely follows the real ksh way of doing > - * things. You need to have the mmap system call for this > - * to work on your system > + * things. > */ > > #include "sh.h" > @@ -20,21 +19,11 @@ > > #ifdef HISTORY > # include <sys/file.h> > -# include <sys/mman.h> > > -/* > - * variables for handling the data file > - */ > -static int histfd; > -static int hsize; > - > -static int hist_count_lines(unsigned char *, int); > -static int hist_shrink(unsigned char *, int); > -static unsigned char *hist_skip_back(unsigned char *,int *,int); > -static void histload(Source *, unsigned char *, int); > -static void histinsert(Source *, int, unsigned char *); > -static void writehistfile(int, char *); > -static int sprinkle(int); > +static void writehistfile(void); > +static FILE *history_open(void); > +static int history_load(Source *); > +static void history_close(void); > > static int hist_execute(char *); > static int hist_replace(char **, const char *, const char *, int); > @@ -42,11 +31,14 @@ static char **hist_get(const char *, i > static char **hist_get_oldest(void); > static void histbackup(void); > > +static FILE *histfd; > static char **current; /* current position in history[] */ > static char *hname; /* current name of history file */ > static int hstarted; /* set after hist_init() called */ > -static Source *hist_source; > +static Source *hist_source; > +static uint32_t line_co; > > +static struct stat last_sb; > > int > c_fc(char **wp) > @@ -529,15 +521,10 @@ sethistfile(const char *name) > /* if the name is the same as the name we have */ > if (hname && strcmp(hname, name) == 0) > return; > - > /* > * its a new name - possibly > */ > - if (histfd) { > - /* yes the file is open */ > - (void) close(histfd); > - histfd = 0; > - hsize = 0; > + if (hname) { > afree(hname, APERM); > hname = NULL; > /* let's reset the history */ > @@ -545,6 +532,9 @@ sethistfile(const char *name) > hist_source->line = 0; > } > > + if (histfd) > + history_close(); > + > hist_init(hist_source); > } > > @@ -561,6 +551,27 @@ init_histvec(void) > } > } > > +static void > +history_lock(void) > +{ > + while (flock(fileno(histfd), LOCK_EX) != 0) { > + if (errno == EINTR || errno == EAGAIN) > + continue; > + else > + break; > + } > +} > + > +static void > +history_unlock(void) > +{ > + while (flock(fileno(histfd), LOCK_UN) != 0) { > + if (errno == EINTR || errno == EAGAIN) > + continue; > + else > + break; > + } > +} > > /* > * Routines added by Peter Collinson BSDI(Europe)/Hillside Systems to > @@ -577,18 +588,29 @@ init_histvec(void) > void > histsave(int lno, const char *cmd, int dowrite) > { > - char **hp; > - char *c, *cp; > + char **hp; > + char *c, *cp; > + struct stat sb; > + > + if (dowrite && histfd) { > + history_lock(); > + if (fstat(fileno(histfd), &sb) != -1) { > + if (timespeccmp(&sb.st_mtim, &last_sb.st_mtim, ==)) > + ; /* file is unchanged */ > + else { > + /* reset history */ > + histptr = history - 1; > + hist_source->line = 0; > + history_load(hist_source); > + } > + } > + } > > c = str_save(cmd, APERM); > if ((cp = strchr(c, '\n')) != NULL) > *cp = '\0'; > > - if (histfd && dowrite) > - writehistfile(lno, c); > - > hp = histptr; > - > if (++hp >= history + histsize) { /* remove oldest command */ > afree((void*)*history, APERM); > for (hp = history; hp < history + histsize - 1; hp++) > @@ -596,371 +618,143 @@ histsave(int lno, const char *cmd, int d > } > *hp = c; > histptr = hp; > -} > - > -/* > - * Write history data to a file nominated by HISTFILE > - * if HISTFILE is unset then history still happens, but > - * the data is not written to a file > - * All copies of ksh looking at the file will maintain the > - * same history. This is ksh behaviour. > - * > - * This stuff uses mmap() > - * if your system ain't got it - then you'll have to undef HISTORYFILE > - */ > > -/* > - * Open a history file > - * Format is: > - * Bytes 1, 2: HMAGIC - just to check that we are dealing with > - * the correct object > - * Then follows a number of stored commands > - * Each command is > - * <command byte><command number(4 bytes)><bytes><null> > - */ > -#define HMAGIC1 0xab > -#define HMAGIC2 0xcd > -#define COMMAND 0xff > + if (dowrite && histfd) { > + /* append to file */ > + if (fseeko(histfd, 0, SEEK_END) == 0) { > + fprintf(histfd, "%s", cmd); > + fflush(histfd); > + fstat(fileno(histfd), &last_sb); > + line_co++; > + writehistfile(); > + } > + history_unlock(); > + } > +} > > -void > -hist_init(Source *s) > +static FILE * > +history_open(void) > { > - unsigned char *base; > - int lines; > - int fd; > - > - if (Flag(FTALKING) == 0) > - return; > - > - hstarted = 1; > - > - hist_source = s; > - > - hname = str_val(global("HISTFILE")); > - if (hname == NULL) > - return; > - hname = str_save(hname, APERM); > + int fd, fddup; > + FILE *f = NULL; > > - retry: > - /* we have a file and are interactive */ > - if ((fd = open(hname, O_RDWR|O_CREAT|O_APPEND, 0600)) < 0) > - return; > - > - histfd = savefd(fd); > - if (histfd != fd) > + if ((fd = open(hname, O_RDWR | O_CREAT | O_EXLOCK, 0600)) == -1) > + return (NULL); > + fddup = savefd(fd); > + if (fddup != fd) > close(fd); > > - (void) flock(histfd, LOCK_EX); > + f = fdopen(fddup, "r+"); > + if (f == NULL) { > + close(fddup); > + goto bad; > + } > > - hsize = lseek(histfd, 0L, SEEK_END); > + fstat(fileno(f), &last_sb); > > - if (hsize == 0) { > - /* add magic */ > - if (sprinkle(histfd)) { > - hist_finish(); > - return; > - } > - } > - else if (hsize > 0) { > - /* > - * we have some data > - */ > - base = (unsigned char *)mmap(0, hsize, PROT_READ, > - MAP_FILE|MAP_PRIVATE, histfd, 0); > - /* > - * check on its validity > - */ > - if (base == MAP_FAILED || *base != HMAGIC1 || base[1] != HMAGIC2) { > - if (base != MAP_FAILED) > - munmap((caddr_t)base, hsize); > - hist_finish(); > - if (unlink(hname) != 0) > - return; > - goto retry; > - } > - if (hsize > 2) { > - lines = hist_count_lines(base+2, hsize-2); > - if (lines > histsize) { > - /* we need to make the file smaller */ > - if (hist_shrink(base, hsize)) > - if (unlink(hname) != 0) > - return; > - munmap((caddr_t)base, hsize); > - hist_finish(); > - goto retry; > - } > - } > - histload(hist_source, base+2, hsize-2); > - munmap((caddr_t)base, hsize); > - } > - (void) flock(histfd, LOCK_UN); > - hsize = lseek(histfd, 0L, SEEK_END); > -} > + return (f); > +bad: > + if (f) > + fclose(f); > > -typedef enum state { > - shdr, /* expecting a header */ > - sline, /* looking for a null byte to end the line */ > - sn1, /* bytes 1 to 4 of a line no */ > - sn2, sn3, sn4 > -} State; > + return (NULL); > +} > > -static int > -hist_count_lines(unsigned char *base, int bytes) > +static void > +history_close(void) > { > - State state = shdr; > - int lines = 0; > - > - while (bytes--) { > - switch (state) { > - case shdr: > - if (*base == COMMAND) > - state = sn1; > - break; > - case sn1: > - state = sn2; break; > - case sn2: > - state = sn3; break; > - case sn3: > - state = sn4; break; > - case sn4: > - state = sline; break; > - case sline: > - if (*base == '\0') > - lines++, state = shdr; > - } > - base++; > + if (histfd) { > + fflush(histfd); > + fclose(histfd); > + histfd = NULL; > } > - return lines; > } > > -/* > - * Shrink the history file to histsize lines > - */ > static int > -hist_shrink(unsigned char *oldbase, int oldbytes) > +history_load(Source *s) > { > - int fd; > - char nfile[1024]; > - struct stat statb; > - unsigned char *nbase = oldbase; > - int nbytes = oldbytes; > + char *p, line[LINE + 1]; > > - nbase = hist_skip_back(nbase, &nbytes, histsize); > - if (nbase == NULL) > - return 1; > - if (nbase == oldbase) > - return 0; > + rewind(histfd); > > - /* > - * create temp file > - */ > - (void) shf_snprintf(nfile, sizeof(nfile), "%s.%d", hname, procpid); > - if ((fd = open(nfile, O_CREAT | O_TRUNC | O_WRONLY, 0600)) < 0) > - return 1; > + /* just read it all; will auto resize history upon next command */ > + for (line_co = 1; ; line_co++) { > + p = fgets(line, sizeof line, histfd); > + if (p == NULL || feof(histfd) || ferror(histfd)) > + break; > + if ((p = strchr(line, '\n')) == NULL) { > + bi_errorf("history file is corrupt"); > + return (1); > + } > + *p = '\0'; > > - if (sprinkle(fd)) { > - close(fd); > - unlink(nfile); > - return 1; > + s->line = line_co; > + s->cmd_offset = line_co; > + histsave(line_co, (char *)line, 0); > } > - if (write(fd, nbase, nbytes) != nbytes) { > - close(fd); > - unlink(nfile); > - return 1; > - } > - /* > - * worry about who owns this file > - */ > - if (fstat(histfd, &statb) >= 0) > - fchown(fd, statb.st_uid, statb.st_gid); > - close(fd); > > - /* > - * rename > - */ > - if (rename(nfile, hname) < 0) > - return 1; > - return 0; > -} > + writehistfile(); > > + return (0); > +} > > -/* > - * find a pointer to the data `no' back from the end of the file > - * return the pointer and the number of bytes left > - */ > -static unsigned char * > -hist_skip_back(unsigned char *base, int *bytes, int no) > +void > +hist_init(Source *s) > { > - int lines = 0; > - unsigned char *ep; > + if (Flag(FTALKING) == 0) > + return; > > - for (ep = base + *bytes; --ep > base; ) { > - /* this doesn't really work: the 4 byte line number that is > - * encoded after the COMMAND byte can itself contain the > - * COMMAND byte.... > - */ > - for (; ep > base && *ep != COMMAND; ep--) > - ; > - if (ep == base) > - break; > - if (++lines == no) { > - *bytes = *bytes - ((char *)ep - (char *)base); > - return ep; > - } > - } > - return NULL; > -} > + hstarted = 1; > > -/* > - * load the history structure from the stored data > - */ > -static void > -histload(Source *s, unsigned char *base, int bytes) > -{ > - State state; > - int lno = 0; > - unsigned char *line = NULL; > - > - for (state = shdr; bytes-- > 0; base++) { > - switch (state) { > - case shdr: > - if (*base == COMMAND) > - state = sn1; > - break; > - case sn1: > - lno = (((*base)&0xff)<<24); > - state = sn2; > - break; > - case sn2: > - lno |= (((*base)&0xff)<<16); > - state = sn3; > - break; > - case sn3: > - lno |= (((*base)&0xff)<<8); > - state = sn4; > - break; > - case sn4: > - lno |= (*base)&0xff; > - line = base+1; > - state = sline; > - break; > - case sline: > - if (*base == '\0') { > - /* worry about line numbers */ > - if (histptr >= history && lno-1 != s->line) { > - /* a replacement ? */ > - histinsert(s, lno, line); > - } > - else { > - s->line = lno; > - s->cmd_offset = lno; > - histsave(lno, (char *)line, 0); > - } > - state = shdr; > - } > - } > - } > -} > + hist_source = s; > > -/* > - * Insert a line into the history at a specified number > - */ > -static void > -histinsert(Source *s, int lno, unsigned char *line) > -{ > - char **hp; > + hname = str_val(global("HISTFILE")); > + if (hname == NULL) > + return; > + hname = str_save(hname, APERM); > + histfd = history_open(); > + if (histfd == NULL) > + return; > > - if (lno >= s->line-(histptr-history) && lno <= s->line) { > - hp = &histptr[lno-s->line]; > - if (*hp) > - afree((void*)*hp, APERM); > - *hp = str_save((char *)line, APERM); > - } > + history_load(s); > + > + history_unlock(); > } > > -/* > - * write a command to the end of the history file > - * This *MAY* seem easy but it's also necessary to check > - * that the history file has not changed in size. > - * If it has - then some other shell has written to it > - * and we should read those commands to update our history > - */ > static void > -writehistfile(int lno, char *cmd) > +writehistfile(void) > { > - int sizenow; > - unsigned char *base; > - unsigned char *new; > - int bytes; > - unsigned char hdr[5]; > - > - (void) flock(histfd, LOCK_EX); > - sizenow = lseek(histfd, 0L, SEEK_END); > - if (sizenow != hsize) { > - /* > - * Things have changed > - */ > - if (sizenow > hsize) { > - /* someone has added some lines */ > - bytes = sizenow - hsize; > - base = (unsigned char *)mmap(0, sizenow, > - PROT_READ, MAP_FILE|MAP_PRIVATE, histfd, 0); > - if (base == MAP_FAILED) > - goto bad; > - new = base + hsize; > - if (*new != COMMAND) { > - munmap((caddr_t)base, sizenow); > - goto bad; > - } > - hist_source->line--; > - histload(hist_source, new, bytes); > - hist_source->line++; > - lno = hist_source->line; > - munmap((caddr_t)base, sizenow); > - hsize = sizenow; > - } else { > - /* it has shrunk */ > - /* but to what? */ > - /* we'll give up for now */ > - goto bad; > - } > + int i; > + char *cmd; > + > + /* see if file has grown over 25% */ > + if (line_co < histsize + (histsize / 4)) > + return; > + > + if (ftruncate(fileno(histfd), 0) == -1) > + return; > + > + /* rewrite the whole caboodle */ > + rewind(histfd); > + for (i = 0; i < histsize; i++) { > + cmd = history[i]; > + if (cmd == NULL) > + break; > + if (fprintf(histfd, "%s\n", cmd) == -1) > + return; > } > - /* > - * we can write our bit now > - */ > - hdr[0] = COMMAND; > - hdr[1] = (lno>>24)&0xff; > - hdr[2] = (lno>>16)&0xff; > - hdr[3] = (lno>>8)&0xff; > - hdr[4] = lno&0xff; > - (void) write(histfd, hdr, 5); > - (void) write(histfd, cmd, strlen(cmd)+1); > - hsize = lseek(histfd, 0L, SEEK_END); > - (void) flock(histfd, LOCK_UN); > - return; > -bad: > - hist_finish(); > + > + line_co = histsize; > + > + fflush(histfd); > + fstat(fileno(histfd), &last_sb); > } > > void > hist_finish(void) > { > - (void) flock(histfd, LOCK_UN); > - (void) close(histfd); > - histfd = 0; > + history_close(); > } > - > -/* > - * add magic to the history file > - */ > -static int > -sprinkle(int fd) > -{ > - static unsigned char mag[] = { HMAGIC1, HMAGIC2 }; > - > - return(write(fd, mag, 2) != 2); > -} > - > #else /* HISTORY */ > > /* No history to be compiled in: dummy routines to avoid lots more ifdefs */