commit: 054269125460aadf20059019a7a106800160c69a
Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Mon Mar 10 07:12:02 2014 +0000
Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Mon Mar 10 07:12:02 2014 +0000
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/portage-utils.git;a=commit;h=05426912
eat_file: convert API to work on dynamic buffers
Rather than use static allocated buffers everywhere where we assume we
picked a size big enough for real world uses (and just silently truncate
or die in large edge cases), convert the API to use malloc instead. We
will grow the buffers (never shrink) so that repeat calls should ramp up
to the max quickly and thus avoid having to call malloc() repeatedly.
This might report memory leaks with some funcs as we hold on to some
buffers indefinitely since we know the buffer isn't used outside the
context of the func. Helps out when the func is called repeatedly.
This should make future enhancements (like eating more than one element
in a vdb pkg dir) a lot easier.
---
main.c | 108 +++++++++++++++++++++++++++++++++++++++++--------------------
qdepends.c | 14 ++++----
qfile.c | 9 ++++--
qlist.c | 5 +--
qlop.c | 13 ++++----
qmerge.c | 12 ++++---
6 files changed, 105 insertions(+), 56 deletions(-)
diff --git a/main.c b/main.c
index a9b5403..436387d 100644
--- a/main.c
+++ b/main.c
@@ -10,7 +10,9 @@
#include "main.h"
/* prototypes and such */
-static char eat_file(const char *file, char *buf, const size_t bufsize);
+static bool eat_file(const char *, char **, size_t *);
+static bool eat_file_fd(int, char **, size_t *);
+static bool eat_file_at(int, const char *, char **, size_t *);
int rematch(const char *, const char *, int);
static char *rmspace(char *);
@@ -170,47 +172,70 @@ static void version_barf(void)
exit(EXIT_SUCCESS);
}
-static char eat_file_fd(int fd, char *buf, const size_t bufsize)
+static bool eat_file_fd(int fd, char **bufptr, size_t *bufsize)
{
+ bool ret = true;
struct stat s;
+ char *buf;
+ size_t read_size;
+
+ /* First figure out how much data we should read from the fd. */
+ if (fd == -1 || fstat(fd, &s) != 0) {
+ ret = false;
+ read_size = 0;
+ /* Fall through so we set the first byte 0 */
+ } else if (!s.st_size) {
+ /* We might be trying to eat a virtual file like in /proc, so
+ * read an arbitrary size that should be "enough". */
+ read_size = BUFSIZE;
+ } else
+ read_size = (size_t)s.st_size;
+
+ /* Now allocate enough space (at least 1 byte). */
+ if (!*bufptr || *bufsize < read_size) {
+ /* We assume a min allocation size so that repeat calls don't
+ * hit ugly ramp ups -- if you read a file that is 1 byte, then
+ * 5 bytes, then 10 bytes, then 20 bytes, ... you'll allocate
+ * constantly. So we round up a few pages as wasiting virtual
+ * memory is cheap when it is unused. */
+ *bufsize = ((read_size + 1) + BUFSIZE - 1) & -BUFSIZE;
+ *bufptr = xrealloc(*bufptr, *bufsize);
+ }
+ buf = *bufptr;
+ /* Finally do the actual read. */
buf[0] = '\0';
- if (fstat(fd, &s) != 0)
- return 0;
- if (s.st_size) {
- if (bufsize < (size_t)s.st_size)
- return 0;
- if (read(fd, buf, s.st_size) != (ssize_t)s.st_size)
- return 0;
- buf[s.st_size] = '\0';
- } else {
- if (read(fd, buf, bufsize) == 0)
- return 0;
- buf[bufsize - 1] = '\0';
+ if (read_size) {
+ if (s.st_size) {
+ if (read(fd, buf, read_size) != (ssize_t)read_size)
+ return false;
+ buf[read_size] = '\0';
+ } else {
+ if (read(fd, buf, read_size) == 0)
+ return false;
+ buf[read_size - 1] = '\0';
+ }
}
- return 1;
+ return ret;
}
-static char eat_file_at(int dfd, const char *file, char *buf, const size_t
bufsize)
+static bool eat_file_at(int dfd, const char *file, char **bufptr, size_t
*bufsize)
{
+ bool ret;
int fd;
- char ret;
- if ((fd = openat(dfd, file, O_CLOEXEC|O_RDONLY)) == -1) {
- buf[0] = '\0';
- return 0;
- }
-
- ret = eat_file_fd(fd, buf, bufsize);
+ fd = openat(dfd, file, O_CLOEXEC|O_RDONLY);
+ ret = eat_file_fd(fd, bufptr, bufsize);
+ if (fd != -1)
+ close(fd);
- close(fd);
return ret;
}
-static char eat_file(const char *file, char *buf, const size_t bufsize)
+static bool eat_file(const char *file, char **bufptr, size_t *bufsize)
{
- return eat_file_at(AT_FDCWD, file, buf, bufsize);
+ return eat_file_at(AT_FDCWD, file, bufptr, bufsize);
}
static bool prompt(const char *p)
@@ -601,7 +626,10 @@ static void read_portage_profile(const char *configroot,
const char *profile, en
{
size_t configroot_len, profile_len, sub_len;
char *profile_file, *sub_file;
- char buf[BUFSIZE], *s;
+ char *s;
+
+ static char *buf;
+ static size_t buf_len;
/* initialize the base profile path */
configroot_len = strlen(configroot);
@@ -620,7 +648,7 @@ static void read_portage_profile(const char *configroot,
const char *profile, en
/* now walk all the parents */
strcpy(sub_file, "parent");
- if (eat_file(profile_file, buf, sizeof(buf)) == 0)
+ if (eat_file(profile_file, &buf, &buf_len) == 0)
goto done;
rmspace(buf);
@@ -1212,17 +1240,25 @@ char *atom_to_pvr(depend_atom *atom) {
return (atom->PR_int == 0 ? atom->P : atom->PVR );
}
+/* TODO: Delete this in favor of libq/vdb.c API. */
static char *grab_vdb_item(const char *item, const char *CATEGORY, const char
*PF)
{
- static char buf[_Q_PATH_MAX];
+ static char *buf;
+ static size_t buf_len;
+
+ if (buf == NULL) {
+ buf_len = _Q_PATH_MAX;
+ buf = xmalloc(buf_len);
+ }
- snprintf(buf, sizeof(buf), "%s%s/%s/%s/%s", portroot, portvdb,
CATEGORY, PF, item);
- eat_file(buf, buf, sizeof(buf));
+ snprintf(buf, buf_len, "%s%s/%s/%s/%s", portroot, portvdb, CATEGORY,
PF, item);
+ eat_file(buf, &buf, &buf_len);
rmspace(buf);
return buf;
}
+/* TODO: Merge this into libq/vdb.c somehow. */
_q_static queue *get_vdb_atoms(int fullcpv)
{
q_vdb_ctx *ctx;
@@ -1232,6 +1268,7 @@ _q_static queue *get_vdb_atoms(int fullcpv)
char buf[_Q_PATH_MAX];
char slot[_Q_PATH_MAX];
+ size_t slot_len;
struct dirent **cat;
struct dirent **pf;
@@ -1255,11 +1292,12 @@ _q_static queue *get_vdb_atoms(int fullcpv)
if ((atom = atom_explode(buf)) == NULL)
continue;
- slot[0] = '0';
- slot[1] = 0;
+ /* XXX: This assumes static slot buf is big enough, but
should be fine
+ * until this is rewritten & merged into libq/vdb.c. */
+ slot_len = sizeof(slot);
strncat(buf, "/SLOT", sizeof(buf));
- eat_file_at(ctx->vdb_fd, buf, buf, sizeof(buf));
- rmspace(buf);
+ eat_file_at(ctx->vdb_fd, buf, (char **)&slot,
&slot_len);
+ rmspace(slot);
if (fullcpv) {
if (atom->PR_int)
diff --git a/qdepends.c b/qdepends.c
index eba8304..8a176b6 100644
--- a/qdepends.c
+++ b/qdepends.c
@@ -396,7 +396,8 @@ _q_static int qdepends_main_vdb_cb(q_vdb_pkg_ctx *pkg_ctx,
void *priv)
int i;
char *ptr;
char buf[_Q_PATH_MAX];
- char depend[65536], use[8192];
+ static char *depend, *use;
+ static size_t depend_len, use_len;
dep_node *dep_tree;
/* see if this cat/pkg is requested */
@@ -412,7 +413,7 @@ _q_static int qdepends_main_vdb_cb(q_vdb_pkg_ctx *pkg_ctx,
void *priv)
IF_DEBUG(warn("matched %s/%s", catname, pkgname));
- if (!eat_file_at(pkg_ctx->fd, state->depend_file, depend,
sizeof(depend)))
+ if (!eat_file_at(pkg_ctx->fd, state->depend_file, &depend, &depend_len))
return 0;
IF_DEBUG(warn("growing tree..."));
@@ -433,7 +434,7 @@ _q_static int qdepends_main_vdb_cb(q_vdb_pkg_ctx *pkg_ctx,
void *priv)
printf("%s%s/%s%s%s: ", BOLD, catname, BLUE, pkgname, NORM);
}
- if (!eat_file_at(pkg_ctx->fd, "USE", use, sizeof(use))) {
+ if (!eat_file_at(pkg_ctx->fd, "USE", &use, &use_len)) {
warn("Could not eat_file(%s), you'll prob have incorrect
output", buf);
} else {
for (ptr = use; *ptr; ++ptr)
@@ -466,12 +467,13 @@ _q_static int qdepends_vdb_deep_cb(q_vdb_pkg_ctx
*pkg_ctx, void *priv)
size_t len;
char *ptr;
char buf[_Q_PATH_MAX];
- char depend[16384], use[8192];
+ static char *depend, *use;
+ static size_t depend_len, use_len;
dep_node *dep_tree;
IF_DEBUG(warn("matched %s/%s for %s", catname, pkgname,
state->depend_file));
- if (!eat_file_at(pkg_ctx->fd, state->depend_file, depend,
sizeof(depend)))
+ if (!eat_file_at(pkg_ctx->fd, state->depend_file, &depend, &depend_len))
return 0;
IF_DEBUG(warn("growing tree..."));
@@ -481,7 +483,7 @@ _q_static int qdepends_vdb_deep_cb(q_vdb_pkg_ctx *pkg_ctx,
void *priv)
IF_DEBUG(puts(depend));
IF_DEBUG(dep_dump_tree(dep_tree));
- if (eat_file_at(pkg_ctx->fd, "USE", use, sizeof(use)) == 1)
+ if (eat_file_at(pkg_ctx->fd, "USE", &use, &use_len))
use[0] = ' ';
for (ptr = use; *ptr; ++ptr)
diff --git a/qfile.c b/qfile.c
index 174cb7d..7094034 100644
--- a/qfile.c
+++ b/qfile.c
@@ -114,7 +114,7 @@ _q_static int qfile_cb(q_vdb_pkg_ctx *pkg_ctx, void *priv)
}
if (state->exclude_slot == NULL)
goto qlist_done; /* "(CAT/)?(PN|PF)" matches, and no
SLOT specified */
- eat_file_at(pkg_ctx->fd, "SLOT", state->buf, state->buflen);
+ eat_file_at(pkg_ctx->fd, "SLOT", &state->buf, &state->buflen);
rmspace(state->buf);
if (strcmp(state->exclude_slot, state->buf) == 0)
goto qlist_done; /* "(CAT/)?(PN|PF):SLOT" matches */
@@ -215,8 +215,11 @@ _q_static int qfile_cb(q_vdb_pkg_ctx *pkg_ctx, void *priv)
}
}
if (state->slotted) {
- eat_file_at(pkg_ctx->fd, "SLOT",
slot+1, sizeof(slot)-1);
- rmspace(slot+1);
+ /* XXX: This assumes the buf is big
enough. */
+ char *slot_hack = slot + 1;
+ size_t slot_len = sizeof(slot) - 1;
+ eat_file_at(pkg_ctx->fd, "SLOT",
&slot_hack, &slot_len);
+ rmspace(slot_hack);
slot[0] = ':';
} else
slot[0] = '\0';
diff --git a/qlist.c b/qlist.c
index 5f8720f..cfb9d98 100644
--- a/qlist.c
+++ b/qlist.c
@@ -63,9 +63,10 @@ _q_static queue *filter_dups(queue *sets)
_q_static char *q_vdb_pkg_eat(q_vdb_pkg_ctx *pkg_ctx, const char *item)
{
- static char buf[_Q_PATH_MAX];
+ static char *buf;
+ static size_t buf_len;
- eat_file_at(pkg_ctx->fd, item, buf, sizeof(buf));
+ eat_file_at(pkg_ctx->fd, item, &buf, &buf_len);
rmspace(buf);
return buf;
diff --git a/qlop.c b/qlop.c
index 21e272d..d5e2226 100644
--- a/qlop.c
+++ b/qlop.c
@@ -385,7 +385,8 @@ void show_current_emerge(void)
DIR *proc;
struct dirent *de;
pid_t pid;
- char buf[BUFSIZE], bufstat[300];
+ static char *cmdline, *bufstat;
+ static size_t cmdline_len, bufstat_len;
char path[50];
char *p, *q;
unsigned long long start_time = 0;
@@ -407,17 +408,17 @@ void show_current_emerge(void)
/* portage renames the cmdline so the package name is first */
snprintf(path, sizeof(path), "/proc/%i/cmdline", pid);
- if (!eat_file(path, buf, sizeof(buf)))
+ if (!eat_file(path, &cmdline, &cmdline_len))
continue;
- if (buf[0] == '[' && (p = strchr(buf, ']')) != NULL &&
strstr(buf, "sandbox") != NULL) {
+ if (cmdline[0] == '[' && (p = strchr(cmdline, ']')) != NULL &&
strstr(cmdline, "sandbox") != NULL) {
*p = '\0';
- p = buf+1;
+ p = cmdline + 1;
q = p + strlen(p) + 1;
/* open the stat file to figure out how long we have
been running */
snprintf(path, sizeof(path), "/proc/%i/stat", pid);
- if (!eat_file(path, bufstat, sizeof(bufstat)))
+ if (!eat_file(path, &bufstat, &bufstat_len))
continue;
/* ripped from procps/proc/readproc.c */
@@ -435,7 +436,7 @@ void show_current_emerge(void)
"%llu ",
&start_time);
/* get uptime */
- if (!eat_file("/proc/uptime", bufstat, sizeof(bufstat)))
+ if (!eat_file("/proc/uptime", &bufstat, &bufstat_len))
continue;
sscanf(bufstat, "%lf", &uptime_secs);
diff --git a/qmerge.c b/qmerge.c
index 1e07347..165caa9 100644
--- a/qmerge.c
+++ b/qmerge.c
@@ -754,7 +754,9 @@ pkg_merge(int level, const depend_atom *atom, const struct
pkg_t *pkg)
{
queue *objs;
FILE *fp, *contents;
- char buf[1024], phases[128];
+ static char *phases;
+ static size_t phases_len;
+ char buf[1024];
char *tbz2, *p, *D, *T;
int i;
char **ARGV;
@@ -891,7 +893,7 @@ pkg_merge(int level, const depend_atom *atom, const struct
pkg_t *pkg)
xsystem(buf);
fflush(stdout);
- eat_file("vdb/DEFINED_PHASES", phases, sizeof(phases));
+ eat_file("vdb/DEFINED_PHASES", &phases, &phases_len);
pkg_run_func("vdb", phases, "pkg_pretend", D, T);
pkg_run_func("vdb", phases, "pkg_setup", D, T);
pkg_run_func("vdb", phases, "pkg_preinst", D, T);
@@ -1019,7 +1021,9 @@ _q_static int
pkg_unmerge(const char *cat, const char *pkgname, queue *keep)
{
size_t buflen;
- char *buf, *vdb_path, *T, phases[128];
+ static char *phases;
+ static size_t phases_len;
+ char *buf, *vdb_path, *T;
FILE *fp;
int ret, fd, vdb_fd, portroot_fd;
int cp_argc, cpm_argc;
@@ -1061,7 +1065,7 @@ pkg_unmerge(const char *cat, const char *pkgname, queue
*keep)
/* First execute the pkg_prerm step */
if (!pretend) {
- eat_file_at(vdb_fd, "DEFINED_PHASES", phases, sizeof(phases));
+ eat_file_at(vdb_fd, "DEFINED_PHASES", &phases, &phases_len);
mkdir_p(T, 0755);
pkg_run_func(vdb_path, phases, "pkg_prerm", T, T);
}
