I wanted to benchmark a http-connection using fetch in a
shell-script. I came to realize that fetch doesn't support
compressed streams when I force-fed it the compressed stream by
ignoring the absence of Accept-Encoding. This doesn't influence my
benchmarking, but I think fetch should have this feature.
I hacked together deflate support for the http part of libfetch.
Next on my todo list is proper error handling, gzip support, code
clean up and general code clean up in http.c (in order of priority).
I'd love to get some feedback, do you consider this useful? Does it
work on your system? Would there be a chance of getting the
finalized version into SVN?
The attached patch applies to RELENG_7. Probably everywhere else,
too. Because I think libfetch development has been at a halt for
some time.
Regards,
Dominic
--- src/lib/libfetch/http.c.orig2008-06-29 15:28:58.0 +0200
+++ src/lib/libfetch/http.c 2008-06-30 19:38:57.0 +0200
@@ -75,6 +75,7 @@
#include
#include
#include
+#include
#include
#include
@@ -105,7 +106,6 @@
#define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599)
-
/*
* I/O functions for decoding chunked streams
*/
@@ -126,6 +126,16 @@
#endif
};
+struct zlibio
+{
+ FILE*source;/* the http connection to read from */
+ z_stream*stream;/* the zlib stream to read the */
+ /* uncompressed data from */
+ charin[65536]; /* read buffer */
+};
+
+typedef FILE * (*funopen_function)(conn_t *, int);
+
/*
* Get next chunk header
*/
@@ -302,10 +312,50 @@
}
/*
+ * Read function for deflate compressed data.
+ */
+static int
+http_inflate_readfn(void *v, char *buf, int len)
+{
+ struct zlibio *io = (struct zlibio *)v;
+ int status;
+
+ /* Only read if the last read chunk has completely been flushed. */
+ if (io->stream->avail_in == 0) {
+ io->stream->avail_in = fread(io->in, sizeof(char),
sizeof(io->in), io->source);
+
+ /* Forward errors and eof */
+ if (io->stream->avail_in <= 0)
+ return io->stream->avail_in;
+
+ io->stream->next_in = io->in;
+ }
+
+ io->stream->avail_out = len;
+ io->stream->next_out = buf;
+ status = inflate(io->stream, Z_SYNC_FLUSH);
+
+ return (len - io->stream->avail_out);
+}
+
+/*
+ * Close function for deflate compressed data
+ */
+static int
+http_inflate_closefn(void *v)
+{
+ struct zlibio *io = (struct zlibio *)v;
+
+ (void)inflateEnd(io->stream);
+ free(io->stream);
+ return (fclose(io->source));
+}
+
+/*
* Wrap a file descriptor up
*/
static FILE *
-http_funopen(conn_t *conn, int chunked)
+http_funopen_raw(conn_t *conn, int chunked)
{
struct httpio *io;
FILE *f;
@@ -316,7 +366,7 @@
}
io->conn = conn;
io->chunked = chunked;
- f = funopen(io, http_readfn, http_writefn, NULL, http_closefn);
+ f = funopen(io, &http_readfn, &http_writefn, NULL, &http_closefn);
if (f == NULL) {
fetch_syserr();
free(io);
@@ -325,6 +375,50 @@
return (f);
}
+/*
+ * Wrap a file descriptor up around the zlip inflate command
+ */
+static FILE *
+http_funopen_inflate(conn_t *conn, int chunked)
+{
+ struct zlibio *io;
+ FILE *f;
+
+ if ((io = calloc(1, sizeof(*io))) == NULL) {
+ fetch_syserr();
+ return (NULL);
+ }
+
+ io->source = http_funopen_raw(conn, chunked);
+
+ if ((io->stream = calloc(1, sizeof(*(io->stream == NULL) {
+ fetch_syserr();
+ free(io);
+ return (NULL);
+ }
+
+ io->stream->zalloc = Z_NULL;
+ io->stream->zfree = Z_NULL;
+ io->stream->opaque = Z_NULL;
+ io->stream->avail_in = 0;
+ io->stream->next_in = Z_NULL;
+ if (inflateInit2(io->stream, -MAX_WBITS) != Z_OK) {
+ fetch_syserr();
+ free(io->source);
+ free(io);
+ return (NULL);
+ }
+
+ f = funopen(io, &http_inflate_readfn, NULL, NULL,
&http_inflate_closefn);
+
+ if (f == NULL) {
+ fetch_syserr();
+ free(io->source);
+ free(io);
+ return (NULL);
+ }
+ return f;
+}
/*
* Helper functions for talking to the server and parsing its replies
@@ -336,6 +430,7 @@
hdr_error = -1,
hdr_end = 0,
hdr_unknown = 1,
+ hdr_content_encoding,
hdr_content_length,
hdr_content_range,
hdr_last_modified,
@@ -349,6 +444,7 @@
hdr_tnum;
const char *name;
} hdr_names[] = {
+ { hdr_con
