--- Begin Message ---
Package: cloop-src
Version: 0.66-1
Severity: minor
Tags: patch
The create_compressed_fs command needs huge amounts of virtual memory when
creating large images since the entire compressed image is kept in memory
and written to outupt file only at the end of the compression stage.
This is very inefficient and (IMHO) is also sign of bad coding practice.
For example when creating a 650MB cloop image the program needs more than
650MB of virtual memory. On systems with less RAM a large swap can be used
instead, but also this is very inefficient because the image is written to
swap, then read back and written again to file.
The problem can be easily solved if input and output files are seekable,
i.e. are regular files istead of stdin/stdout pipes.
The patch included in the attachment allows creating large compressed images
using very little virtual memory. To do this the input file must be a real
file and the output file must be specified in the third optional argument:
$ mkisofs -r -o uncompressed.iso /root_dir
$ create_compressed_fs uncompressed.iso 65536 compressed.cloop
On the other hand the entire uncompressed image must be written to file,
but at least this doesn't need large ram or swap space and works better on
systems with limited memory.
--
Massimo Dal Zotto <[EMAIL PROTECTED]>
--- create_compressed_fs.c.orig Thu Mar 21 00:13:01 2002
+++ create_compressed_fs.c Wed Oct 9 23:03:07 2002
@@ -9,6 +9,8 @@
* - Changed Preamble.
* * Sat Jul 28 2001 Klaus Knopper <[EMAIL PROTECTED]>
* - cleanup and gcc 2.96 / glibc checking
+ * * Wed Oct 09 2002 Massimo Dal Zotto <[EMAIL PROTECTED]>
+ * - don't keep compressed image in memory if output file is seekable
*/
#include <stdio.h>
@@ -25,180 +27,212 @@
#define CLOOP_PREAMBLE "#!/bin/sh\n" "insmod cloop.o file=$0 && mount -r -t
iso9660 /dev/cloop $1\n" "exit $?\n"
+static int can_seek = 0;
+static int out = -1;
+
struct cb_list
{
- struct cb_list *next;
- size_t size;
- char data[0];
+ struct cb_list *next;
+ size_t size;
+ char data[0];
};
void free_cb_list(struct cb_list *cbl)
{
- if(cbl->next) free_cb_list(cbl->next);
- free(cbl);
+ if(cbl->next) free_cb_list(cbl->next);
+ free(cbl);
}
/* Now using the goto style because it is quicker to read */
static struct cb_list *create_compressed_blocks(int handle, unsigned long
- blocksize, unsigned long *numblocks)
+ blocksize, unsigned long
*numblocks)
{
- struct cb_list *cbl,**cbp=&cbl;
- unsigned long i=0;
- unsigned int last;
- unsigned long long total_uncompressed=0,total_compressed=0;
- unsigned long maxlen = blocksize + blocksize/1000 + 12;
- char *compressed, *uncompressed;
- if((uncompressed=malloc(blocksize))==NULL)
- {
- fprintf(stderr, "*** Can't malloc(%ld).\n",blocksize);
- return NULL;
- }
- if((compressed=malloc(maxlen))==NULL)
- {
- fprintf(stderr, "*** Can't malloc(%ld).\n",blocksize);
- goto free_uncompressed;
- }
- for(i=0,last=0; !last; i++)
- {
- int z_error;
- unsigned long total=0, len = maxlen;
- memset(compressed,0,len); memset(uncompressed,0,blocksize);
- while(total<blocksize) /* Read a complete block */
- {
- ssize_t r=read(handle, uncompressed+total, blocksize-total);
- if(r<=0) { last=1; break; }
- total+=r;
- }
- total_uncompressed += total;
- if (total != blocksize)
- {
- last=1;
- fprintf(stderr, "Partial read (%lu bytes of %lu), padding with zeros.\n",
- total, blocksize);
- }
-/* BEST_COMPRESSION does not seem to improve compression at a blocksize */
-/* of 64k or less, but is significantly slower. KK. */
- if((z_error=compress2(compressed, &len, uncompressed, blocksize,
Z_DEFAULT_COMPRESSION)) != Z_OK)
- {
- fprintf(stderr, "*** Error %d compressing block %lu! (compressed=%p,
len=%lu, uncompressed=%p, blocksize=%lu)\n", z_error, i,
compressed,len,uncompressed,blocksize);
- goto error_free_cb_list;
- }
- if((*cbp = malloc(sizeof(struct cb_list)+len))==NULL) /* get another block
*/
- {
- fprintf(stderr, "*** Out of memory allocating block ptrs (virtual memory
exhausted).\n");
- goto error_free_cb_list;
- }
- total_compressed+=len;
- /* Print status */
- fprintf(stderr, "Block# %5lu size %6lu -> %6lu [compression ratio %3lu%%,
overall: %3Lu%%]\n", i, total, len,
total>0?((len*100)/total):100,total_uncompressed>0?((total_compressed*100)/total_uncompressed):100);
- (*cbp)->size = len;
- memcpy((*cbp)->data, compressed, len);
- (*cbp)->next=NULL;
- cbp=&((*cbp)->next);
- } /* for */
- goto free_compressed;
+ struct cb_list *cbl=NULL,**cbp=&cbl;
+ unsigned long i=0;
+ unsigned int last;
+ unsigned long long total_uncompressed=0,total_compressed=0;
+ unsigned long maxlen = blocksize + blocksize/1000 + 12;
+ char *compressed, *uncompressed;
+
+ if((uncompressed=malloc(blocksize))==NULL) {
+ fprintf(stderr, "*** Can't malloc(%ld).\n",blocksize);
+ return NULL;
+ }
+ if((compressed=malloc(maxlen))==NULL) {
+ fprintf(stderr, "*** Can't malloc(%ld).\n",blocksize);
+ goto free_uncompressed;
+ }
+
+ for(i=0,last=0; !last; i++) {
+ int z_error;
+ unsigned long total=0, len = maxlen;
+ memset(compressed,0,len); memset(uncompressed,0,blocksize);
+ /* Read a complete block */
+ while(total<blocksize) {
+ ssize_t r=read(handle, uncompressed+total, blocksize-total);
+ if(r<=0) { last=1; break; }
+ total+=r;
+ }
+ total_uncompressed += total;
+ if (total != blocksize) {
+ last=1;
+ if (total == 0) break;
+ fprintf(stderr, "Partial read (%lu bytes of %lu), padding with
zeros.\n",
+ total, blocksize);
+ }
+ /* BEST_COMPRESSION does not seem to improve compression at a blocksize
*/
+ /* of 64k or less, but is significantly slower. KK.
*/
+ if((z_error=compress2(compressed, &len, uncompressed, blocksize,
Z_DEFAULT_COMPRESSION)) != Z_OK) {
+ fprintf(stderr, "*** Error %d compressing block %lu!
(compressed=%p, len=%lu, uncompressed=%p, blocksize=%lu)\n", z_error, i,
compressed,len,uncompressed,blocksize);
+ goto error_free_cb_list;
+ }
+ /* get another block */
+ if((*cbp = malloc(sizeof(struct cb_list)+(can_seek?0:len)))==NULL) {
+ fprintf(stderr, "*** Out of memory allocating block ptrs (virtual
memory exhausted).\n");
+ goto error_free_cb_list;
+ }
+ total_compressed+=len;
+ /* Print status */
+ fprintf(stderr, "Block# %5lu size %6lu -> %6lu [compression ratio
%3lu%%, overall: %3Lu%%]\n", i, total, len,
total>0?((len*100)/total):100,total_uncompressed>0?((total_compressed*100)/total_uncompressed):100);
+ (*cbp)->size = len;
+ (*cbp)->next=NULL;
+ if (can_seek) {
+ /* Write compressed block to output file */
+ if (write(out, compressed, len) != len) {
+ perror("writing block");
+ goto error_free_cb_list;
+ }
+ } else {
+ /* Save compressed block in cb_list */
+ memcpy((*cbp)->data, compressed, len);
+ }
+ cbp=&((*cbp)->next);
+ } /* for */
+ goto free_compressed;
- error_free_cb_list:
+error_free_cb_list:
if(cbl) { free_cb_list(cbl); cbl=NULL; i=0; }
- free_compressed:
+free_compressed:
free(compressed);
- free_uncompressed:
+free_uncompressed:
free(uncompressed);
-
- *numblocks=i;
- return cbl;
+
+ if (!(*numblocks)) *numblocks=i;
+ return cbl;
}
int main(int argc, char **argv)
{
- int in;
- unsigned long blocksize;
- struct cloop_head head;
- unsigned long numblocks;
- unsigned long bytes_so_far;
- unsigned long i;
- struct cb_list *compressed_blocks,*cbp;
-
- if (argc != 3)
- {
- fprintf(stderr, "Usage: %s filename blocksize(bytes).\n",argv[0]);
- fprintf(stderr, "Use '-' as filename for stdin.\n");
- return 1;
- }
-
- blocksize = atoi(argv[2]);
- if (blocksize == 0 || blocksize % 512 != 0)
- {
- fprintf(stderr, "*** Blocksize must be a multiple of 512.\n");
- return 1;
- }
-
- if (blocksize > MAX_KMALLOC_SIZE)
- {
- fprintf(stderr, "WARNING: Blocksize %lu may be too big for a kmalloc() (%lu
max).\n",blocksize,MAX_KMALLOC_SIZE);
- sleep(2);
- }
-
- if (sizeof(CLOOP_PREAMBLE) > CLOOP_HEADROOM)
- {
- fprintf(stderr, "*** Preamble (%u chars) > headroom (%u)\n",
- sizeof(CLOOP_PREAMBLE), CLOOP_HEADROOM);
- return 1;
- }
+ int in;
+ unsigned long blocksize;
+ struct cloop_head head;
+ unsigned long numblocks=0;
+ unsigned long bytes_so_far;
+ unsigned long i;
+ off_t size, pos=0;
+ struct cb_list *compressed_blocks,*cbp;
+
+ if ((argc < 3) || (argc > 4)) {
+ fprintf(stderr, "Usage: %s filename blocksize(bytes)
[outfile].\n",argv[0]);
+ fprintf(stderr, "Use '-' as filename for stdin.\n");
+ return 1;
+ }
+
+ blocksize = atoi(argv[2]);
+ if (blocksize == 0 || blocksize % 512 != 0) {
+ fprintf(stderr, "*** Blocksize must be a multiple of 512.\n");
+ return 1;
+ }
+
+ if (blocksize > MAX_KMALLOC_SIZE) {
+ fprintf(stderr, "WARNING: Blocksize %lu may be too big for a kmalloc()
(%lu max).\n",blocksize,MAX_KMALLOC_SIZE);
+ sleep(2);
+ }
+
+ if (sizeof(CLOOP_PREAMBLE) > CLOOP_HEADROOM) {
+ fprintf(stderr, "*** Preamble (%u chars) > headroom (%u)\n",
+ sizeof(CLOOP_PREAMBLE), CLOOP_HEADROOM);
+ return 1;
+ }
- in=strcmp(argv[1],"-")==0?dup(fileno(stdin)):open(argv[1], O_RDONLY);
+ in=strcmp(argv[1],"-")==0?dup(fileno(stdin)):open(argv[1], O_RDONLY);
+ if (in < 0) {
+ perror("Opening input");
+ return 1;
+ }
+
+ if ((argc >= 4) && (strcmp(argv[3],"-")!=0) &&
(strcmp(argv[3],"/dev/stdout")!=0)) {
+ out = open(argv[3], O_WRONLY|O_CREAT|O_TRUNC, 0644);
+ if (out < 0) {
+ perror("Opening output");
+ return 1;
+ }
+ /* Check if both input and output are seekable */
+ size = lseek(in, 0, SEEK_END); lseek(in, 0, SEEK_SET);
+ if (size > 0) {
+ numblocks = (size+blocksize-1)/blocksize;
+ pos = lseek(out, sizeof(head)+(numblocks+1)*sizeof(int), SEEK_SET);
+ if (pos > 0) {
+ can_seek = 1;
+ fprintf(stderr, "Allocating %ld+1 blocks, seek to %08lx
(%ld)\n", numblocks, pos, pos);
+ }
+ }
+ } else {
+ out = STDOUT_FILENO;
+ can_seek = 0;
+ }
+
+ compressed_blocks = create_compressed_blocks(in, blocksize, &numblocks);
+ if (!compressed_blocks) return 1;
+
+ close(in);
+
+ memset(head.preamble, 0, CLOOP_HEADROOM);
+ memcpy(head.preamble, CLOOP_PREAMBLE, sizeof(CLOOP_PREAMBLE));
+ head.block_size = htonl(blocksize);
+ head.num_blocks = htonl(numblocks);
+
+ fprintf(stderr, "Block size %lu, number of blocks %lu.\n",
+ blocksize, numblocks);
+
+ bytes_so_far = sizeof(head) + sizeof(u_int32_t) * (numblocks+1);
+ if(bytes_so_far > MAX_KMALLOC_SIZE) {
+ fprintf(stderr, "\nWARNING: cloop block index may not fit into a
kmalloc()\n*** (%lu bytes max, current %lu)\n\n", MAX_KMALLOC_SIZE,
bytes_so_far);
+ sleep(2);
+ }
+
+ /* Write out head... */
+ if (can_seek) {
+ lseek(out, 0, SEEK_SET);
+ }
+ write(out, &head, sizeof(head));
+
+ /* Write offsets */
+ for (i=0,cbp=compressed_blocks; i < numblocks+1; i++) {
+ u_int32_t tmp;
+ tmp = htonl(bytes_so_far);
+ write(out, &tmp, sizeof(tmp));
+ if(cbp) { bytes_so_far += cbp->size; cbp=cbp->next; }
+ }
+
+ if (can_seek) {
+ lseek (out, 0, SEEK_END);
+ goto exit;
+ }
+
+ /* Now write blocks and free them. */
+ for (i = 0, cbp=compressed_blocks; cbp && i < numblocks; i++) {
+ if (write(out, cbp->data, cbp->size) != cbp->size) {
+ perror("writing block");
+ free_cb_list(compressed_blocks);
+ return 1;
+ }
+ cbp=cbp->next;
+ free(compressed_blocks); compressed_blocks=cbp;
+ }
- if (in < 0)
- {
- perror("Opening input");
- return 1;
- }
-
- compressed_blocks = create_compressed_blocks(in, blocksize, &numblocks);
-
- close(in);
-
- memcpy(head.preamble, CLOOP_PREAMBLE, sizeof(CLOOP_PREAMBLE));
- head.block_size = htonl(blocksize);
- head.num_blocks = htonl(numblocks);
-
- fprintf(stderr, "Block size %lu, number of blocks %lu.\n",
- blocksize, numblocks);
-
- bytes_so_far = sizeof(head) + sizeof(u_int32_t) * (numblocks + 1);
- if(bytes_so_far > MAX_KMALLOC_SIZE)
- {
- fprintf(stderr, "\nWARNING: cloop block index may not fit into a
kmalloc()\n*** (%lu bytes max, current %lu)\n\n", MAX_KMALLOC_SIZE,
bytes_so_far);
- sleep(2);
- }
-
- /* Write out head... */
- write(STDOUT_FILENO, &head, sizeof(head));
-
- if (!compressed_blocks) return 1;
-
- /* Write offsets */
- for (i=0,cbp=compressed_blocks; i < numblocks+1; i++)
- {
- u_int32_t tmp;
- tmp = htonl(bytes_so_far);
- write(STDOUT_FILENO, &tmp, sizeof(tmp));
- if(cbp) { bytes_so_far += cbp->size; cbp=cbp->next; }
- }
-
- /* Now write blocks and free them. */
- for (i = 0, cbp=compressed_blocks; cbp && i < numblocks; i++)
- {
- if (write(STDOUT_FILENO, cbp->data, cbp->size) != cbp->size)
- {
- perror("writing block");
- free_cb_list(compressed_blocks);
- return 1;
- }
- cbp=cbp->next;
- free(compressed_blocks); compressed_blocks=cbp;
- }
- fprintf(stderr,"Done.\n");
- return 0;
+exit:
+ fprintf(stderr,"Done.\n");
+ return 0;
}
--- End Message ---
