revised patch attached. Not tested, but I've convinced myself it does
the same thing. Feel free to prune my verbose comments...
-dmc
Douglas McClendon wrote:
I agree with all of these. Revised patch forthcoming.
-dmc
Jeremy Katz wrote:
On Mon, 2007-09-17 at 12:37 -0500, Douglas McClendon wrote:
diff -Naur
livecd.3.resize2fsToMinimal_implicitsize/creator/livecd-creator
livecd.turboliveinst/creator/livecd-creator
---
livecd.3.resize2fsToMinimal_implicitsize/creator/livecd-creator
2007-09-17 17:09:57.000000000 +0000
+++ livecd.turboliveinst/creator/livecd-creator 2007-09-14
17:40:21.000000000 +0000 + def genMinInstDelta(self):'
[snip]
+ # associate os image with loop device
+ osloop = LoopbackMount("%s/data/os.img" %(self.build_dir,),
+ "not_going_to_actually_get_mounted")
Maybe just use None here instead of that. But nitpicky and not a big
deal
+ # calculate how much delta data to keep
[snip]
+ try:
+ minInstDeltaDataLength =
int((dmsetupOutput.split()[3]).split('/')[0])
Ugh, depending on dmsetup's output to not change scares me. But I don't
see any alternative. Probably worth giving an example output of what's
being split apart just so that when it _does_ change, it's easier to
figure out what we were using.
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/creator/mayflower
livecd.turboliveinst/creator/mayflower
--- livecd.3.resize2fsToMinimal_implicitsize/creator/mayflower
2007-09-14 05:49:42.000000000 +0000
+++ livecd.turboliveinst/creator/mayflower 2007-09-14
18:28:20.000000000 +0000
@@ -625,6 +625,24 @@
mount -n -o ro,remount /sysroot
}
+modprobe loop max_loop=128
+
+# we might have a genMinInstDelta delta file for anaconda to take
advantage of
+if [ -e /sysroot/LiveOS/osmin.gz ]; then
+ mknod /dev/loop118 b 7 118
+ # note: osmin.gz should typically only be about 7kb. + dd
if=/sysroot/LiveOS/osmin.gz of=/osmin.gz bs=512 2> /dev/null
+ # pad to at least next sector boundry
+ dd if=/dev/zero of=/osmin.gz bs=512 count=1 oflag=append
conv=notrunc 2> /dev/null
+ losetup /dev/loop118 /osmin.gz
+elif [ -e /sysroot/osmin.gz ] ; then
+ mknod /dev/loop118 b 7 118
+ dd if=/sysroot/osmin.gz of=/osmin.gz bs=512 2> /dev/null
+ # pad to at least next sector boundry
+ dd if=/dev/zero of=/osmin.gz bs=512 count=1 oflag=append
conv=notrunc 2> /dev/null
+ losetup /dev/loop118 /osmin.gz
+fi
Better to check for the existence of the files and then set $MINIMG or
similar. That way, the rest of the check can avoid suffering cut and
paste problems
Jeremy
--
Fedora-livecd-list mailing list
Fedora-livecd-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-livecd-list
--
Fedora-livecd-list mailing list
Fedora-livecd-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-livecd-list
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/creator/isotostick.sh livecd.turboliveinst.revised/creator/isotostick.sh
--- livecd.3.resize2fsToMinimal_implicitsize/creator/isotostick.sh 2007-09-14 00:49:42.000000000 -0500
+++ livecd.turboliveinst.revised/creator/isotostick.sh 2007-09-14 11:00:12.000000000 -0500
@@ -195,6 +195,10 @@
elif [ -f $CDMNT/ext3fs.img ]; then
cp $CDMNT/ext3fs.img $USBMNT/LiveOS/ext3fs.img || exitclean
fi
+if [ -f $CDMNT/osmin.gz ]; then
+ cp $CDMNT/osmin.gz $USBMNT/LiveOS/osmin.gz || exitclean
+fi
+
cp $CDMNT/isolinux/* $USBMNT/$SYSLINUXPATH
echo "Updating boot config file"
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/creator/livecd-creator livecd.turboliveinst.revised/creator/livecd-creator
--- livecd.3.resize2fsToMinimal_implicitsize/creator/livecd-creator 2007-09-17 12:09:57.000000000 -0500
+++ livecd.turboliveinst.revised/creator/livecd-creator 2007-09-18 01:46:02.000000000 -0500
@@ -343,6 +343,7 @@
self.image_size = 4096 # in megabytes
self.blocksize = 4096 # in kilobytes
+ self.minimized_image_size = 0 # in kilobytes
def _getRequiredPackages(self):
return []
@@ -934,7 +935,7 @@
for line in output.split("\n"):
if line.startswith(field + ":"):
return line[len(field) + 1:].strip()
-
+
raise KeyError("Failed to find field '%s' in output" % field)
def getBlockCountOfExt2FS(self, filesystem):
@@ -942,7 +943,7 @@
stdout=subprocess.PIPE,
stderr=open('/dev/null', 'w')
).communicate()[0]
-
+
return int(self.parseField(output, "Block count"))
def resize2fs(self, image, n_blocks):
@@ -990,10 +991,113 @@
os.ftruncate(fd, min_blocks * self.blocksize)
os.close(fd)
- print >> sys.stderr, "Installation target minimized to %dK" % (min_blocks * self.blocksize / 1024L)
+ self.minimized_image_size = min_blocks * self.blocksize / 1024L
+ print >> sys.stderr, "Installation target minimized to %dK" % (self.minimized_image_size)
self.resize2fs(image, n_blocks)
+
+ #
+ # genMinInstDelta: generates an osmin overlay file to sit alongside
+ # os.img. liveinst may then detect the existence of
+ # osmin, and use it to create a minimized os.img
+ # which can be installed more quickly, and to smaller
+ # destination volumes.
+ #
+ def genMinInstDelta(self):
+ # create the sparse file for the minimized overlay
+ fd = os.open("%s/out/osmin" %(self.build_dir,),
+ os.O_WRONLY | os.O_CREAT)
+ off = long(64L * 1024L * 1024L)
+ os.lseek(fd, off, 0)
+ os.write(fd, '\x00')
+ os.close(fd)
+
+ # associate os image with loop device
+ osloop = LoopbackMount("%s/data/os.img" %(self.build_dir,), "None")
+ osloop.loopsetup()
+
+ # associate overlay with loop device
+ minloop = LoopbackMount("%s/out/osmin" %(self.build_dir,), "None")
+ minloop.loopsetup()
+
+ # create a snapshot device
+ rc = subprocess.call(["/sbin/dmsetup",
+ "--table",
+ "0 %d snapshot %s %s p 8"
+ %(self.image_size * 1024L * 2L,
+ osloop.loopdev, minloop.loopdev),
+ "create",
+ "livecd-creator-%d" %(os.getpid(),) ])
+ if rc != 0:
+ raise InstallationError("Could not create genMinInstDelta snapshot device")
+ # resize snapshot device back to minimal (self.minimized_image_size)
+ rc = subprocess.call(["/sbin/resize2fs",
+ "/dev/mapper/livecd-creator-%d" %(os.getpid(),),
+ "%dK" %(self.minimized_image_size,)])
+ if rc != 0:
+ raise InstallationError("Could not shrink ext3fs image")
+
+ # calculate how much delta data to keep
+ dmsetupOutput = subprocess.Popen(['/sbin/dmsetup', 'status',
+ "livecd-creator-%d" %(os.getpid(),)],
+ stdout=subprocess.PIPE,
+ stderr=open('/dev/null', 'w')
+ ).communicate()[0]
+
+ # note: It is unknown how static the format of dmsetup status is.
+ # Perhaps a long term solution would be to look at the C code
+ # for dmsetup, and write a special tool, also linked against
+ # libdevmapper which can extract this specific data in a more
+ # bulletproof way.
+ #
+ # The format for dmsetup status on a snapshot device that we are
+ # counting on here is as follows.
+ # e.g. "0 8388608 snapshot 416/1048576" or "A B snapshot C/D"
+ #
+ # A is (I think*) the location of this snapshot-data-segment's data
+ # in the device, as an offset in 512-byte sectors from the beginning
+ # of the device.
+ #
+ # B is the length in sectors of data this snapshot-data-segment
+ # provides
+ #
+ # C is the number of sectors of data in use by the Copy-On-Write
+ # device. I assume(*) these are packed without holes in the front
+ # of the device.
+ #
+ # D is the number of sectors available from the Copy-On-Write device
+ #
+ # (*) man dmsetup and the documentation in
+ # /usr/share/doc/kernel-doc-*/Documentation/device-mapper/*
+ # could be much more thorough than they currently are.
+
+ try:
+ minInstDeltaDataLength = int((dmsetupOutput.split()[3]).split('/')[0])
+ print >> sys.stderr, "genMinInstDelta data length is %d 512 byte sectors" % (minInstDeltaDataLength)
+ except ValueError:
+ raise InstallationError("Could not calculate amount of data used by genMinInstDelta")
+
+ # tear down snapshot and loop devices
+ rc = subprocess.call(["/sbin/dmsetup", "remove",
+ "livecd-creator-%d" %(os.getpid(),) ])
+ if rc != 0:
+ raise InstallationError("Could not remove genMinInstDelta snapshot device")
+ osloop.lounsetup()
+ minloop.lounsetup()
+
+ # truncate the unused excess portion of the sparse file
+ fd = os.open("%s/out/osmin" %(self.build_dir,), os.O_WRONLY )
+ os.ftruncate(fd, minInstDeltaDataLength * 512)
+ os.close(fd)
+
+ # the delta data is *extremely* compressible (e.g. 1.2M->7kb)
+ rc = subprocess.call(["/usr/bin/gzip", "osmin"],
+ cwd="%s/out" %(self.build_dir,),
+ env={"PWD": "%s/out" %(self.build_dir,)})
+ if rc != 0:
+ raise InstallationError("Could not compress genMinInstDelta data")
+
def package(self):
self.createSquashFS()
self.createIso()
@@ -1316,7 +1420,6 @@
parser.add_option("-s", "--skip-compression", action="store_true", dest="skip_compression",
help=optparse.SUPPRESS_HELP)
-
(options, args) = parser.parse_args()
if not options.kscfg or not os.path.isfile(options.kscfg):
raise Usage("Kickstart config '%s' does not exist" %(options.kscfg,))
@@ -1388,7 +1491,10 @@
target.cleanupDeleted()
+ target.genMinInstDelta()
+
target.package()
+
except InstallationError, e:
print >> sys.stderr, "Error creating Live CD : %s" % e
target.teardown()
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/creator/mayflower livecd.turboliveinst.revised/creator/mayflower
--- livecd.3.resize2fsToMinimal_implicitsize/creator/mayflower 2007-09-14 00:49:42.000000000 -0500
+++ livecd.turboliveinst.revised/creator/mayflower 2007-09-18 01:48:03.000000000 -0500
@@ -625,6 +625,26 @@
mount -n -o ro,remount /sysroot
}
+modprobe loop max_loop=128
+
+# we might have a genMinInstDelta delta file for anaconda to take advantage of
+if [ -e /sysroot/LiveOS/osmin.gz ]; then
+ osminfile=/sysroot/LiveOS/osmin.gz
+elif [ -e /sysroot/osmin.gz ]; then
+ osminfile=/sysroot/osmin.gz
+else
+ osminfile=none
+fi
+
+if [ -e $osminfile ]; then
+ mknod /dev/loop118 b 7 118
+ dd if=${osminfile} of=/osmin.gz bs=512 2> /dev/null
+ # pad to at least next sector boundry
+ dd if=/dev/zero of=/osmin.gz bs=512 count=1 oflag=append conv=notrunc 2> /dev/null
+ losetup /dev/loop118 /osmin.gz
+fi
+
+
# we might have an uncompressed embedded ext3 to use as rootfs (uncompressed live)
#
if [ -e /sysroot/LiveOS/ext3fs.img ]; then
@@ -638,13 +658,11 @@
echo "setting up embedded ext3 fs "
fi
- mknod /dev/loop118 b 7 118
mknod /dev/loop119 b 7 119
mknod /dev/loop120 b 7 120
mknod /dev/loop121 b 7 121
mkdir -p /dev/mapper
mknod /dev/mapper/control c 10 63
- modprobe loop max_loop=128
modprobe dm_snapshot
losetup /dev/loop121 \$EXT3FS
@@ -667,13 +685,11 @@
echo "setting up embedded squash -> ext3 fs "
fi
- mknod /dev/loop118 b 7 118
mknod /dev/loop119 b 7 119
mknod /dev/loop120 b 7 120
mknod /dev/loop121 b 7 121
mkdir -p /dev/mapper
mknod /dev/mapper/control c 10 63
- modprobe loop max_loop=128
modprobe dm_snapshot
if [ "\$live_ram" == "1" ] ; then
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/README livecd.turboliveinst.revised/README
--- livecd.3.resize2fsToMinimal_implicitsize/README 2007-09-17 11:47:31.000000000 -0500
+++ livecd.turboliveinst.revised/README 2007-09-14 12:13:34.000000000 -0500
@@ -74,6 +74,13 @@
o Unmounts the installation root
+ o Runs resize2fs to minimize and unminimize the ext3 file to remove data
+ from deleted files
+
+ o Runs resize2fs to minimize on a devicemapper snapshot, to generate a
+ small minimized delta image file which can be used by anaconda to
+ reduce installation time by not copying unused data to disk
+
o Creates a squashfs file system containing only the ext3 file (compression)
o Configures the boot loader
--
Fedora-livecd-list mailing list
Fedora-livecd-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-livecd-list
