Re: [PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
On Tue, Jul 7, 2020 at 3:21 PM Philippe Mathieu-Daudé wrote: > In the next commit we won't allow SD card images with invalid > size (not aligned to a power of 2). Prepare the tests: add the > pow2ceil() and image_pow2ceil_truncate() methods and truncate > the images of the tests using SD cards. > > Signed-off-by: Philippe Mathieu-Daudé > --- > tests/acceptance/boot_linux_console.py | 15 +++ > 1 file changed, 15 insertions(+) > > diff --git a/tests/acceptance/boot_linux_console.py > b/tests/acceptance/boot_linux_console.py > index 3d02519660..f4d4e3635f 100644 > --- a/tests/acceptance/boot_linux_console.py > +++ b/tests/acceptance/boot_linux_console.py > @@ -28,6 +28,18 @@ > except CmdNotFoundError: > P7ZIP_AVAILABLE = False > > +# round up to next power of 2 > +def pow2ceil(x): > +return 1 if x == 0 else 2**(x - 1).bit_length() > + > +# truncate file size to next power of 2 > +def image_pow2ceil_truncate(path): > +size = os.path.getsize(path) > +size_aligned = pow2ceil(size) > +if size != size_aligned: > +with open(path, 'ab+') as fd: > +fd.truncate(size_aligned) > + > class LinuxKernelTest(Test): > KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' > > @@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self): > rootfs_path_xz = self.fetch_asset(rootfs_url, > asset_hash=rootfs_hash) > rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') > archive.lzma_uncompress(rootfs_path_xz, rootfs_path) > +image_pow2ceil_truncate(rootfs_path) > > self.vm.set_console() > kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + > @@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self): > image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img' > image_path = os.path.join(self.workdir, image_name) > process.run("7z e -o%s %s" % (self.workdir, image_path_7z)) > +image_pow2ceil_truncate(image_path) > > self.vm.set_console() > self.vm.add_args('-drive', 'file=' + image_path + > ',if=sd,format=raw', > @@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self): > image_hash = '2babb29d36d8360adcb39c09e31060945259917a' > image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash) > image_path = os.path.join(self.workdir, 'armv7.img') > +image_pow2ceil_truncate(image_path) > image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + > image_path > archive.gzip_uncompress(image_path_gz, image_path) > > -- > 2.21.3 > > Hi Philippe, This patch works OK for the linux part, but the NetBSD didn't work, it prints this error: (5/5) tests/acceptance/boot_linux_console.py:BootLinuxConsole.test_arm_orangepi_uboot_netbsd9: ERROR: [Errno 2] No such file or directory: '/var/tmp/avocado_6hoo815w/avocado_job_40aayif8/ 5-tests_acceptance_boot_linux_console.py_BootLinuxConsole.test_arm_orangepi_uboot_netbsd9/armv7.img' (0.18 s) Basically the truncate should just be moved after the uncompress to fix it. And the lines that we use before to extend the image size can be removed now. That was needed to avoid conflict with the partition size inside image. So with these small changes, I got it working fine: diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py index f4d4e3635f..69607a5840 100644 --- a/tests/acceptance/boot_linux_console.py +++ b/tests/acceptance/boot_linux_console.py @@ -684,7 +684,7 @@ class BootLinuxConsole(LinuxKernelTest): :avocado: tags=machine:orangepi-pc """ -# This test download a 196MB compressed image and expand it to 932MB... +# This test download a 196MB compressed image and expand it to 1G image_url = ('https://dl.armbian.com/orangepipc/archive/' 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.7z') image_hash = '196a8ffb72b0123d92cea4a070894813d305c71e' @@ -725,7 +725,7 @@ class BootLinuxConsole(LinuxKernelTest): :avocado: tags=arch:arm :avocado: tags=machine:orangepi-pc """ -# This test download a 304MB compressed image and expand it to 1.3GB... +# This test download a 304MB compressed image and expand it to 2GB... deb_url = ('http://snapshot.debian.org/archive/debian/' '20200108T145233Z/pool/main/u/u-boot/' 'u-boot-sunxi_2020.01%2Bdfsg-1_armhf.deb') @@ -742,9 +742,9 @@ class BootLinuxConsole(LinuxKernelTest): image_hash = '2babb29d36d8360adcb39c09e31060945259917a' image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash) image_path = os.path.join(self.workdir, 'armv7.img') -image_pow2ceil_truncate(image_path) image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path archive.gzip_uncompress(image_path_gz, image_path) +image_pow2ceil_truncate(image_path) # dd if
Re: [PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
On 7/7/20 5:53 PM, Alistair Francis wrote: > n Tue, Jul 7, 2020 at 6:21 AM Philippe Mathieu-Daudé wrote: >> >> In the next commit we won't allow SD card images with invalid >> size (not aligned to a power of 2). Prepare the tests: add the >> pow2ceil() and image_pow2ceil_truncate() methods and truncate >> the images of the tests using SD cards. >> >> Signed-off-by: Philippe Mathieu-Daudé >> --- >> tests/acceptance/boot_linux_console.py | 15 +++ >> 1 file changed, 15 insertions(+) >> >> diff --git a/tests/acceptance/boot_linux_console.py >> b/tests/acceptance/boot_linux_console.py >> index 3d02519660..f4d4e3635f 100644 >> --- a/tests/acceptance/boot_linux_console.py >> +++ b/tests/acceptance/boot_linux_console.py >> @@ -28,6 +28,18 @@ >> except CmdNotFoundError: >> P7ZIP_AVAILABLE = False >> >> +# round up to next power of 2 >> +def pow2ceil(x): >> +return 1 if x == 0 else 2**(x - 1).bit_length() >> + >> +# truncate file size to next power of 2 >> +def image_pow2ceil_truncate(path): >> +size = os.path.getsize(path) >> +size_aligned = pow2ceil(size) >> +if size != size_aligned: >> +with open(path, 'ab+') as fd: >> +fd.truncate(size_aligned) > > Why truncate the image, can't we expand it instead? pow2ceil() round UP to the next power of 2. I think this Python truncate() is a simple wrapper around the ftruncate() syscall. IOW we only "expand the image". Note, these are test images copied in tmpdir and discarded after the tests ran. I'll improve the description. > > Alistair > >> + >> class LinuxKernelTest(Test): >> KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' >> >> @@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self): >> rootfs_path_xz = self.fetch_asset(rootfs_url, >> asset_hash=rootfs_hash) >> rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') >> archive.lzma_uncompress(rootfs_path_xz, rootfs_path) >> +image_pow2ceil_truncate(rootfs_path) >> >> self.vm.set_console() >> kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + >> @@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self): >> image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img' >> image_path = os.path.join(self.workdir, image_name) >> process.run("7z e -o%s %s" % (self.workdir, image_path_7z)) >> +image_pow2ceil_truncate(image_path) >> >> self.vm.set_console() >> self.vm.add_args('-drive', 'file=' + image_path + >> ',if=sd,format=raw', >> @@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self): >> image_hash = '2babb29d36d8360adcb39c09e31060945259917a' >> image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash) >> image_path = os.path.join(self.workdir, 'armv7.img') >> +image_pow2ceil_truncate(image_path) >> image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path >> archive.gzip_uncompress(image_path_gz, image_path) >> >> -- >> 2.21.3 >> >> >
Re: [PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
n Tue, Jul 7, 2020 at 6:21 AM Philippe Mathieu-Daudé wrote: > > In the next commit we won't allow SD card images with invalid > size (not aligned to a power of 2). Prepare the tests: add the > pow2ceil() and image_pow2ceil_truncate() methods and truncate > the images of the tests using SD cards. > > Signed-off-by: Philippe Mathieu-Daudé > --- > tests/acceptance/boot_linux_console.py | 15 +++ > 1 file changed, 15 insertions(+) > > diff --git a/tests/acceptance/boot_linux_console.py > b/tests/acceptance/boot_linux_console.py > index 3d02519660..f4d4e3635f 100644 > --- a/tests/acceptance/boot_linux_console.py > +++ b/tests/acceptance/boot_linux_console.py > @@ -28,6 +28,18 @@ > except CmdNotFoundError: > P7ZIP_AVAILABLE = False > > +# round up to next power of 2 > +def pow2ceil(x): > +return 1 if x == 0 else 2**(x - 1).bit_length() > + > +# truncate file size to next power of 2 > +def image_pow2ceil_truncate(path): > +size = os.path.getsize(path) > +size_aligned = pow2ceil(size) > +if size != size_aligned: > +with open(path, 'ab+') as fd: > +fd.truncate(size_aligned) Why truncate the image, can't we expand it instead? Alistair > + > class LinuxKernelTest(Test): > KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' > > @@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self): > rootfs_path_xz = self.fetch_asset(rootfs_url, asset_hash=rootfs_hash) > rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') > archive.lzma_uncompress(rootfs_path_xz, rootfs_path) > +image_pow2ceil_truncate(rootfs_path) > > self.vm.set_console() > kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + > @@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self): > image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img' > image_path = os.path.join(self.workdir, image_name) > process.run("7z e -o%s %s" % (self.workdir, image_path_7z)) > +image_pow2ceil_truncate(image_path) > > self.vm.set_console() > self.vm.add_args('-drive', 'file=' + image_path + > ',if=sd,format=raw', > @@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self): > image_hash = '2babb29d36d8360adcb39c09e31060945259917a' > image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash) > image_path = os.path.join(self.workdir, 'armv7.img') > +image_pow2ceil_truncate(image_path) > image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path > archive.gzip_uncompress(image_path_gz, image_path) > > -- > 2.21.3 > >
[PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
In the next commit we won't allow SD card images with invalid size (not aligned to a power of 2). Prepare the tests: add the pow2ceil() and image_pow2ceil_truncate() methods and truncate the images of the tests using SD cards. Signed-off-by: Philippe Mathieu-Daudé --- tests/acceptance/boot_linux_console.py | 15 +++ 1 file changed, 15 insertions(+) diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py index 3d02519660..f4d4e3635f 100644 --- a/tests/acceptance/boot_linux_console.py +++ b/tests/acceptance/boot_linux_console.py @@ -28,6 +28,18 @@ except CmdNotFoundError: P7ZIP_AVAILABLE = False +# round up to next power of 2 +def pow2ceil(x): +return 1 if x == 0 else 2**(x - 1).bit_length() + +# truncate file size to next power of 2 +def image_pow2ceil_truncate(path): +size = os.path.getsize(path) +size_aligned = pow2ceil(size) +if size != size_aligned: +with open(path, 'ab+') as fd: +fd.truncate(size_aligned) + class LinuxKernelTest(Test): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' @@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self): rootfs_path_xz = self.fetch_asset(rootfs_url, asset_hash=rootfs_hash) rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') archive.lzma_uncompress(rootfs_path_xz, rootfs_path) +image_pow2ceil_truncate(rootfs_path) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self): image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img' image_path = os.path.join(self.workdir, image_name) process.run("7z e -o%s %s" % (self.workdir, image_path_7z)) +image_pow2ceil_truncate(image_path) self.vm.set_console() self.vm.add_args('-drive', 'file=' + image_path + ',if=sd,format=raw', @@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self): image_hash = '2babb29d36d8360adcb39c09e31060945259917a' image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash) image_path = os.path.join(self.workdir, 'armv7.img') +image_pow2ceil_truncate(image_path) image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path archive.gzip_uncompress(image_path_gz, image_path) -- 2.21.3