Re: [PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
On Tue, Jul 7, 2020 at 3:21 PM Philippe Mathieu-Daudé
wrote:
> In the next commit we won't allow SD card images with invalid
> size (not aligned to a power of 2). Prepare the tests: add the
> pow2ceil() and image_pow2ceil_truncate() methods and truncate
> the images of the tests using SD cards.
>
> Signed-off-by: Philippe Mathieu-Daudé
> ---
> tests/acceptance/boot_linux_console.py | 15 +++
> 1 file changed, 15 insertions(+)
>
> diff --git a/tests/acceptance/boot_linux_console.py
> b/tests/acceptance/boot_linux_console.py
> index 3d02519660..f4d4e3635f 100644
> --- a/tests/acceptance/boot_linux_console.py
> +++ b/tests/acceptance/boot_linux_console.py
> @@ -28,6 +28,18 @@
> except CmdNotFoundError:
> P7ZIP_AVAILABLE = False
>
> +# round up to next power of 2
> +def pow2ceil(x):
> +return 1 if x == 0 else 2**(x - 1).bit_length()
> +
> +# truncate file size to next power of 2
> +def image_pow2ceil_truncate(path):
> +size = os.path.getsize(path)
> +size_aligned = pow2ceil(size)
> +if size != size_aligned:
> +with open(path, 'ab+') as fd:
> +fd.truncate(size_aligned)
> +
> class LinuxKernelTest(Test):
> KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
>
> @@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self):
> rootfs_path_xz = self.fetch_asset(rootfs_url,
> asset_hash=rootfs_hash)
> rootfs_path = os.path.join(self.workdir, 'rootfs.cpio')
> archive.lzma_uncompress(rootfs_path_xz, rootfs_path)
> +image_pow2ceil_truncate(rootfs_path)
>
> self.vm.set_console()
> kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
> @@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self):
> image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img'
> image_path = os.path.join(self.workdir, image_name)
> process.run("7z e -o%s %s" % (self.workdir, image_path_7z))
> +image_pow2ceil_truncate(image_path)
>
> self.vm.set_console()
> self.vm.add_args('-drive', 'file=' + image_path +
> ',if=sd,format=raw',
> @@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self):
> image_hash = '2babb29d36d8360adcb39c09e31060945259917a'
> image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash)
> image_path = os.path.join(self.workdir, 'armv7.img')
> +image_pow2ceil_truncate(image_path)
> image_drive_args = 'if=sd,format=raw,snapshot=on,file=' +
> image_path
> archive.gzip_uncompress(image_path_gz, image_path)
>
> --
> 2.21.3
>
>
Hi Philippe,
This patch works OK for the linux part, but the NetBSD didn't work, it
prints this error:
(5/5)
tests/acceptance/boot_linux_console.py:BootLinuxConsole.test_arm_orangepi_uboot_netbsd9:
ERROR: [Errno 2] No such file or directory:
'/var/tmp/avocado_6hoo815w/avocado_job_40aayif8/
5-tests_acceptance_boot_linux_console.py_BootLinuxConsole.test_arm_orangepi_uboot_netbsd9/armv7.img'
(0.18 s)
Basically the truncate should just be moved after the uncompress to fix it.
And the lines that we use before
to extend the image size can be removed now. That was needed to avoid
conflict with the partition size inside image.
So with these small changes, I got it working fine:
diff --git a/tests/acceptance/boot_linux_console.py
b/tests/acceptance/boot_linux_console.py
index f4d4e3635f..69607a5840 100644
--- a/tests/acceptance/boot_linux_console.py
+++ b/tests/acceptance/boot_linux_console.py
@@ -684,7 +684,7 @@ class BootLinuxConsole(LinuxKernelTest):
:avocado: tags=machine:orangepi-pc
"""
-# This test download a 196MB compressed image and expand it to
932MB...
+# This test download a 196MB compressed image and expand it to 1G
image_url = ('https://dl.armbian.com/orangepipc/archive/'
'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.7z')
image_hash = '196a8ffb72b0123d92cea4a070894813d305c71e'
@@ -725,7 +725,7 @@ class BootLinuxConsole(LinuxKernelTest):
:avocado: tags=arch:arm
:avocado: tags=machine:orangepi-pc
"""
-# This test download a 304MB compressed image and expand it to
1.3GB...
+# This test download a 304MB compressed image and expand it to
2GB...
deb_url = ('http://snapshot.debian.org/archive/debian/'
'20200108T145233Z/pool/main/u/u-boot/'
'u-boot-sunxi_2020.01%2Bdfsg-1_armhf.deb')
@@ -742,9 +742,9 @@ class BootLinuxConsole(LinuxKernelTest):
image_hash = '2babb29d36d8360adcb39c09e31060945259917a'
image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash)
image_path = os.path.join(self.workdir, 'armv7.img')
-image_pow2ceil_truncate(image_path)
image_drive_args = 'if=sd,format=raw,snapshot=on,file=' +
image_path
archive.gzip_uncompress(image_path_gz, image_path)
+image_pow2ceil_truncate(image_path)
# dd
Re: [PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
On 7/7/20 5:53 PM, Alistair Francis wrote:
> n Tue, Jul 7, 2020 at 6:21 AM Philippe Mathieu-Daudé wrote:
>>
>> In the next commit we won't allow SD card images with invalid
>> size (not aligned to a power of 2). Prepare the tests: add the
>> pow2ceil() and image_pow2ceil_truncate() methods and truncate
>> the images of the tests using SD cards.
>>
>> Signed-off-by: Philippe Mathieu-Daudé
>> ---
>> tests/acceptance/boot_linux_console.py | 15 +++
>> 1 file changed, 15 insertions(+)
>>
>> diff --git a/tests/acceptance/boot_linux_console.py
>> b/tests/acceptance/boot_linux_console.py
>> index 3d02519660..f4d4e3635f 100644
>> --- a/tests/acceptance/boot_linux_console.py
>> +++ b/tests/acceptance/boot_linux_console.py
>> @@ -28,6 +28,18 @@
>> except CmdNotFoundError:
>> P7ZIP_AVAILABLE = False
>>
>> +# round up to next power of 2
>> +def pow2ceil(x):
>> +return 1 if x == 0 else 2**(x - 1).bit_length()
>> +
>> +# truncate file size to next power of 2
>> +def image_pow2ceil_truncate(path):
>> +size = os.path.getsize(path)
>> +size_aligned = pow2ceil(size)
>> +if size != size_aligned:
>> +with open(path, 'ab+') as fd:
>> +fd.truncate(size_aligned)
>
> Why truncate the image, can't we expand it instead?
pow2ceil() round UP to the next power of 2. I think this
Python truncate() is a simple wrapper around the ftruncate()
syscall. IOW we only "expand the image". Note, these are
test images copied in tmpdir and discarded after the tests
ran. I'll improve the description.
>
> Alistair
>
>> +
>> class LinuxKernelTest(Test):
>> KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
>>
>> @@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self):
>> rootfs_path_xz = self.fetch_asset(rootfs_url,
>> asset_hash=rootfs_hash)
>> rootfs_path = os.path.join(self.workdir, 'rootfs.cpio')
>> archive.lzma_uncompress(rootfs_path_xz, rootfs_path)
>> +image_pow2ceil_truncate(rootfs_path)
>>
>> self.vm.set_console()
>> kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
>> @@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self):
>> image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img'
>> image_path = os.path.join(self.workdir, image_name)
>> process.run("7z e -o%s %s" % (self.workdir, image_path_7z))
>> +image_pow2ceil_truncate(image_path)
>>
>> self.vm.set_console()
>> self.vm.add_args('-drive', 'file=' + image_path +
>> ',if=sd,format=raw',
>> @@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self):
>> image_hash = '2babb29d36d8360adcb39c09e31060945259917a'
>> image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash)
>> image_path = os.path.join(self.workdir, 'armv7.img')
>> +image_pow2ceil_truncate(image_path)
>> image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path
>> archive.gzip_uncompress(image_path_gz, image_path)
>>
>> --
>> 2.21.3
>>
>>
>
Re: [PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
n Tue, Jul 7, 2020 at 6:21 AM Philippe Mathieu-Daudé wrote:
>
> In the next commit we won't allow SD card images with invalid
> size (not aligned to a power of 2). Prepare the tests: add the
> pow2ceil() and image_pow2ceil_truncate() methods and truncate
> the images of the tests using SD cards.
>
> Signed-off-by: Philippe Mathieu-Daudé
> ---
> tests/acceptance/boot_linux_console.py | 15 +++
> 1 file changed, 15 insertions(+)
>
> diff --git a/tests/acceptance/boot_linux_console.py
> b/tests/acceptance/boot_linux_console.py
> index 3d02519660..f4d4e3635f 100644
> --- a/tests/acceptance/boot_linux_console.py
> +++ b/tests/acceptance/boot_linux_console.py
> @@ -28,6 +28,18 @@
> except CmdNotFoundError:
> P7ZIP_AVAILABLE = False
>
> +# round up to next power of 2
> +def pow2ceil(x):
> +return 1 if x == 0 else 2**(x - 1).bit_length()
> +
> +# truncate file size to next power of 2
> +def image_pow2ceil_truncate(path):
> +size = os.path.getsize(path)
> +size_aligned = pow2ceil(size)
> +if size != size_aligned:
> +with open(path, 'ab+') as fd:
> +fd.truncate(size_aligned)
Why truncate the image, can't we expand it instead?
Alistair
> +
> class LinuxKernelTest(Test):
> KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
>
> @@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self):
> rootfs_path_xz = self.fetch_asset(rootfs_url, asset_hash=rootfs_hash)
> rootfs_path = os.path.join(self.workdir, 'rootfs.cpio')
> archive.lzma_uncompress(rootfs_path_xz, rootfs_path)
> +image_pow2ceil_truncate(rootfs_path)
>
> self.vm.set_console()
> kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
> @@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self):
> image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img'
> image_path = os.path.join(self.workdir, image_name)
> process.run("7z e -o%s %s" % (self.workdir, image_path_7z))
> +image_pow2ceil_truncate(image_path)
>
> self.vm.set_console()
> self.vm.add_args('-drive', 'file=' + image_path +
> ',if=sd,format=raw',
> @@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self):
> image_hash = '2babb29d36d8360adcb39c09e31060945259917a'
> image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash)
> image_path = os.path.join(self.workdir, 'armv7.img')
> +image_pow2ceil_truncate(image_path)
> image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path
> archive.gzip_uncompress(image_path_gz, image_path)
>
> --
> 2.21.3
>
>
[PATCH 1/2] tests/acceptance/boot_linux: Truncate SD card image to power of 2
In the next commit we won't allow SD card images with invalid
size (not aligned to a power of 2). Prepare the tests: add the
pow2ceil() and image_pow2ceil_truncate() methods and truncate
the images of the tests using SD cards.
Signed-off-by: Philippe Mathieu-Daudé
---
tests/acceptance/boot_linux_console.py | 15 +++
1 file changed, 15 insertions(+)
diff --git a/tests/acceptance/boot_linux_console.py
b/tests/acceptance/boot_linux_console.py
index 3d02519660..f4d4e3635f 100644
--- a/tests/acceptance/boot_linux_console.py
+++ b/tests/acceptance/boot_linux_console.py
@@ -28,6 +28,18 @@
except CmdNotFoundError:
P7ZIP_AVAILABLE = False
+# round up to next power of 2
+def pow2ceil(x):
+return 1 if x == 0 else 2**(x - 1).bit_length()
+
+# truncate file size to next power of 2
+def image_pow2ceil_truncate(path):
+size = os.path.getsize(path)
+size_aligned = pow2ceil(size)
+if size != size_aligned:
+with open(path, 'ab+') as fd:
+fd.truncate(size_aligned)
+
class LinuxKernelTest(Test):
KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
@@ -635,6 +647,7 @@ def test_arm_orangepi_sd(self):
rootfs_path_xz = self.fetch_asset(rootfs_url, asset_hash=rootfs_hash)
rootfs_path = os.path.join(self.workdir, 'rootfs.cpio')
archive.lzma_uncompress(rootfs_path_xz, rootfs_path)
+image_pow2ceil_truncate(rootfs_path)
self.vm.set_console()
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
@@ -679,6 +692,7 @@ def test_arm_orangepi_bionic(self):
image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img'
image_path = os.path.join(self.workdir, image_name)
process.run("7z e -o%s %s" % (self.workdir, image_path_7z))
+image_pow2ceil_truncate(image_path)
self.vm.set_console()
self.vm.add_args('-drive', 'file=' + image_path + ',if=sd,format=raw',
@@ -728,6 +742,7 @@ def test_arm_orangepi_uboot_netbsd9(self):
image_hash = '2babb29d36d8360adcb39c09e31060945259917a'
image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash)
image_path = os.path.join(self.workdir, 'armv7.img')
+image_pow2ceil_truncate(image_path)
image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path
archive.gzip_uncompress(image_path_gz, image_path)
--
2.21.3
