[pve-devel] [PATCH zfsonlinux v2 1/2] update zfs submodule to 2.2.4 and refresh patches

2024-05-07 Thread Stoiko Ivanov 
mostly - drop all patches we had queued up to get kernel 6.8
supported.

Signed-off-by: Stoiko Ivanov 
---
 ...md-unit-for-importing-specific-pools.patch |   4 +-
 ...-move-manpage-arcstat-1-to-arcstat-8.patch |   2 +-
 ...-guard-access-to-l2arc-MFU-MRU-stats.patch |  12 +-
 ...hten-bounds-for-noalloc-stat-availab.patch |   4 +-
 ...rectly-handle-partition-16-and-later.patch |  52 --
 ...-use-splice_copy_file_range-for-fall.patch | 135 
 .../0014-linux-5.4-compat-page_size.patch | 121 
 .../patches/0015-abd-add-page-iterator.patch  | 334 -
 ...-existing-functions-to-vdev_classic_.patch | 349 -
 ...v_disk-reorganise-vdev_disk_io_start.patch | 111 ---
 ...-read-write-IO-function-configurable.patch |  69 --
 ...e-BIO-filling-machinery-to-avoid-spl.patch | 671 --
 ...dule-parameter-to-select-BIO-submiss.patch | 104 ---
 ...se-bio_chain-to-submit-multiple-BIOs.patch | 363 --
 ...on-t-use-compound-heads-on-Linux-4.5.patch |  96 ---
 ...ault-to-classic-submission-for-2.2.x.patch |  90 ---
 ...ion-caused-by-mmap-flushing-problems.patch | 104 ---
 ...touch-vbio-after-its-handed-off-to-t.patch |  57 --
 debian/patches/series |  14 -
 upstream  |   2 +-
 20 files changed, 12 insertions(+), 2682 deletions(-)
 delete mode 100644 
debian/patches/0012-udev-correctly-handle-partition-16-and-later.patch
 delete mode 100644 
debian/patches/0013-Linux-6.8-compat-use-splice_copy_file_range-for-fall.patch
 delete mode 100644 debian/patches/0014-linux-5.4-compat-page_size.patch
 delete mode 100644 debian/patches/0015-abd-add-page-iterator.patch
 delete mode 100644 
debian/patches/0016-vdev_disk-rename-existing-functions-to-vdev_classic_.patch
 delete mode 100644 
debian/patches/0017-vdev_disk-reorganise-vdev_disk_io_start.patch
 delete mode 100644 
debian/patches/0018-vdev_disk-make-read-write-IO-function-configurable.patch
 delete mode 100644 
debian/patches/0019-vdev_disk-rewrite-BIO-filling-machinery-to-avoid-spl.patch
 delete mode 100644 
debian/patches/0020-vdev_disk-add-module-parameter-to-select-BIO-submiss.patch
 delete mode 100644 
debian/patches/0021-vdev_disk-use-bio_chain-to-submit-multiple-BIOs.patch
 delete mode 100644 
debian/patches/0022-abd_iter_page-don-t-use-compound-heads-on-Linux-4.5.patch
 delete mode 100644 
debian/patches/0023-vdev_disk-default-to-classic-submission-for-2.2.x.patch
 delete mode 100644 
debian/patches/0024-Fix-corruption-caused-by-mmap-flushing-problems.patch
 delete mode 100644 
debian/patches/0025-vdev_disk-don-t-touch-vbio-after-its-handed-off-to-t.patch

diff --git 
a/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch 
b/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch
index 8232978c..0600296f 100644
--- a/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch
+++ b/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch
@@ -18,7 +18,7 @@ Signed-off-by: Thomas Lamprecht 
 ---
  etc/Makefile.am   |  1 +
  etc/systemd/system/50-zfs.preset  |  1 +
- etc/systemd/system/zfs-imp...@.service.in | 18 
+ etc/systemd/system/zfs-imp...@.service.in | 18 ++
  3 files changed, 20 insertions(+)
  create mode 100644 etc/systemd/system/zfs-imp...@.service.in
 
@@ -48,7 +48,7 @@ index e4056a92c..030611419 100644
  enable zfs-share.service
 diff --git a/etc/systemd/system/zfs-imp...@.service.in 
b/etc/systemd/system/zfs-imp...@.service.in
 new file mode 100644
-index 0..9b4ee9371
+index 0..5bd19fb79
 --- /dev/null
 +++ b/etc/systemd/system/zfs-imp...@.service.in
 @@ -0,0 +1,18 @@
diff --git 
a/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch 
b/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch
index c11c1ae8..9a4aea56 100644
--- a/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch
+++ b/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch
@@ -15,7 +15,7 @@ Signed-off-by: Thomas Lamprecht 
  rename man/{man1/arcstat.1 => man8/arcstat.8} (99%)
 
 diff --git a/man/Makefile.am b/man/Makefile.am
-index 45156571e..3713e9371 100644
+index 43bb014dd..a9293468a 100644
 --- a/man/Makefile.am
 +++ b/man/Makefile.am
 @@ -2,7 +2,6 @@ dist_noinst_man_MANS = \
diff --git 
a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
 
b/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
index f8cb3539..2e7c207d 100644
--- 
a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
+++ 
b/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
@@ -27,7 +27,7 @@ Signed-off-by: Thomas Lamprecht 
  2 files changed, 21 insertions(+), 21 deletions(-)
 
 diff --git a/cmd/arc_summary b/cmd/arc_summary
-index 9c69ec4f8..edf94ea2a 100755
+index 100fb1987..86b2260a1 100755
 --- a/cmd/arc_summary
 +++ b/cmd/

[pve-devel] [PATCH zfsonlinux v2 2/2] update arc_summary arcstat patch with new introduced values

2024-05-07 Thread Stoiko Ivanov 
ZFS 2.2.4 added new kstats for speculative prefetch in:
026fe796465e3da7b27d06ef5338634ee6dd30d8

Adapt our patch introduced with ZFS 2.1 (for the then added MFU/MRU
stats), to also deal with the now introduced values not being present
(because an old kernel-module does not offer them).

Signed-off-by: Stoiko Ivanov 
---
 ...-guard-access-to-freshly-introduced-.patch | 438 ++
 ...-guard-access-to-l2arc-MFU-MRU-stats.patch | 113 -
 debian/patches/series |   2 +-
 3 files changed, 439 insertions(+), 114 deletions(-)
 create mode 100644 
debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
 delete mode 100644 
debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch

diff --git 
a/debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
 
b/debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
new file mode 100644
index ..bc7db2a9
--- /dev/null
+++ 
b/debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
@@ -0,0 +1,438 @@
+From  Mon Sep 17 00:00:00 2001
+From: Thomas Lamprecht 
+Date: Wed, 10 Nov 2021 09:29:47 +0100
+Subject: [PATCH] arc stat/summary: guard access to freshly introduced stats
+
+l2arc MFU/MRU and zfetch past future and stride stats were introduced
+in 2.1 and 2.2.4 respectively:
+
+commit 085321621e79a75bea41c2b6511da6ebfbf2ba0a added printing MFU
+and MRU stats for 2.1 user space tools, but those keys are not
+available in the 2.0 module. That means it may break the arcstat and
+arc_summary tools after upgrade to 2.1 (user space), before a reboot
+to the new 2.1 ZFS kernel-module happened, due to python raising a
+KeyError on the dict access then.
+
+Move those two keys to a .get accessor with `0` as fallback, as it
+should be better to show some possible wrong data for new stat-keys
+than throwing an exception.
+
+also move l2_mfu_asize  l2_mru_asize l2_prefetch_asize
+l2_bufc_data_asize l2_bufc_metadata_asize to .get accessor
+(these are only present with a cache device in the pool)
+
+guard access to iohits and uncached state introduced in
+792a6ee462efc15a7614f27e13f0f8aaa9414a08
+
+guard access to zfetch past future stride stats introduced in
+026fe796465e3da7b27d06ef5338634ee6dd30d8
+
+These are present in the current kernel, but lead to an exception, if
+running the new user-space with an old kernel module.
+
+Signed-off-by: Stoiko Ivanov 
+---
+ cmd/arc_summary | 132 
+ cmd/arcstat.in  |  48 +-
+ 2 files changed, 90 insertions(+), 90 deletions(-)
+
+diff --git a/cmd/arc_summary b/cmd/arc_summary
+index 100fb1987..30f5d23e9 100755
+--- a/cmd/arc_summary
 b/cmd/arc_summary
+@@ -551,21 +551,21 @@ def section_arc(kstats_dict):
+ arc_target_size = arc_stats['c']
+ arc_max = arc_stats['c_max']
+ arc_min = arc_stats['c_min']
+-meta = arc_stats['meta']
+-pd = arc_stats['pd']
+-pm = arc_stats['pm']
+-anon_data = arc_stats['anon_data']
+-anon_metadata = arc_stats['anon_metadata']
+-mfu_data = arc_stats['mfu_data']
+-mfu_metadata = arc_stats['mfu_metadata']
+-mru_data = arc_stats['mru_data']
+-mru_metadata = arc_stats['mru_metadata']
+-mfug_data = arc_stats['mfu_ghost_data']
+-mfug_metadata = arc_stats['mfu_ghost_metadata']
+-mrug_data = arc_stats['mru_ghost_data']
+-mrug_metadata = arc_stats['mru_ghost_metadata']
+-unc_data = arc_stats['uncached_data']
+-unc_metadata = arc_stats['uncached_metadata']
++meta = arc_stats.get('meta', 0)
++pd = arc_stats.get('pd', 0)
++pm = arc_stats.get('pm', 0)
++anon_data = arc_stats.get('anon_data', 0)
++anon_metadata = arc_stats.get('anon_metadata', 0)
++mfu_data = arc_stats.get('mfu_data', 0)
++mfu_metadata = arc_stats.get('mfu_metadata', 0)
++mru_data = arc_stats.get('mru_data', 0)
++mru_metadata = arc_stats.get('mru_metadata', 0)
++mfug_data = arc_stats.get('mfu_ghost_data', 0)
++mfug_metadata = arc_stats.get('mfu_ghost_metadata', 0)
++mrug_data = arc_stats.get('mru_ghost_data', 0)
++mrug_metadata = arc_stats.get('mru_ghost_metadata', 0)
++unc_data = arc_stats.get('uncached_data', 0)
++unc_metadata = arc_stats.get('uncached_metadata', 0)
+ bonus_size = arc_stats['bonus_size']
+ dnode_limit = arc_stats['arc_dnode_limit']
+ dnode_size = arc_stats['dnode_size']
+@@ -655,13 +655,13 @@ def section_arc(kstats_dict):
+ prt_i1('L2 cached evictions:', f_bytes(arc_stats['evict_l2_cached']))
+ prt_i1('L2 eligible evictions:', f_bytes(arc_stats['evict_l2_eligible']))
+ prt_i2('L2 eligible MFU evictions:',
+-   f_perc(arc_stats['evict_l2_eligible_mfu'],
++   f_perc(arc_stats.get('evict_l2_eligible_mfu', 0), # 2.0 module 
compat
+arc_stats['evict_l2_eligible']),
+-   f_bytes(arc_stats['evict_l2_eligible_mfu']

[pve-devel] [PATCH zfsonlinux v2 0/2] Update to ZFS 2.2.4

2024-05-07 Thread Stoiko Ivanov 
v1->v2:
Patch 2/2 (adaptation of arc_summary/arcstat patch) modified:
* right after sending the v1 I saw a report where pinning kernel 6.2 (thus
  ZFS 2.1) leads to a similar traceback - which I seem to have overlooked
  when packaging 2.2.0 ...
  adapted the patch by booting a VM with kernel 6.2 and the current
  userspace and running arc_summary /arcstat -a until no traceback was
  displayed with a single-disk pool.

original cover-letter for v1:
This patchset updates ZFS to the recently released 2.2.4

We had about half of the patches already in 2.2.3-2, due to the needed
support for kernel 6.8.

Compared to the last 2.2 point releases this one compares quite a few
potential performance improvments:
* for ZVOL workloads (relevant for qemu guests) multiple taskq were
  introduced [1] - this change is active by default (can be put back to
  the old behavior with explicitly setting `zvol_num_taskqs=1`
* the interface for ZFS submitting operations to the kernel's block layer
  was augmented to better deal with split-pages [2] - which should also
  improve performance, and prevent unaligned writes which are rejected by
  e.g. the SCSI subsystem. - The default remains with the current code
  (`zfs_vdev_disk_classic=0` turns on the 'new' behavior...)
* Speculative prefetching was improved [3], which introduced not kstats,
  which are reported by`arc_summary` and `arcstat`, as before with the
  MRU/MFU additions there was not guard for running the new user-space
  with an old kernel resulting in Python exceptions of both tools.
  I adapted the patch where Thomas fixed that back in the 2.1 release
  times. - sending as separate patch for easier review - and I hope it's
  ok that I dropped the S-o-b tag (as it's changed code) - glad to resend
  it, if this should be adapted.

Minimally tested on 2 VMs (the arcstat/arc_summary changes by running with
an old kernel and new user-space)


[0] https://github.com/openzfs/zfs/releases/tag/zfs-2.2.4
[1] https://github.com/openzfs/zfs/pull/15992
[2] https://github.com/openzfs/zfs/pull/15588
[3] https://github.com/openzfs/zfs/pull/16022

Stoiko Ivanov (2):
  update zfs submodule to 2.2.4 and refresh patches
  update arc_summary arcstat patch with new introduced values

 ...md-unit-for-importing-specific-pools.patch |   4 +-
 ...-move-manpage-arcstat-1-to-arcstat-8.patch |   2 +-
 ...-guard-access-to-freshly-introduced-.patch | 438 
 ...-guard-access-to-l2arc-MFU-MRU-stats.patch | 113 ---
 ...hten-bounds-for-noalloc-stat-availab.patch |   4 +-
 ...rectly-handle-partition-16-and-later.patch |  52 --
 ...-use-splice_copy_file_range-for-fall.patch | 135 
 .../0014-linux-5.4-compat-page_size.patch | 121 
 .../patches/0015-abd-add-page-iterator.patch  | 334 -
 ...-existing-functions-to-vdev_classic_.patch | 349 -
 ...v_disk-reorganise-vdev_disk_io_start.patch | 111 ---
 ...-read-write-IO-function-configurable.patch |  69 --
 ...e-BIO-filling-machinery-to-avoid-spl.patch | 671 --
 ...dule-parameter-to-select-BIO-submiss.patch | 104 ---
 ...se-bio_chain-to-submit-multiple-BIOs.patch | 363 --
 ...on-t-use-compound-heads-on-Linux-4.5.patch |  96 ---
 ...ault-to-classic-submission-for-2.2.x.patch |  90 ---
 ...ion-caused-by-mmap-flushing-problems.patch | 104 ---
 ...touch-vbio-after-its-handed-off-to-t.patch |  57 --
 debian/patches/series |  16 +-
 upstream  |   2 +-
 21 files changed, 445 insertions(+), 2790 deletions(-)
 create mode 100644 
debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
 delete mode 100644 
debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
 delete mode 100644 
debian/patches/0012-udev-correctly-handle-partition-16-and-later.patch
 delete mode 100644 
debian/patches/0013-Linux-6.8-compat-use-splice_copy_file_range-for-fall.patch
 delete mode 100644 debian/patches/0014-linux-5.4-compat-page_size.patch
 delete mode 100644 debian/patches/0015-abd-add-page-iterator.patch
 delete mode 100644 
debian/patches/0016-vdev_disk-rename-existing-functions-to-vdev_classic_.patch
 delete mode 100644 
debian/patches/0017-vdev_disk-reorganise-vdev_disk_io_start.patch
 delete mode 100644 
debian/patches/0018-vdev_disk-make-read-write-IO-function-configurable.patch
 delete mode 100644 
debian/patches/0019-vdev_disk-rewrite-BIO-filling-machinery-to-avoid-spl.patch
 delete mode 100644 
debian/patches/0020-vdev_disk-add-module-parameter-to-select-BIO-submiss.patch
 delete mode 100644 
debian/patches/0021-vdev_disk-use-bio_chain-to-submit-multiple-BIOs.patch
 delete mode 100644 
debian/patches/0022-abd_iter_page-don-t-use-compound-heads-on-Linux-4.5.patch
 delete mode 100644 
debian/patches/0023-vdev_disk-default-to-classic-submission-for-2.2.x.patch
 delete mode 100644 
debian/patches/0024-Fix-corruption-caused-by-mmap-flushing-problems.patch
 delete mode 100644 
debian/patches/0025-vdev_disk-don-t-touch-

Re: [pve-devel] [PATCH zfsonlinux 2/2] update arc_summary arcstat patch with new introduced values

2024-05-07 Thread Stoiko Ivanov 
10 minutes after sending this - I saw a report about pvereport ending in a
Python stacktrace - took me a while to see that a similar issue is present
between 2.1 and 2.2 - will send the series again with those changes also
added (this time the method was changing the source until no more
stacktraces were present with the current userspace and kernel 6.2 (with
ZFS 2.1) running).

Not sure if dropping the whole patch or alternatively cleaning it up once
every major PVE release would also be an option (although tbh - I expect
quite a few monitoring tools to collect data from these utils - and having
that throw exceptions will probably cause some discomfort to our users...)


On Tue,  7 May 2024 15:38:36 +0200
Stoiko Ivanov  wrote:

> ZFS 2.2.4 added new kstats for speculative prefetch in:
> 026fe796465e3da7b27d06ef5338634ee6dd30d8
> 
> Adapt our patch introduced with ZFS 2.1 (for the then added MFU/MRU
> stats), to also deal with the now introduced values not being present
> (because an old kernel-module does not offer them).
> 
> Signed-off-by: Stoiko Ivanov 
> ---
>  ...guard-access-to-freshly-introduced-.patch} | 79 ---
>  debian/patches/series |  2 +-
>  2 files changed, 69 insertions(+), 12 deletions(-)
>  rename 
> debian/patches/{0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
>  => 0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch} (61%)
> 
> diff --git 
> a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
>  
> b/debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
> similarity index 61%
> rename from 
> debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
> rename to 
> debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
> index 2e7c207d..a0768923 100644
> --- 
> a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
> +++ 
> b/debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
> @@ -1,7 +1,10 @@
>  From  Mon Sep 17 00:00:00 2001
>  From: Thomas Lamprecht 
>  Date: Wed, 10 Nov 2021 09:29:47 +0100
> -Subject: [PATCH] arc stat/summary: guard access to l2arc MFU/MRU stats
> +Subject: [PATCH] arc stat/summary: guard access to freshly introduced stats
> +
> +l2arc MFU/MRU and zfetch past future and stride stats were introduced
> +in 2.1 and 2.2.4 respectively:
>  
>  commit 085321621e79a75bea41c2b6511da6ebfbf2ba0a added printing MFU
>  and MRU stats for 2.1 user space tools, but those keys are not
> @@ -14,20 +17,24 @@ Move those two keys to a .get accessor with `0` as 
> fallback, as it
>  should be better to show some possible wrong data for new stat-keys
>  than throwing an exception.
>  
> -Signed-off-by: Thomas Lamprecht 
> -
>  also move l2_mfu_asize  l2_mru_asize l2_prefetch_asize
>  l2_bufc_data_asize l2_bufc_metadata_asize to .get accessor
>  (these are only present with a cache device in the pool)
> +
> +guard access to zfetch past future stride stats introduced in
> +026fe796465e3da7b27d06ef5338634ee6dd30d8
> +
> +These are present in the current kernel, but lead to an exception, if
> +running the new user-space with an old kernel module.
> +
>  Signed-off-by: Stoiko Ivanov 
> -Signed-off-by: Thomas Lamprecht 
>  ---
> - cmd/arc_summary | 28 ++--
> - cmd/arcstat.in  | 14 +++---
> - 2 files changed, 21 insertions(+), 21 deletions(-)
> + cmd/arc_summary | 40 
> + cmd/arcstat.in  | 26 +-
> + 2 files changed, 33 insertions(+), 33 deletions(-)
>  
>  diff --git a/cmd/arc_summary b/cmd/arc_summary
> -index 100fb1987..86b2260a1 100755
> +index 100fb1987..5fb2cdbbc 100755
>  --- a/cmd/arc_summary
>  +++ b/cmd/arc_summary
>  @@ -655,13 +655,13 @@ def section_arc(kstats_dict):
> @@ -48,6 +55,39 @@ index 100fb1987..86b2260a1 100755
>   prt_i1('L2 ineligible evictions:',
>  f_bytes(arc_stats['evict_l2_ineligible']))
>   print()
> +@@ -794,26 +794,26 @@ def section_dmu(kstats_dict):
> + zfetch_stats = isolate_section('zfetchstats', kstats_dict)
> + 
> + zfetch_access_total = int(zfetch_stats['hits']) +\
> +-int(zfetch_stats['future']) + int(zfetch_stats['stride']) +\
> +-int(zfetch_stats['past']) + int(zfetch_stats['misses'])
> ++int(zfetch_stats.get('future', 0)) + int(zfetch_stats.get('stride', 
> 0)) +\
> ++int(zfetch_stats.get('past', 0)) + int(zfetch_stats['misses'])
> + 
> + prt_1('DMU predictive prefetcher calls:', f_hits(zfetch_access_total))
> + prt_i2('Stream hits:',
> +f_perc(zfetch_stats['hits'], zfetch_access_total),
> +f_hits(zfetch_stats['hits']))
> +-future = int(zfetch_stats['future']) + int(zfetch_stats['stride'])
> ++future = int(zfetch_stats.get('future', 0)) + 
> int(zfetch_stats.get('stride', 0))
> + prt_i2('Hits ahead o

[pve-devel] [PATCH zfsonlinux 1/2] update zfs submodule to 2.2.4 and refresh patches

2024-05-07 Thread Stoiko Ivanov 
mostly - drop all patches we had queued up to get kernel 6.8
supported.

Signed-off-by: Stoiko Ivanov 
---
 ...md-unit-for-importing-specific-pools.patch |   4 +-
 ...-move-manpage-arcstat-1-to-arcstat-8.patch |   2 +-
 ...-guard-access-to-l2arc-MFU-MRU-stats.patch |  12 +-
 ...hten-bounds-for-noalloc-stat-availab.patch |   4 +-
 ...rectly-handle-partition-16-and-later.patch |  52 --
 ...-use-splice_copy_file_range-for-fall.patch | 135 
 .../0014-linux-5.4-compat-page_size.patch | 121 
 .../patches/0015-abd-add-page-iterator.patch  | 334 -
 ...-existing-functions-to-vdev_classic_.patch | 349 -
 ...v_disk-reorganise-vdev_disk_io_start.patch | 111 ---
 ...-read-write-IO-function-configurable.patch |  69 --
 ...e-BIO-filling-machinery-to-avoid-spl.patch | 671 --
 ...dule-parameter-to-select-BIO-submiss.patch | 104 ---
 ...se-bio_chain-to-submit-multiple-BIOs.patch | 363 --
 ...on-t-use-compound-heads-on-Linux-4.5.patch |  96 ---
 ...ault-to-classic-submission-for-2.2.x.patch |  90 ---
 ...ion-caused-by-mmap-flushing-problems.patch | 104 ---
 ...touch-vbio-after-its-handed-off-to-t.patch |  57 --
 debian/patches/series |  14 -
 upstream  |   2 +-
 20 files changed, 12 insertions(+), 2682 deletions(-)
 delete mode 100644 
debian/patches/0012-udev-correctly-handle-partition-16-and-later.patch
 delete mode 100644 
debian/patches/0013-Linux-6.8-compat-use-splice_copy_file_range-for-fall.patch
 delete mode 100644 debian/patches/0014-linux-5.4-compat-page_size.patch
 delete mode 100644 debian/patches/0015-abd-add-page-iterator.patch
 delete mode 100644 
debian/patches/0016-vdev_disk-rename-existing-functions-to-vdev_classic_.patch
 delete mode 100644 
debian/patches/0017-vdev_disk-reorganise-vdev_disk_io_start.patch
 delete mode 100644 
debian/patches/0018-vdev_disk-make-read-write-IO-function-configurable.patch
 delete mode 100644 
debian/patches/0019-vdev_disk-rewrite-BIO-filling-machinery-to-avoid-spl.patch
 delete mode 100644 
debian/patches/0020-vdev_disk-add-module-parameter-to-select-BIO-submiss.patch
 delete mode 100644 
debian/patches/0021-vdev_disk-use-bio_chain-to-submit-multiple-BIOs.patch
 delete mode 100644 
debian/patches/0022-abd_iter_page-don-t-use-compound-heads-on-Linux-4.5.patch
 delete mode 100644 
debian/patches/0023-vdev_disk-default-to-classic-submission-for-2.2.x.patch
 delete mode 100644 
debian/patches/0024-Fix-corruption-caused-by-mmap-flushing-problems.patch
 delete mode 100644 
debian/patches/0025-vdev_disk-don-t-touch-vbio-after-its-handed-off-to-t.patch

diff --git 
a/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch 
b/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch
index 8232978c..0600296f 100644
--- a/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch
+++ b/debian/patches/0007-Add-systemd-unit-for-importing-specific-pools.patch
@@ -18,7 +18,7 @@ Signed-off-by: Thomas Lamprecht 
 ---
  etc/Makefile.am   |  1 +
  etc/systemd/system/50-zfs.preset  |  1 +
- etc/systemd/system/zfs-imp...@.service.in | 18 
+ etc/systemd/system/zfs-imp...@.service.in | 18 ++
  3 files changed, 20 insertions(+)
  create mode 100644 etc/systemd/system/zfs-imp...@.service.in
 
@@ -48,7 +48,7 @@ index e4056a92c..030611419 100644
  enable zfs-share.service
 diff --git a/etc/systemd/system/zfs-imp...@.service.in 
b/etc/systemd/system/zfs-imp...@.service.in
 new file mode 100644
-index 0..9b4ee9371
+index 0..5bd19fb79
 --- /dev/null
 +++ b/etc/systemd/system/zfs-imp...@.service.in
 @@ -0,0 +1,18 @@
diff --git 
a/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch 
b/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch
index c11c1ae8..9a4aea56 100644
--- a/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch
+++ b/debian/patches/0008-Patch-move-manpage-arcstat-1-to-arcstat-8.patch
@@ -15,7 +15,7 @@ Signed-off-by: Thomas Lamprecht 
  rename man/{man1/arcstat.1 => man8/arcstat.8} (99%)
 
 diff --git a/man/Makefile.am b/man/Makefile.am
-index 45156571e..3713e9371 100644
+index 43bb014dd..a9293468a 100644
 --- a/man/Makefile.am
 +++ b/man/Makefile.am
 @@ -2,7 +2,6 @@ dist_noinst_man_MANS = \
diff --git 
a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
 
b/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
index f8cb3539..2e7c207d 100644
--- 
a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
+++ 
b/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
@@ -27,7 +27,7 @@ Signed-off-by: Thomas Lamprecht 
  2 files changed, 21 insertions(+), 21 deletions(-)
 
 diff --git a/cmd/arc_summary b/cmd/arc_summary
-index 9c69ec4f8..edf94ea2a 100755
+index 100fb1987..86b2260a1 100755
 --- a/cmd/arc_summary
 +++ b/cmd/

[pve-devel] [PATCH zfsonlinux 2/2] update arc_summary arcstat patch with new introduced values

2024-05-07 Thread Stoiko Ivanov 
ZFS 2.2.4 added new kstats for speculative prefetch in:
026fe796465e3da7b27d06ef5338634ee6dd30d8

Adapt our patch introduced with ZFS 2.1 (for the then added MFU/MRU
stats), to also deal with the now introduced values not being present
(because an old kernel-module does not offer them).

Signed-off-by: Stoiko Ivanov 
---
 ...guard-access-to-freshly-introduced-.patch} | 79 ---
 debian/patches/series |  2 +-
 2 files changed, 69 insertions(+), 12 deletions(-)
 rename 
debian/patches/{0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch 
=> 0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch} (61%)

diff --git 
a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
 
b/debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
similarity index 61%
rename from 
debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
rename to 
debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
index 2e7c207d..a0768923 100644
--- 
a/debian/patches/0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch
+++ 
b/debian/patches/0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch
@@ -1,7 +1,10 @@
 From  Mon Sep 17 00:00:00 2001
 From: Thomas Lamprecht 
 Date: Wed, 10 Nov 2021 09:29:47 +0100
-Subject: [PATCH] arc stat/summary: guard access to l2arc MFU/MRU stats
+Subject: [PATCH] arc stat/summary: guard access to freshly introduced stats
+
+l2arc MFU/MRU and zfetch past future and stride stats were introduced
+in 2.1 and 2.2.4 respectively:
 
 commit 085321621e79a75bea41c2b6511da6ebfbf2ba0a added printing MFU
 and MRU stats for 2.1 user space tools, but those keys are not
@@ -14,20 +17,24 @@ Move those two keys to a .get accessor with `0` as 
fallback, as it
 should be better to show some possible wrong data for new stat-keys
 than throwing an exception.
 
-Signed-off-by: Thomas Lamprecht 
-
 also move l2_mfu_asize  l2_mru_asize l2_prefetch_asize
 l2_bufc_data_asize l2_bufc_metadata_asize to .get accessor
 (these are only present with a cache device in the pool)
+
+guard access to zfetch past future stride stats introduced in
+026fe796465e3da7b27d06ef5338634ee6dd30d8
+
+These are present in the current kernel, but lead to an exception, if
+running the new user-space with an old kernel module.
+
 Signed-off-by: Stoiko Ivanov 
-Signed-off-by: Thomas Lamprecht 
 ---
- cmd/arc_summary | 28 ++--
- cmd/arcstat.in  | 14 +++---
- 2 files changed, 21 insertions(+), 21 deletions(-)
+ cmd/arc_summary | 40 
+ cmd/arcstat.in  | 26 +-
+ 2 files changed, 33 insertions(+), 33 deletions(-)
 
 diff --git a/cmd/arc_summary b/cmd/arc_summary
-index 100fb1987..86b2260a1 100755
+index 100fb1987..5fb2cdbbc 100755
 --- a/cmd/arc_summary
 +++ b/cmd/arc_summary
 @@ -655,13 +655,13 @@ def section_arc(kstats_dict):
@@ -48,6 +55,39 @@ index 100fb1987..86b2260a1 100755
  prt_i1('L2 ineligible evictions:',
 f_bytes(arc_stats['evict_l2_ineligible']))
  print()
+@@ -794,26 +794,26 @@ def section_dmu(kstats_dict):
+ zfetch_stats = isolate_section('zfetchstats', kstats_dict)
+ 
+ zfetch_access_total = int(zfetch_stats['hits']) +\
+-int(zfetch_stats['future']) + int(zfetch_stats['stride']) +\
+-int(zfetch_stats['past']) + int(zfetch_stats['misses'])
++int(zfetch_stats.get('future', 0)) + int(zfetch_stats.get('stride', 
0)) +\
++int(zfetch_stats.get('past', 0)) + int(zfetch_stats['misses'])
+ 
+ prt_1('DMU predictive prefetcher calls:', f_hits(zfetch_access_total))
+ prt_i2('Stream hits:',
+f_perc(zfetch_stats['hits'], zfetch_access_total),
+f_hits(zfetch_stats['hits']))
+-future = int(zfetch_stats['future']) + int(zfetch_stats['stride'])
++future = int(zfetch_stats.get('future', 0)) + 
int(zfetch_stats.get('stride', 0))
+ prt_i2('Hits ahead of stream:', f_perc(future, zfetch_access_total),
+f_hits(future))
+ prt_i2('Hits behind stream:',
+-   f_perc(zfetch_stats['past'], zfetch_access_total),
+-   f_hits(zfetch_stats['past']))
++   f_perc(zfetch_stats.get('past', 0), zfetch_access_total),
++   f_hits(zfetch_stats.get('past', 0)))
+ prt_i2('Stream misses:',
+f_perc(zfetch_stats['misses'], zfetch_access_total),
+f_hits(zfetch_stats['misses']))
+ prt_i2('Streams limit reached:',
+f_perc(zfetch_stats['max_streams'], zfetch_stats['misses']),
+f_hits(zfetch_stats['max_streams']))
+-prt_i1('Stream strides:', f_hits(zfetch_stats['stride']))
++prt_i1('Stream strides:', f_hits(zfetch_stats.get('stride', 0)))
+ prt_i1('Prefetches issued', f_hits(zfetch_stats['io_issued']))
+ print()
+ 
 @@ -860,20 +860,20 @@ def section_l2arc(kstats_dict):

[pve-devel] [PATCH zfsonlinux 0/2] Update to ZFS 2.2.4

2024-05-07 Thread Stoiko Ivanov 
This patchset updates ZFS to the recently released 2.2.4

We had about half of the patches already in 2.2.3-2, due to the needed
support for kernel 6.8.

Compared to the last 2.2 point releases this one compares quite a few
potential performance improvments:
* for ZVOL workloads (relevant for qemu guests) multiple taskq were
  introduced [1] - this change is active by default (can be put back to
  the old behavior with explicitly setting `zvol_num_taskqs=1`
* the interface for ZFS submitting operations to the kernel's block layer
  was augmented to better deal with split-pages [2] - which should also
  improve performance, and prevent unaligned writes which are rejected by
  e.g. the SCSI subsystem. - The default remains with the current code
  (`zfs_vdev_disk_classic=0` turns on the 'new' behavior...)
* Speculative prefetching was improved [3], which introduced not kstats,
  which are reported by`arc_summary` and `arcstat`, as before with the
  MRU/MFU additions there was not guard for running the new user-space
  with an old kernel resulting in Python exceptions of both tools.
  I adapted the patch where Thomas fixed that back in the 2.1 release
  times. - sending as separate patch for easier review - and I hope it's
  ok that I dropped the S-o-b tag (as it's changed code) - glad to resend
  it, if this should be adapted.

Minimally tested on 2 VMs (the arcstat/arc_summary changes by running with
an old kernel and new user-space)


[0] https://github.com/openzfs/zfs/releases/tag/zfs-2.2.4
[1] https://github.com/openzfs/zfs/pull/15992
[2] https://github.com/openzfs/zfs/pull/15588
[3] https://github.com/openzfs/zfs/pull/16022
Stoiko Ivanov (2):
  update zfs submodule to 2.2.4 and refresh patches
  update arc_summary arcstat patch with new introduced values

 ...md-unit-for-importing-specific-pools.patch |   4 +-
 ...-move-manpage-arcstat-1-to-arcstat-8.patch |   2 +-
 ...guard-access-to-freshly-introduced-.patch} |  81 ++-
 ...hten-bounds-for-noalloc-stat-availab.patch |   4 +-
 ...rectly-handle-partition-16-and-later.patch |  52 --
 ...-use-splice_copy_file_range-for-fall.patch | 135 
 .../0014-linux-5.4-compat-page_size.patch | 121 
 .../patches/0015-abd-add-page-iterator.patch  | 334 -
 ...-existing-functions-to-vdev_classic_.patch | 349 -
 ...v_disk-reorganise-vdev_disk_io_start.patch | 111 ---
 ...-read-write-IO-function-configurable.patch |  69 --
 ...e-BIO-filling-machinery-to-avoid-spl.patch | 671 --
 ...dule-parameter-to-select-BIO-submiss.patch | 104 ---
 ...se-bio_chain-to-submit-multiple-BIOs.patch | 363 --
 ...on-t-use-compound-heads-on-Linux-4.5.patch |  96 ---
 ...ault-to-classic-submission-for-2.2.x.patch |  90 ---
 ...ion-caused-by-mmap-flushing-problems.patch | 104 ---
 ...touch-vbio-after-its-handed-off-to-t.patch |  57 --
 debian/patches/series |  16 +-
 upstream  |   2 +-
 20 files changed, 76 insertions(+), 2689 deletions(-)
 rename 
debian/patches/{0009-arc-stat-summary-guard-access-to-l2arc-MFU-MRU-stats.patch 
=> 0009-arc-stat-summary-guard-access-to-freshly-introduced-.patch} (59%)
 delete mode 100644 
debian/patches/0012-udev-correctly-handle-partition-16-and-later.patch
 delete mode 100644 
debian/patches/0013-Linux-6.8-compat-use-splice_copy_file_range-for-fall.patch
 delete mode 100644 debian/patches/0014-linux-5.4-compat-page_size.patch
 delete mode 100644 debian/patches/0015-abd-add-page-iterator.patch
 delete mode 100644 
debian/patches/0016-vdev_disk-rename-existing-functions-to-vdev_classic_.patch
 delete mode 100644 
debian/patches/0017-vdev_disk-reorganise-vdev_disk_io_start.patch
 delete mode 100644 
debian/patches/0018-vdev_disk-make-read-write-IO-function-configurable.patch
 delete mode 100644 
debian/patches/0019-vdev_disk-rewrite-BIO-filling-machinery-to-avoid-spl.patch
 delete mode 100644 
debian/patches/0020-vdev_disk-add-module-parameter-to-select-BIO-submiss.patch
 delete mode 100644 
debian/patches/0021-vdev_disk-use-bio_chain-to-submit-multiple-BIOs.patch
 delete mode 100644 
debian/patches/0022-abd_iter_page-don-t-use-compound-heads-on-Linux-4.5.patch
 delete mode 100644 
debian/patches/0023-vdev_disk-default-to-classic-submission-for-2.2.x.patch
 delete mode 100644 
debian/patches/0024-Fix-corruption-caused-by-mmap-flushing-problems.patch
 delete mode 100644 
debian/patches/0025-vdev_disk-don-t-touch-vbio-after-its-handed-off-to-t.patch

-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH installer 4/4] assistant: avoid regex for simple prefix matching

2024-05-07 Thread Christoph Heiss 
udev properties are very easy to parse and can be done by doing a
line-based scan and matching the prefix, splitting once for properties.
Avoids the use of regexes and signicantly reduces binary size by about
-38%(!).

Tested by comparing the output of `proxmox-auto-install-assistant
device-info`, running it before and after the changes.

Stripped binary size for release builds:
  before: 3103104 bytes ~ 2.96MiB
  after:  1935744 bytes ~ 1.85MiB

   textdata bss dec hex filename
2906765  187144 537 3094446  2f37ae proxmox-auto-install-assistant-before
1871090   55736 497 1927323  1d689b proxmox-auto-install-assistant-after

No functional changes.

Signed-off-by: Christoph Heiss 
---
 proxmox-auto-install-assistant/Cargo.toml  |  1 -
 proxmox-auto-install-assistant/src/main.rs | 57 +-
 2 files changed, 24 insertions(+), 34 deletions(-)

diff --git a/proxmox-auto-install-assistant/Cargo.toml 
b/proxmox-auto-install-assistant/Cargo.toml
index 0286c80..766b445 100644
--- a/proxmox-auto-install-assistant/Cargo.toml
+++ b/proxmox-auto-install-assistant/Cargo.toml
@@ -15,7 +15,6 @@ anyhow = "1.0"
 clap = { version = "4.0", features = ["derive"] }
 glob = "0.3"
 proxmox-auto-installer = { path = "../proxmox-auto-installer" }
-regex = "1.7"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 toml = "0.7"
diff --git a/proxmox-auto-install-assistant/src/main.rs 
b/proxmox-auto-install-assistant/src/main.rs
index 906f144..828ba76 100644
--- a/proxmox-auto-install-assistant/src/main.rs
+++ b/proxmox-auto-install-assistant/src/main.rs
@@ -1,7 +1,6 @@
 use anyhow::{bail, Result};
 use clap::{Args, Parser, Subcommand, ValueEnum};
 use glob::Pattern;
-use regex::Regex;
 use serde::Serialize;
 use std::{
 collections::BTreeMap,
@@ -401,13 +400,9 @@ fn get_disks() -> Result>> {
 Pattern::new("sr[0-9]*")?,
 ];

-// compile Regex here once and not inside the loop
-let re_disk = Regex::new(r"(?m)^E: DEVTYPE=disk")?;
-let re_cdrom = Regex::new(r"(?m)^E: ID_CDROM")?;
-let re_iso9660 = Regex::new(r"(?m)^E: ID_FS_TYPE=iso9660")?;
-
-let re_name = Regex::new(r"(?m)^N: (.*)$")?;
-let re_props = Regex::new(r"(?m)^E: ([^=]+)=(.*)$")?;
+const PROP_DEVTYP_PREFIX: &str = "E: DEVTYPE=";
+const PROP_CDROM: &str = "E: ID_CDROM";
+const PROP_ISO9660_FS: &str = "E: ID_FS_TYPE=iso9660";

 let mut disks: BTreeMap> = 
BTreeMap::new();

@@ -429,30 +424,27 @@ fn get_disks() -> Result>> {
 }
 };

-if !re_disk.is_match(&output) {
-continue 'outer;
-};
-if re_cdrom.is_match(&output) {
-continue 'outer;
-};
-if re_iso9660.is_match(&output) {
-continue 'outer;
-};
-
 let mut name = filename;
-if let Some(cap) = re_name.captures(&output) {
-if let Some(res) = cap.get(1) {
-name = String::from(res.as_str());
+let mut udev_props: BTreeMap = BTreeMap::new();
+for line in output.lines() {
+if let Some(prop) = line.strip_prefix(PROP_DEVTYP_PREFIX) {
+if prop != "disk" {
+continue 'outer;
+}
 }
-}

-let mut udev_props: BTreeMap = BTreeMap::new();
+if line.starts_with(PROP_CDROM) || 
line.starts_with(PROP_ISO9660_FS) {
+continue 'outer;
+}

-for line in output.lines() {
-if let Some(caps) = re_props.captures(line) {
-let key = String::from(caps.get(1).unwrap().as_str());
-let value = String::from(caps.get(2).unwrap().as_str());
-udev_props.insert(key, value);
+if let Some(prop) = line.strip_prefix("N: ") {
+name = prop.to_owned();
+};
+
+if let Some(prop) = line.strip_prefix("E: ") {
+if let Some((key, val)) = prop.split_once('=') {
+udev_props.insert(key.to_owned(), val.to_owned());
+}
 }
 }

@@ -462,7 +454,6 @@ fn get_disks() -> Result>> {
 }

 fn get_nics() -> Result>> {
-let re_props = Regex::new(r"(?m)^E: (.*)=(.*)$")?;
 let mut nics: BTreeMap> = BTreeMap::new();

 let links = get_nic_list()?;
@@ -480,10 +471,10 @@ fn get_nics() -> Result>> {
 let mut udev_props: BTreeMap = BTreeMap::new();

 for line in output.lines() {
-if let Some(caps) = re_props.captures(line) {
-let key = String::from(caps.get(1).unwrap().as_str());
-let value = String::from(caps.get(2).unwrap().as_str());
-udev_props.insert(key, value);
+if let Some(prop) = line.strip_prefix("E: ") {
+if let Some((key, val)) = prop.split_once('=') {
+udev_props.insert(key.to_owned(), val.to_owned());
+}
 }
 }

--
2.44.0



__

[pve-devel] [PATCH installer 0/4] assistant: clean up glob patterns & regexes

2024-05-07 Thread Christoph Heiss 
The proxmox-auto-install-assistant uses
  - glob patterns for disk matching, which can be pre-compiled for
efficiency
  - regexes for udev property matching, which can be simplified by some
simple prefix matching & splitting on =

The latter also significantly reduces binary size due to the removing
the regex dependency, for details see patch #4.

Overall no functional changes in this series.

Christoph Heiss (4):
  tree-wide: run rustfmt, fix clippy warnings
  assistant: drop unused `log` dependency
  assistant: pre-compile ignored block device patterns
  assistant: avoid regex for simple prefix matching

 proxmox-auto-install-assistant/Cargo.toml |  2 -
 proxmox-auto-install-assistant/src/main.rs| 75 ---
 proxmox-auto-installer/tests/parse-answer.rs  | 14 ++--
 .../src/fetch_plugins/partition.rs| 10 +--
 4 files changed, 45 insertions(+), 56 deletions(-)

--
2.44.0



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH installer 2/4] assistant: drop unused `log` dependency

2024-05-07 Thread Christoph Heiss 
No functional changes.

Signed-off-by: Christoph Heiss 
---
 proxmox-auto-install-assistant/Cargo.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/proxmox-auto-install-assistant/Cargo.toml 
b/proxmox-auto-install-assistant/Cargo.toml
index eaca7f8..0286c80 100644
--- a/proxmox-auto-install-assistant/Cargo.toml
+++ b/proxmox-auto-install-assistant/Cargo.toml
@@ -14,7 +14,6 @@ homepage = "https://www.proxmox.com";
 anyhow = "1.0"
 clap = { version = "4.0", features = ["derive"] }
 glob = "0.3"
-log = "0.4.20"
 proxmox-auto-installer = { path = "../proxmox-auto-installer" }
 regex = "1.7"
 serde = { version = "1.0", features = ["derive"] }
--
2.44.0



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH installer 1/4] tree-wide: run rustfmt, fix clippy warnings

2024-05-07 Thread Christoph Heiss 
No functional changes.

Signed-off-by: Christoph Heiss 
---
 proxmox-auto-installer/tests/parse-answer.rs   | 14 +++---
 .../src/fetch_plugins/partition.rs | 10 +-
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/proxmox-auto-installer/tests/parse-answer.rs 
b/proxmox-auto-installer/tests/parse-answer.rs
index 4014b6d..e77a769 100644
--- a/proxmox-auto-installer/tests/parse-answer.rs
+++ b/proxmox-auto-installer/tests/parse-answer.rs
@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};

 use serde_json::Value;
 use std::fs;
@@ -24,9 +24,9 @@ fn get_answer(path: PathBuf) -> Result {
 Ok(answer)
 }

-fn setup_test_basic(path: &PathBuf) -> (SetupInfo, LocaleInfo, RuntimeInfo, 
UdevInfo) {
+fn setup_test_basic(path: &Path) -> (SetupInfo, LocaleInfo, RuntimeInfo, 
UdevInfo) {
 let installer_info: SetupInfo = {
-let mut path = path.clone();
+let mut path = path.to_path_buf();
 path.push("iso-info.json");

 read_json(&path)
@@ -35,7 +35,7 @@ fn setup_test_basic(path: &PathBuf) -> (SetupInfo, 
LocaleInfo, RuntimeInfo, Udev
 };

 let locale_info = {
-let mut path = path.clone();
+let mut path = path.to_path_buf();
 path.push("locales.json");

 read_json(&path)
@@ -44,7 +44,7 @@ fn setup_test_basic(path: &PathBuf) -> (SetupInfo, 
LocaleInfo, RuntimeInfo, Udev
 };

 let mut runtime_info: RuntimeInfo = {
-let mut path = path.clone();
+let mut path = path.to_path_buf();
 path.push("run-env-info.json");

 read_json(&path)
@@ -53,7 +53,7 @@ fn setup_test_basic(path: &PathBuf) -> (SetupInfo, 
LocaleInfo, RuntimeInfo, Udev
 };

 let udev_info: UdevInfo = {
-let mut path = path.clone();
+let mut path = path.to_path_buf();
 path.push("run-env-udev.json");

 read_json(&path)
@@ -71,7 +71,7 @@ fn setup_test_basic(path: &PathBuf) -> (SetupInfo, 
LocaleInfo, RuntimeInfo, Udev
 fn test_parse_answers() {
 let path = get_test_resource_path().unwrap();
 let (setup_info, locales, runtime_info, udev_info) = 
setup_test_basic(&path);
-let mut tests_path = path.clone();
+let mut tests_path = path;
 tests_path.push("parse_answer");
 let test_dir = fs::read_dir(tests_path.clone()).unwrap();

diff --git a/proxmox-fetch-answer/src/fetch_plugins/partition.rs 
b/proxmox-fetch-answer/src/fetch_plugins/partition.rs
index 0479c8f..7213493 100644
--- a/proxmox-fetch-answer/src/fetch_plugins/partition.rs
+++ b/proxmox-fetch-answer/src/fetch_plugins/partition.rs
@@ -31,7 +31,7 @@ impl FetchFromPartition {
 }

 fn path_exists_logged(file_name: &str, search_path: &str) -> Option {
-let path = Path::new(search_path).join(&file_name);
+let path = Path::new(search_path).join(file_name);
 info!("Testing partition search path {path:?}");
 match path.try_exists() {
 Ok(true) => Some(path),
@@ -51,14 +51,14 @@ fn path_exists_logged(file_name: &str, search_path: &str) 
-> Option {
 fn scan_partlabels(partlabel: &str, search_path: &str) -> Result {
 let partlabel_upper_case = partlabel.to_uppercase();
 if let Some(path) = path_exists_logged(&partlabel_upper_case, search_path) 
{
-info!("Found partition with label '{partlabel_upper_case}'");
-return Ok(path);
+info!("Found partition with label '{partlabel_upper_case}'");
+return Ok(path);
 }

 let partlabel_lower_case = partlabel.to_lowercase();
 if let Some(path) = path_exists_logged(&partlabel_lower_case, search_path) 
{
-info!("Found partition with label '{partlabel_lower_case}'");
-return Ok(path);
+info!("Found partition with label '{partlabel_lower_case}'");
+return Ok(path);
 }

 bail!("Could not detect upper or lower case labels for '{partlabel}'");
--
2.44.0



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH installer 3/4] assistant: pre-compile ignored block device patterns

2024-05-07 Thread Christoph Heiss 
No functional changes.

Signed-off-by: Christoph Heiss 
---
 proxmox-auto-install-assistant/src/main.rs | 18 +-
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/proxmox-auto-install-assistant/src/main.rs 
b/proxmox-auto-install-assistant/src/main.rs
index 0debd29..906f144 100644
--- a/proxmox-auto-install-assistant/src/main.rs
+++ b/proxmox-auto-install-assistant/src/main.rs
@@ -392,13 +392,13 @@ fn inject_file_to_iso(iso: &PathBuf, file: &PathBuf, 
location: &str) -> Result<(
 }

 fn get_disks() -> Result>> {
-let unwantend_block_devs = vec![
-"ram[0-9]*",
-"loop[0-9]*",
-"md[0-9]*",
-"dm-*",
-"fd[0-9]*",
-"sr[0-9]*",
+let unwanted_block_devs = [
+Pattern::new("ram[0-9]*")?,
+Pattern::new("loop[0-9]*")?,
+Pattern::new("md[0-9]*")?,
+Pattern::new("dm-*")?,
+Pattern::new("fd[0-9]*")?,
+Pattern::new("sr[0-9]*")?,
 ];

 // compile Regex here once and not inside the loop
@@ -415,8 +415,8 @@ fn get_disks() -> Result>> {
 let entry = entry.unwrap();
 let filename = entry.file_name().into_string().unwrap();

-for p in &unwantend_block_devs {
-if Pattern::new(p)?.matches(&filename) {
+for p in &unwanted_block_devs {
+if p.matches(&filename) {
 continue 'outer;
 }
 }
--
2.44.0



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server v9 1/3] add C program to get hardware capabilities from CPUID

2024-05-07 Thread Filip Schauer 

Ran this on an Intel(R) Core(TM) i7-7700K CPU at Markus' request to see
how this behaves on an Intel processor. This results in the following
being written to /run/qemu-server/host-hw-capabilities.json:

{ "amd-sev": { "cbitpos": 0, "reduced-phys-bits": 0, "sev-support": 
false, "sev-support-es": false, "sev-support-snp": false } }



PS: The sys/types.h include can be omitted. Otherwise the code looks
fine to me.


On 26/04/2024 11:58, Markus Frank wrote:

Implement a systemd service that runs a C program that extracts AMD
SEV hardware information such as reduced-phys-bios and cbitpos from
CPUID at boot time, looks if SEV, SEV-ES & SEV-SNP are enabled, and
outputs these details as JSON to /run/qemu-server/hw-params.json.

This programm can also be used to read and save other hardware
information at boot time.

Signed-off-by: Markus Frank 
Co-authored-by: Thomas Lamprecht 
---
v9:
* added directory existance check
* print error messages

v8:
* renamed query-machine-params to query-machine-capabilities

v7:
* renamed amd-sev-support to query-machine-params
* mv /run/amd-sev-params to /run/qemu-server/hw-params.json
* add "mkdir /run/qemu-server" to ensure that the directory exists
* moved json content to amd-sev property inside a bigger json
  so that other hardware parameters could also be read at boot time and
  included in this json file.


  Makefile  |  1 +
  query-machine-capabilities/Makefile   | 21 ++
  .../query-machine-capabilities.c  | 71 +++
  .../query-machine-capabilities.service| 12 
  4 files changed, 105 insertions(+)
  create mode 100644 query-machine-capabilities/Makefile
  create mode 100644 query-machine-capabilities/query-machine-capabilities.c
  create mode 100644 
query-machine-capabilities/query-machine-capabilities.service

diff --git a/Makefile b/Makefile
index 133468d..ed67fe0 100644
--- a/Makefile
+++ b/Makefile
@@ -65,6 +65,7 @@ install: $(PKGSOURCES)
install -m 0644 -D bootsplash.jpg $(DESTDIR)/usr/share/$(PACKAGE)
$(MAKE) -C PVE install
$(MAKE) -C qmeventd install
+   $(MAKE) -C query-machine-capabilities install
$(MAKE) -C qemu-configs install
$(MAKE) -C vm-network-scripts install
install -m 0755 qm $(DESTDIR)$(SBINDIR)
diff --git a/query-machine-capabilities/Makefile 
b/query-machine-capabilities/Makefile
new file mode 100644
index 000..c5f6348
--- /dev/null
+++ b/query-machine-capabilities/Makefile
@@ -0,0 +1,21 @@
+DESTDIR=
+PREFIX=/usr
+SBINDIR=${PREFIX}/libexec/qemu-server
+SERVICEDIR=/lib/systemd/system
+
+CC ?= gcc
+CFLAGS += -O2 -fanalyzer -Werror -Wall -Wextra -Wpedantic -Wtype-limits 
-Wl,-z,relro -std=gnu11
+
+query-machine-capabilities: query-machine-capabilities.c
+   $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS)
+
+.PHONY: install
+install: query-machine-capabilities
+   install -d ${DESTDIR}/${SBINDIR}
+   install -d ${DESTDIR}${SERVICEDIR}
+   install -m 0644 query-machine-capabilities.service 
${DESTDIR}${SERVICEDIR}
+   install -m 0755 query-machine-capabilities ${DESTDIR}${SBINDIR}
+
+.PHONY: clean
+clean:
+   rm -f query-machine-capabilities
diff --git a/query-machine-capabilities/query-machine-capabilities.c 
b/query-machine-capabilities/query-machine-capabilities.c
new file mode 100644
index 000..7b8c59f
--- /dev/null
+++ b/query-machine-capabilities/query-machine-capabilities.c
@@ -0,0 +1,71 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+int main() {
+uint32_t eax, ebx, ecx, edx;
+
+// query Encrypted Memory Capabilities, see:
+// 
https://en.wikipedia.org/wiki/CPUID#EAX=801Fh:_Encrypted_Memory_Capabilities
+uint32_t query_function = 0x801F;
+asm volatile("cpuid"
+: "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
+: "0"(query_function)
+);
+
+bool sev_support = (eax & (1<<1)) != 0;
+bool sev_es_support = (eax & (1<<3)) != 0;
+bool sev_snp_support = (eax & (1<<4)) != 0;
+
+uint8_t cbitpos = ebx & 0x3f;
+uint8_t reduced_phys_bits = (ebx >> 6) & 0x3f;
+
+const char *path = "/run/qemu-server/";
+// Check that the directory exists and create it if it does not.
+struct stat statbuf;
+int stats = stat(path, &statbuf);
+if (stats == 0 && S_ISDIR(statbuf.st_mode)) {
+   printf("Directory %s already exists.\n", path);
+} else if (errno == ENOENT) {
+   printf("%s does not exist. Creating directory.\n", path);
+   if (mkdir(path, 0755) != 0) {
+   printf("Error creating directory %s: %s\n", path, strerror(errno));
+   return 1;
+   }
+} else {
+   printf("Error checking path %s: %s\n", path, strerror(errno));
+   return 1;
+}
+
+FILE *file;
+const char *filename = "/run/qemu-server/host-hw-capabilities.json";
+file = fopen(filename, "w");
+if (file == NULL) {
+   perror("Error opening file");
+   return 1;
+