Re: [pve-devel] [PATCH v2 pve-common 0/4] Section Config: Documentation & Code Cleanup

[Max Carrara]

On Tue Jul 2, 2024 at 4:13 PM CEST, Max Carrara wrote:
> Section Config: Documentation & Code Cleanup - v2
> =

Note for reviewers: This series will be superseded in the near future,
so reviewing this isn't necessary at the moment.

>
> Notable Changes Since v1
> 
>
>   * Incorporate @Fabian's feedback regarding the docs [0]
>   * Reword commit message of patch 03
>   * NEW: Patch 04: Make sub `delete_from_config` private
>
> Please see the comments in the individual patches for a detailed list of
> changes.
>
> Older Versions
> --
>
> v1: https://lists.proxmox.com/pipermail/pve-devel/2024-June/064062.html
>
> References
> --
>
> [0]: https://lists.proxmox.com/pipermail/pve-devel/2024-June/064165.html
>
> Summary of Changes
> --
>
> Max Carrara (4):
>   section config: document package and its methods with POD
>   section config: update code style
>   section config: clean up parser logic
>   section config: make subroutine `delete_from_config` private
>
>  src/PVE/SectionConfig.pm | 1200 --
>  1 file changed, 1015 insertions(+), 185 deletions(-)



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH proxmox-perl-rs 21/21] add PVE::RS::Firewall::SDN module

[Max Carrara]

On Wed Jun 26, 2024 at 2:15 PM CEST, Stefan Hanreich wrote:
> Used for obtaining the IPSets that get autogenerated by the nftables
> firewall. The returned configuration has the same format as the
> pve-firewall uses internally, making it compatible with the existing
> pve-firewall code.
>
> Signed-off-by: Stefan Hanreich 
> ---
>  pve-rs/Cargo.toml  |   1 +
>  pve-rs/Makefile|   1 +
>  pve-rs/src/firewall/mod.rs |   1 +
>  pve-rs/src/firewall/sdn.rs | 130 +
>  pve-rs/src/lib.rs  |   1 +
>  5 files changed, 134 insertions(+)
>  create mode 100644 pve-rs/src/firewall/mod.rs
>  create mode 100644 pve-rs/src/firewall/sdn.rs
>
> diff --git a/pve-rs/Cargo.toml b/pve-rs/Cargo.toml
> index e40588d..f612b3a 100644
> --- a/pve-rs/Cargo.toml
> +++ b/pve-rs/Cargo.toml
> @@ -43,3 +43,4 @@ proxmox-subscription = "0.4"
>  proxmox-sys = "0.5"
>  proxmox-tfa = { version = "4.0.4", features = ["api"] }
>  proxmox-time = "2"
> +proxmox-ve-config = { version = "0.1.0" }

This hunk doesn't apply anymore because proxmox-sys was bumped.

Manually adding proxmox-ve-config as depencency works just fine though,
so this needs just a little rebase.

> diff --git a/pve-rs/Makefile b/pve-rs/Makefile
> index c6b4e08..d01da69 100644
> --- a/pve-rs/Makefile
> +++ b/pve-rs/Makefile
> @@ -28,6 +28,7 @@ PERLMOD_GENPACKAGE := /usr/lib/perlmod/genpackage.pl \
>  
>  PERLMOD_PACKAGES := \
> PVE::RS::APT::Repositories \
> +   PVE::RS::Firewall::SDN \
> PVE::RS::OpenId \
> PVE::RS::ResourceScheduling::Static \
> PVE::RS::TFA
> diff --git a/pve-rs/src/firewall/mod.rs b/pve-rs/src/firewall/mod.rs
> new file mode 100644
> index 000..8bd18a8
> --- /dev/null
> +++ b/pve-rs/src/firewall/mod.rs
> @@ -0,0 +1 @@
> +pub mod sdn;
> diff --git a/pve-rs/src/firewall/sdn.rs b/pve-rs/src/firewall/sdn.rs
> new file mode 100644
> index 000..55f3e93
> --- /dev/null
> +++ b/pve-rs/src/firewall/sdn.rs
> @@ -0,0 +1,130 @@
> +#[perlmod::package(name = "PVE::RS::Firewall::SDN", lib = "pve_rs")]
> +mod export {
> +use std::collections::HashMap;
> +use std::{fs, io};
> +
> +use anyhow::{bail, Context, Error};
> +use serde::Serialize;
> +
> +use proxmox_ve_config::{
> +common::Allowlist,
> +firewall::types::ipset::{IpsetAddress, IpsetEntry},
> +firewall::types::Ipset,
> +guest::types::Vmid,
> +sdn::{
> +config::{RunningConfig, SdnConfig},
> +ipam::{Ipam, IpamJson},
> +SdnNameError, VnetName,

SdnNameError isn't used here.

> +},
> +};
> +
> +#[derive(Clone, Debug, Default, Serialize)]
> +pub struct LegacyIpsetEntry {
> +nomatch: bool,
> +cidr: String,
> +comment: Option,
> +}
> +
> +impl LegacyIpsetEntry {
> +pub fn from_ipset_entry(entry: &IpsetEntry) -> Vec 
> {
> +let mut entries = Vec::new();
> +
> +match &entry.address {
> +IpsetAddress::Alias(name) => {
> +entries.push(Self {
> +nomatch: entry.nomatch,
> +cidr: name.to_string(),
> +comment: entry.comment.clone(),
> +});
> +}
> +IpsetAddress::Cidr(cidr) => {
> +entries.push(Self {
> +nomatch: entry.nomatch,
> +cidr: cidr.to_string(),
> +comment: entry.comment.clone(),
> +});
> +}
> +IpsetAddress::Range(range) => {
> +entries.extend(range.to_cidrs().into_iter().map(|cidr| 
> Self {
> +nomatch: entry.nomatch,
> +cidr: cidr.to_string(),
> +comment: entry.comment.clone(),
> +}))
> +}
> +};
> +
> +entries
> +}
> +}
> +
> +#[derive(Clone, Debug, Default, Serialize)]
> +pub struct SdnFirewallConfig {
> +ipset: HashMap>,
> +ipset_comments: HashMap,
> +}
> +
> +impl SdnFirewallConfig {
> +pub fn new() -> Self {
> +Default::default()
> +}
> +
> +pub fn extend_ipsets(&mut self, ipsets: impl IntoIterator Ipset>) {
> +for ipset in ipsets {
> +let entries = ipset
> +.iter()
> +.flat_map(LegacyIpsetEntry::from_ipset_entry)
> +.collect();
> +
> +self.ipset.insert(ipset.name().name().to_string(), entries);
> +
> +if let Some(comment) = &ipset.comment {
> +self.ipset_comments
> +.insert(ipset.name().name().to_string(), 
> comment.to_string());
> +}
> +}
> +}
> +}
> +
> +const SDN_RUNNING_CONFIG: &str = "/etc/pve/sdn/.runn

Re: [pve-devel] [PATCH pve-firewall 19/21] add support for loading sdn firewall configuration

[Max Carrara]

On Wed Jun 26, 2024 at 2:15 PM CEST, Stefan Hanreich wrote:
> Signed-off-by: Stefan Hanreich 
> ---
>  src/PVE/Firewall.pm | 43 +--
>  1 file changed, 41 insertions(+), 2 deletions(-)
>
> diff --git a/src/PVE/Firewall.pm b/src/PVE/Firewall.pm
> index 09544ba..95325a0 100644
> --- a/src/PVE/Firewall.pm
> +++ b/src/PVE/Firewall.pm
> @@ -25,6 +25,7 @@ use PVE::Tools qw($IPV4RE $IPV6RE);
>  use PVE::Tools qw(run_command lock_file dir_glob_foreach);
>  
>  use PVE::Firewall::Helpers;
> +use PVE::RS::Firewall::SDN;
>  
>  my $pvefw_conf_dir = "/etc/pve/firewall";
>  my $clusterfw_conf_filename = "$pvefw_conf_dir/cluster.fw";
> @@ -3644,7 +3645,7 @@ sub lock_clusterfw_conf {
>  }
>  
>  sub load_clusterfw_conf {
> -my ($filename) = @_;
> +my ($filename, $load_sdn_config) = @_;

Small thing:

I would suggest using a prototype here and also accept a hash reference
OR a hash as the last parameter, so that the call signature is a little
more readable.

E.g. right now it's:

load_clusterfw_conf(undef, 1)

VS:

load_clusterfw_conf(undef, { load_sdn_config => 1 })

Or:

load_clusterfw_conf(undef, load_sdn_config => 1)

I know we're gonna phase this whole thing out eventually, but little
things like this help a lot in the long run, IMO. It makes it a little
clearer what the subroutine does at call sites.

I'm not sure if these subroutines are used elsewhere (didn't really
bother to check, sorry), so perhaps you could pass `$filename` via the
hash as well, as an optional parameter. Then it's immediately clear what
*everything* stands for, because a sole `undef` "hides" what's actually
passed to the subroutine.

>  
>  $filename = $clusterfw_conf_filename if !defined($filename);
>  my $empty_conf = {
> @@ -3657,12 +3658,50 @@ sub load_clusterfw_conf {
>   ipset_comments => {},
>  };
>  
> +if ($load_sdn_config) {
> + my $sdn_conf = load_sdn_conf();
> + $empty_conf = { %$empty_conf, %$sdn_conf };
> +}
> +
>  my $cluster_conf = generic_fw_config_parser($filename, $empty_conf, 
> $empty_conf, 'cluster');
>  $set_global_log_ratelimit->($cluster_conf->{options});
>  
>  return $cluster_conf;
>  }
>  
> +sub load_sdn_conf {
> +my $rpcenv = PVE::RPCEnvironment::get();
> +my $authuser = $rpcenv->get_user();
> +
> +my $guests = PVE::Cluster::get_vmlist();
> +my $allowed_vms = [];
> +foreach my $vmid (sort keys %{$guests->{ids}}) {
> + next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);
> + push @$allowed_vms, $vmid;
> +}
> +
> +my $vnets = PVE::Network::SDN::Vnets::config(1);
> +my $privs = [ 'SDN.Audit', 'SDN.Allocate' ];
> +my $allowed_vnets = [];
> +foreach my $vnet (sort keys %{$vnets->{ids}}) {
> + my $zone = $vnets->{ids}->{$vnet}->{zone};
> + next if !$rpcenv->check_any($authuser, "/sdn/zones/$zone/$vnet", 
> $privs, 1);
> + push @$allowed_vnets, $vnet;
> +}
> +
> +my $sdn_config = {
> + ipset => {} ,
> + ipset_comments => {},
> +};
> +
> +eval {
> + $sdn_config = PVE::RS::Firewall::SDN::config($allowed_vnets, 
> $allowed_vms);
> +};
> +warn $@ if $@;
> +
> +return $sdn_config;
> +}
> +
>  sub save_clusterfw_conf {
>  my ($cluster_conf) = @_;
>  
> @@ -4731,7 +4770,7 @@ sub init {
>  sub update {
>  my $code = sub {
>  
> - my $cluster_conf = load_clusterfw_conf();
> + my $cluster_conf = load_clusterfw_conf(undef, 1);
>   my $hostfw_conf = load_hostfw_conf($cluster_conf);
>  
>   if (!is_enabled_and_not_nftables($cluster_conf, $hostfw_conf)) {



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH proxmox-ve-rs 10/21] sdn: add ipam module

[Max Carrara]

On Wed Jun 26, 2024 at 2:15 PM CEST, Stefan Hanreich wrote:
> This module includes structs for representing the JSON schema from the
> PVE ipam. Those can be used to parse the current IPAM state.
>
> We also include a general Ipam struct, and provide a method for
> converting the PVE IPAM to the general struct. The idea behind this
> is that we have multiple IPAM plugins in PVE and will likely add
> support for importing them in the future. With the split, we can have
> our dedicated structs for representing the different data formats from
> the different IPAM plugins and then convert them into a common
> representation that can then be used internally, decoupling the
> concrete plugin from the code using the IPAM configuration.

Big fan of this - as I had already mentioned, I find it always nice to
have different types for such things.

IMO it would be neat if those types were logically grouped though, e.g.
the types for PVE could live in a separate `mod` inside the file. Or, if
you want, you can also convert `ipam.rs` to a `ipam/mod.rs` and add
further files depending on the types of different representations there.

Either solution would make it harder for these types to become
"intermingled" in the future, IMO. So this kind of grouping would simply
serve as a "decent barrier" between the separate representations.
Perhaps the module(s) could then also use a little bit of developer
documentation or something that describes how and why the types are
organized that way.

It's probably best to just add `mod`s for now, as those can be split up
into files later anyway. (Just don't `use super::*;` :P )

>
> Enforcing the invariants the way we currently do adds a bit of runtime
> complexity when building the object, but we get the upside of never
> being able to construct an invalid struct. For the amount of entries
> the ipam usually has, this should be fine. Should it turn out to be
> not performant enough we could always add a HashSet for looking up
> values and speeding up the validation. For now, I wanted to avoid the
> additional complexity.
>
> Signed-off-by: Stefan Hanreich 
> ---
>  .../src/firewall/types/address.rs |   8 +
>  proxmox-ve-config/src/guest/vm.rs |   4 +
>  proxmox-ve-config/src/sdn/ipam.rs | 330 ++
>  proxmox-ve-config/src/sdn/mod.rs  |   2 +
>  4 files changed, 344 insertions(+)
>  create mode 100644 proxmox-ve-config/src/sdn/ipam.rs
>
> diff --git a/proxmox-ve-config/src/firewall/types/address.rs 
> b/proxmox-ve-config/src/firewall/types/address.rs
> index a0b82c5..3ad1a7a 100644
> --- a/proxmox-ve-config/src/firewall/types/address.rs
> +++ b/proxmox-ve-config/src/firewall/types/address.rs
> @@ -61,6 +61,14 @@ impl Cidr {
>  pub fn is_ipv6(&self) -> bool {
>  matches!(self, Cidr::Ipv6(_))
>  }
> +
> +pub fn contains_address(&self, ip: &IpAddr) -> bool {
> +match (self, ip) {
> +(Cidr::Ipv4(cidr), IpAddr::V4(ip)) => cidr.contains_address(ip),
> +(Cidr::Ipv6(cidr), IpAddr::V6(ip)) => cidr.contains_address(ip),
> +_ => false,
> +}
> +}
>  }
>  
>  impl fmt::Display for Cidr {
> diff --git a/proxmox-ve-config/src/guest/vm.rs 
> b/proxmox-ve-config/src/guest/vm.rs
> index a7ea9bb..6a706c7 100644
> --- a/proxmox-ve-config/src/guest/vm.rs
> +++ b/proxmox-ve-config/src/guest/vm.rs
> @@ -17,6 +17,10 @@ static LOCAL_PART: [u8; 8] = [0xFE, 0x80, 0x00, 0x00, 
> 0x00, 0x00, 0x00, 0x00];
>  static EUI64_MIDDLE_PART: [u8; 2] = [0xFF, 0xFE];
>  
>  impl MacAddress {
> +pub fn new(address: [u8; 6]) -> Self {
> +Self(address)
> +}
> +
>  /// generates a link local IPv6-address according to RFC 4291 (Appendix 
> A)
>  pub fn eui64_link_local_address(&self) -> Ipv6Addr {
>  let head = &self.0[..3];
> diff --git a/proxmox-ve-config/src/sdn/ipam.rs 
> b/proxmox-ve-config/src/sdn/ipam.rs
> new file mode 100644
> index 000..682bbe7
> --- /dev/null
> +++ b/proxmox-ve-config/src/sdn/ipam.rs
> @@ -0,0 +1,330 @@
> +use std::{
> +collections::{BTreeMap, HashMap},
> +error::Error,
> +fmt::Display,
> +net::IpAddr,
> +};
> +
> +use serde::Deserialize;
> +
> +use crate::{
> +firewall::types::Cidr,
> +guest::{types::Vmid, vm::MacAddress},
> +sdn::{SdnNameError, SubnetName, ZoneName},
> +};
> +
> +/// struct for deserializing a gateway entry in PVE IPAM
> +///
> +/// They are automatically generated by the PVE SDN module when creating a 
> new subnet.
> +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)]
> +pub struct IpamJsonDataGateway {
> +#[serde(rename = "gateway")]
> +_gateway: u8,
> +}
> +
> +/// struct for deserializing a guest entry in PVE IPAM
> +///
> +/// They are automatically created when adding a guest to a VNet that has a 
> Subnet with DHCP
> +/// configured.
> +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)]
> +pub struct IpamJsonDataVm {
> +  

Re: [pve-devel] [PATCH proxmox-ve-rs 05/21] iprange: add methods for converting an ip range to cidrs

[Max Carrara]

On Wed Jun 26, 2024 at 2:15 PM CEST, Stefan Hanreich wrote:
> This is mainly used in proxmox-perl-rs, so the generated ipsets can be
> used in pve-firewall where only CIDRs are supported.
>
> Signed-off-by: Stefan Hanreich 
> ---
>  .../src/firewall/types/address.rs | 818 ++
>  1 file changed, 818 insertions(+)
>
> diff --git a/proxmox-ve-config/src/firewall/types/address.rs 
> b/proxmox-ve-config/src/firewall/types/address.rs
> index 8db3942..3238601 100644
> --- a/proxmox-ve-config/src/firewall/types/address.rs
> +++ b/proxmox-ve-config/src/firewall/types/address.rs
> @@ -303,6 +303,17 @@ impl IpRange {
>  ) -> Result {
>  Ok(IpRange::V6(AddressRange::new_v6(start, end)?))
>  }
> +
> +/// converts an IpRange into the minimal amount of CIDRs
> +///
> +/// see the concrete implementations of [`AddressRange`] or 
> [`AddressRange`]
> +/// respectively
> +pub fn to_cidrs(&self) -> Vec {
> +match self {
> +IpRange::V4(range) => 
> range.to_cidrs().into_iter().map(Cidr::from).collect(),
> +IpRange::V6(range) => 
> range.to_cidrs().into_iter().map(Cidr::from).collect(),
> +}
> +}
>  }
>  
>  impl std::str::FromStr for IpRange {
> @@ -362,6 +373,71 @@ impl AddressRange {
>  
>  Ok(Self { start, end })
>  }
> +
> +/// returns the minimum amount of CIDRs that exactly represent the range
> +///
> +/// The idea behind this algorithm is as follows:
> +///
> +/// Start iterating with current = start of the IP range
> +///
> +/// Find two netmasks
> +/// * The largest CIDR that the current IP can be the first of
> +/// * The largest CIDR that *only* contains IPs from current - end
> +///
> +/// Add the smaller of the two CIDRs to our result and current to the 
> first IP that is in
> +/// the range but not in the CIDR we just added. Proceed until we 
> reached the end of the IP
> +/// range.

Would mybe prefer some more inline formatting / minor rewording
regarding the algorithm's steps above, simply for readability's sake
(e.g. when rendering the docs).

Sort of like:


1. Start iteration: Set `current` to `start` of the IP range

2. Find two netmasks:
  - The largest CIDR that the `current` IP can be the first of
  - The largest CIDR that *only* contains IPs from `current` to `end`

3. Add the smaller of the two CIDRs to our result and `current` to the first IP 
that is in
the range but *not* in the CIDR we just added. Proceed until we reached the end 
of the IP
range.


Again, just a small thing, but thought I'd mention it.

> +///
> +pub fn to_cidrs(&self) -> Vec {
> +let mut cidrs = Vec::new();
> +
> +let mut current = u32::from_be_bytes(self.start.octets());
> +let end = u32::from_be_bytes(self.end.octets());
> +
> +if current == end {
> +// valid Ipv4 since netmask is 32
> +cidrs.push(Ipv4Cidr::new(current, 32).unwrap());
> +return cidrs;
> +}
> +
> +// special case this, since this is the only possibility of overflow
> +// when calculating delta_min_mask - makes everything a lot easier
> +if current == u32::MIN && end == u32::MAX {
> +// valid Ipv4 since it is `0.0.0.0/0`
> +cidrs.push(Ipv4Cidr::new(current, 0).unwrap());
> +return cidrs;
> +}
> +
> +while current <= end {
> +// netmask of largest CIDR that current IP can be the first of
> +// cast is safe, because trailing zeroes can at most be 32
> +let current_max_mask = IPV4_LENGTH - (current.trailing_zeros() 
> as u8);
> +
> +// netmask of largest CIDR that *only* contains IPs of the 
> remaining range
> +// is at most 32 due to unwrap_or returning 32 and ilog2 being 
> at most 31
> +let delta_min_mask = ((end - current) + 1) // safe due to 
> special case above
> +.checked_ilog2() // should never occur due to special case, 
> but for good measure
> +.map(|mask| IPV4_LENGTH - mask as u8)
> +.unwrap_or(IPV4_LENGTH);
> +
> +// at most 32, due to current/delta being at most 32
> +let netmask = u8::max(current_max_mask, delta_min_mask);
> +
> +// netmask is at most 32, therefore safe to unwrap
> +cidrs.push(Ipv4Cidr::new(current, netmask).unwrap());
> +
> +let delta = 2u32.saturating_pow((IPV4_LENGTH - netmask).into());
> +
> +if let Some(result) = current.checked_add(delta) {
> +current = result
> +} else {
> +// we reached the end of IP address space
> +break;
> +}
> +}
> +
> +cidrs
> +}
>  }
>  
>  impl AddressRange {
> @@ -377,6 +453,61 @@ impl AddressRange {
>  
>  Ok(Self { start, end })
>  }
> +
> +/// returns the minimum amount 

Re: [pve-devel] [PATCH proxmox-ve-rs 02/21] firewall: add ip range types

[Max Carrara]

On Wed Jun 26, 2024 at 2:15 PM CEST, Stefan Hanreich wrote:
> Currently we are using tuples to represent IP ranges which is
> suboptimal. Validation logic and invariant checking needs to happen at
> every site using the IP range rather than having a unified struct for
> enforcing those invariants.

That's something I completely support; as you know I'm a fan of
representing state / invariants / etc. via types ;)

>
> Signed-off-by: Stefan Hanreich 
> ---
>  .../src/firewall/types/address.rs | 230 +-
>  1 file changed, 228 insertions(+), 2 deletions(-)
>
> diff --git a/proxmox-ve-config/src/firewall/types/address.rs 
> b/proxmox-ve-config/src/firewall/types/address.rs
> index e48ac1b..ddf4652 100644
> --- a/proxmox-ve-config/src/firewall/types/address.rs
> +++ b/proxmox-ve-config/src/firewall/types/address.rs
> @@ -1,9 +1,9 @@
> -use std::fmt;
> +use std::fmt::{self, Display};
>  use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
>  use std::ops::Deref;
>  
>  use anyhow::{bail, format_err, Error};
> -use serde_with::DeserializeFromStr;
> +use serde_with::{DeserializeFromStr, SerializeDisplay};
>  
>  #[derive(Clone, Copy, Debug, Eq, PartialEq)]
>  pub enum Family {
> @@ -239,6 +239,202 @@ impl> From for Ipv6Cidr {
>  }
>  }
>  
> +#[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)]
> +pub enum IpRangeError {
> +MismatchedFamilies,
> +StartGreaterThanEnd,
> +InvalidFormat,
> +}
> +
> +impl std::error::Error for IpRangeError {}
> +
> +impl Display for IpRangeError {
> +fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
> +f.write_str(match self {
> +IpRangeError::MismatchedFamilies => "mismatched ip address 
> families",
> +IpRangeError::StartGreaterThanEnd => "start is greater than end",
> +IpRangeError::InvalidFormat => "invalid ip range format",
> +})
> +}
> +}
> +
> +/// represents a range of IPv4 or IPv6 addresses

Small thing: I'd prefer

Represents a range of IPv4 or IPv6 addresses.

  instead.

Mainly because most docstrings are written that way, and you do it in
later patches in a couple places as well anyways. Just gonna mention
this here once as you do this a couple times in this series in order to
avoid unnecessary noise.

IMO it's a really minor thing, but since you told me off-list that
you're in the process of neatly documenting everything, I thought I'd
mention it here.

> +///
> +/// For more information see [`AddressRange`]
> +#[derive(Clone, Copy, Debug, PartialEq, Eq, SerializeDisplay, 
> DeserializeFromStr)]
> +pub enum IpRange {
> +V4(AddressRange),
> +V6(AddressRange),
> +}
> +
> +impl IpRange {
> +/// returns the family of the IpRange
> +pub fn family(&self) -> Family {
> +match self {
> +IpRange::V4(_) => Family::V4,
> +IpRange::V6(_) => Family::V6,
> +}
> +}
> +
> +/// creates a new [`IpRange`] from two [`IpAddr`]
> +///
> +/// # Errors
> +///
> +/// This function will return an error if start and end IP address are 
> not from the same family.
> +pub fn new(start: impl Into, end: impl Into) -> 
> Result {
> +match (start.into(), end.into()) {
> +(IpAddr::V4(start), IpAddr::V4(end)) => Self::new_v4(start, end),
> +(IpAddr::V6(start), IpAddr::V6(end)) => Self::new_v6(start, end),
> +_ => Err(IpRangeError::MismatchedFamilies),
> +}
> +}
> +
> +/// construct a new Ipv4 Range
> +pub fn new_v4(
> +start: impl Into,
> +end: impl Into,
> +) -> Result {
> +Ok(IpRange::V4(AddressRange::new_v4(start, end)?))
> +}
> +
> +pub fn new_v6(
> +start: impl Into,
> +end: impl Into,
> +) -> Result {
> +Ok(IpRange::V6(AddressRange::new_v6(start, end)?))
> +}
> +}
> +
> +impl std::str::FromStr for IpRange {
> +type Err = IpRangeError;
> +
> +fn from_str(s: &str) -> Result {
> +if let Ok(range) = s.parse() {
> +return Ok(IpRange::V4(range));
> +}
> +
> +if let Ok(range) = s.parse() {
> +return Ok(IpRange::V6(range));
> +}
> +
> +Err(IpRangeError::InvalidFormat)
> +}
> +}
> +
> +impl fmt::Display for IpRange {
> +fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
> +match self {
> +IpRange::V4(range) => range.fmt(f),
> +IpRange::V6(range) => range.fmt(f),
> +}
> +}
> +}
> +
> +/// represents a range of IP addresses from start to end
> +///
> +/// This type is for encapsulation purposes for the [`IpRange`] enum and 
> should be instantiated via
> +/// that enum.
> +///
> +/// # Invariants
> +///
> +/// * start and end have the same IP address family
> +/// * start is lesser than or equal to end
> +///
> +/// # Textual representation
> +///
> +/// Two IP addresses separated by a hyphen, e.g.: `127.0.0.1-127.0.0.255`
> +#[derive(Clone, Copy, Debug,

Re: [pve-devel] [PATCH proxmox-ve-rs 01/21] debian: add files for packaging

[Max Carrara]

On Wed Jun 26, 2024 at 2:15 PM CEST, Stefan Hanreich wrote:
> Since we now have a standalone repository for Proxmox VE related
> crates, add the required files for packaging the crates contained in
> this repository.
>
> Signed-off-by: Stefan Hanreich 
> ---
>  .cargo/config.toml |  5 ++
>  .gitignore |  8 +++
>  Cargo.toml | 17 +++
>  Makefile   | 69 ++
>  build.sh   | 35 +
>  bump.sh| 44 
>  proxmox-ve-config/Cargo.toml   | 16 +++---
>  proxmox-ve-config/debian/changelog |  5 ++
>  proxmox-ve-config/debian/control   | 43 
>  proxmox-ve-config/debian/copyright | 19 +++
>  proxmox-ve-config/debian/debcargo.toml |  4 ++
>  11 files changed, 255 insertions(+), 10 deletions(-)
>  create mode 100644 .cargo/config.toml
>  create mode 100644 .gitignore
>  create mode 100644 Cargo.toml
>  create mode 100644 Makefile
>  create mode 100755 build.sh
>  create mode 100755 bump.sh
>  create mode 100644 proxmox-ve-config/debian/changelog
>  create mode 100644 proxmox-ve-config/debian/control
>  create mode 100644 proxmox-ve-config/debian/copyright
>  create mode 100644 proxmox-ve-config/debian/debcargo.toml
>
> diff --git a/.cargo/config.toml b/.cargo/config.toml
> new file mode 100644
> index 000..3b5b6e4
> --- /dev/null
> +++ b/.cargo/config.toml
> @@ -0,0 +1,5 @@
> +[source]
> +[source.debian-packages]
> +directory = "/usr/share/cargo/registry"
> +[source.crates-io]
> +replace-with = "debian-packages"
> diff --git a/.gitignore b/.gitignore
> new file mode 100644
> index 000..d72b68b
> --- /dev/null
> +++ b/.gitignore
> @@ -0,0 +1,8 @@
> +/target
> +/*/target
> +Cargo.lock
> +**/*.rs.bk
> +/*.buildinfo
> +/*.changes
> +/build
> +/*-deb
> diff --git a/Cargo.toml b/Cargo.toml
> new file mode 100644
> index 000..ab23d89
> --- /dev/null
> +++ b/Cargo.toml
> @@ -0,0 +1,17 @@
> +[workspace]
> +members = [
> +"proxmox-ve-config",
> +]
> +exclude = [
> +"build",
> +]
> +resolver = "2"
> +
> +[workspace.package]
> +authors = ["Proxmox Support Team "]
> +edition = "2021"
> +license = "AGPL-3"
> +homepage = "https://proxmox.com";
> +exclude = [ "debian" ]
> +rust-version = "1.70"
> +
> diff --git a/Makefile b/Makefile
> new file mode 100644
> index 000..0da9b74
> --- /dev/null
> +++ b/Makefile
> @@ -0,0 +1,69 @@
> +# Shortcut for common operations:
> +
> +CRATES != echo proxmox-*/Cargo.toml | sed -e 's|/Cargo.toml||g'
> +
> +# By default we just run checks:
> +.PHONY: all
> +all: check
> +
> +.PHONY: deb
> +deb: $(foreach c,$(CRATES), $c-deb)
> + echo $(foreach c,$(CRATES), $c-deb)
> + lintian build/*.deb
> +
> +.PHONY: dsc
> +dsc: $(foreach c,$(CRATES), $c-dsc)
> + echo $(foreach c,$(CRATES), $c-dsc)
> + lintian build/*.dsc
> +
> +.PHONY: autopkgtest
> +autopkgtest: $(foreach c,$(CRATES), $c-autopkgtest)
> +
> +.PHONY: dinstall
> +dinstall:
> + $(MAKE) clean
> + $(MAKE) deb
> + sudo -k dpkg -i build/librust-*.deb
> +
> +%-deb:
> + ./build.sh $*
> + touch $@
> +
> +%-dsc:
> + BUILDCMD='dpkg-buildpackage -S -us -uc -d' ./build.sh $*
> + touch $@
> +
> +%-autopkgtest:
> + autopkgtest build/$* build/*.deb -- null
> + touch $@
> +
> +.PHONY: check
> +check:
> + cargo test
> +
> +# Prints a diff between the current code and the one rustfmt would produce
> +.PHONY: fmt
> +fmt:
> + cargo +nightly fmt -- --check
> +
> +# Doc without dependencies
> +.PHONY: doc
> +doc:
> + cargo doc --no-deps
> +
> +.PHONY: clean
> +clean:
> + cargo clean
> + rm -rf build/
> + rm -f -- *-deb *-dsc *-autopkgtest *.build *.buildinfo *.changes
> +
> +.PHONY: update
> +update:
> + cargo update
> +
> +%-upload: %-deb
> + cd build; \
> + dcmd --deb rust-$*_*.changes \
> + | grep -v '.changes$$' \
> + | tar -cf "$@.tar" -T-; \
> + cat "$@.tar" | ssh -X repo...@repo.proxmox.com upload --product 
> devel --dist bookworm
> diff --git a/build.sh b/build.sh
> new file mode 100755
> index 000..39a8302
> --- /dev/null
> +++ b/build.sh
> @@ -0,0 +1,35 @@
> +#!/bin/sh
> +
> +set -eux
> +
> +export CARGO=/usr/bin/cargo
> +export RUSTC=/usr/bin/rustc
> +
> +CRATE=$1
> +BUILDCMD=${BUILDCMD:-"dpkg-buildpackage -b -uc -us"}
> +
> +mkdir -p build
> +echo system >build/rust-toolchain
> +rm -rf "build/${CRATE}"
> +
> +CONTROL="$PWD/${CRATE}/debian/control"
> +
> +if [ -e "$CONTROL" ]; then
> +# check but only warn, debcargo fails anyway if crates are missing
> +dpkg-checkbuilddeps $PWD/${CRATE}/debian/control || true
> +# rm -f "$PWD/${CRATE}/debian/control"
> +fi
> +
> +debcargo package \
> +--config "$PWD/${CRATE}/debian/debcargo.toml" \
> +--changelog-ready \
> +--no-overlay-write-back \
> +--directory "$PWD/build/${CRATE}" \
> +"${CRATE}" \
> + 

Re: [pve-devel] [RFC firewall/proxmox{-ve-rs, -firewall, -perl-rs} 00/21] autogenerate ipsets for sdn objects

[Max Carrara]

-102`,
also blocks the VM's outgoing web traffic, as expected.

- Output of `iptables-save | grep -E '\-\-dport (80|443) '` on the node:

# iptables-save | grep -E '\-\-dport (80|443) ' 



  [17:59:48]
-A tap102i0-OUT -p tcp -m set --match-set PVEFW-0-guest-ipam-102-v4 src -m tcp 
--dport 80 -m limit --limit 1/sec -j NFLOG --nflog-prefix ":102:6:tap102i0-OUT: 
REJECT: "
-A tap102i0-OUT -p tcp -m set --match-set PVEFW-0-guest-ipam-102-v4 src -m tcp 
--dport 80 -j PVEFW-reject
-A tap102i0-OUT -p tcp -m set --match-set PVEFW-0-guest-ipam-102-v4 src -m tcp 
--dport 443 -m limit --limit 1/sec -j NFLOG --nflog-prefix 
":102:6:tap102i0-OUT: REJECT: "
-A tap102i0-OUT -p tcp -m set --match-set PVEFW-0-guest-ipam-102-v4 src -m tcp 
--dport 443 -j PVEFW-reject

Code Review
---

There's not much to say here except that the code looks fantastic as
always - I'm especially a fan of the custom error types. As Gabriel
mentioned, maybe the `thiserror` crate would come in handy eventually,
but that's honestly your decision.

There are a couple more comments inline, but they're rather minor, IMO.

Slightly off-topic, but because you already mentioned off-list that
you're working on updating / adding docstrings and implementing custom
error types for the other things as well, I've got no more things to
mention.

Great work! I really like this feature a lot.

Reviewed-by: Max Carrara 
Tested-by: Max Carrara 

>
> Additionally it generates an IPSet for every guest that has one or more IPAM
> entries in the pve IPAM.
>
> Those can then be used in the cluster / host / guest firewalls. Firewall rules
> automatically update on changes of the SDN / IPAM configuration. This patch
> series works for the old firewall as well as the new firewall.
>
> The ipsets in nftables currently get generated as named ipsets in every table,
> this means that the `nft list ruleset` output can get quite crowded for large
> SDN configurations or large IPAM databases. Another option would be to only
> include them as anonymous IPsets in the rules, which would make the nft output
> far less crowded but this way would use more memory when making extensive use 
> of
> the sdn ipsets, since everytime it is used in a rule we create an entirely new
> ipset.
>
> The current series generates all those ipsets in the datacenter scope. My
> initial approach was to introduce an separate scope (sdn/), but I changed my
> mind during the development because that would require non-trivial changes in
> pve-firewall, which is something I wanted to avoid. With this approach we just
> pass a flag to the cluster config loading wherever we need the SDN config - we
> get everything else (rule validation, API output, rule generation) for 'free'
> basically.
>
> Otherwise, the other way I see would need to introduce a completely new
> parameter into all function calls, or at least a new key in the dc config. All
> call sites would need privileges, due to the IPAM being in /etc/pve/priv. We
> would need to parse the SDN configuration everywhere we need the cluster
> configuration, since otherwise we wouldn't be able to parse / validate the
> cluster configuration and then generate rules.
>
> I'm still unsure whether the upside of having a separate scope is worth the
> effort, so any input w.r.t this topic is much appreciated. Introducing a new
> scope and then adapting the firewall is something I wanted to get some 
> feedback
> on before diving into it, which is why I've refrained from doing it for now.
>
> Of course one downside is that we're kinda locking us in here with this
> decision. With the new firewall adding new scopes should be a lot easier, but 
> if
> we decide to go forward with the SDN ipsets in the datacenter scope we would
> need to support that as well or find some migration path.
>
>
> This patch series is based on my private repositories that split the existing
> proxmox-firewall package into proxmox-firewall and proxmox-ve-rs. Those can be
> found in my staff repo:
>
> staff/s.hanreich/proxmox-ve-rs.git master
> staff/s.hanreich/proxmox-firewall.git no-config
>
> Please note that I included the debian packaging commit in this patch series,
> since it is new and should get reviewed as well, I suppose. It is already
> included when pulling from the proxmox-ve-rs repository.
>
> Dependencies:
> * proxmox-perl-rs and proxmox-firewall depend on proxmox-ve-rs
> * pve-firewall depends on proxmox-perl-rs
>
> proxmox-ve-rs:
>
> Stefan Hanreich (15):
>   debian: add files for packaging
>   firewall: add ip range types
>   firewall: address: use new iprange type for ip entries
>   ipset: add range variant to a

[pve-devel] [PATCH v1 pve-common 18/18] deb: split PBSClient.pm into new package libproxmox-backup-client-perl

[Max Carrara]

.. and also add corresponding 'debian/*.install' files for
'libpve-common-perl' and the new package.

Signed-off-by: Max Carrara 
---
 Makefile |  1 +
 debian/control   | 12 +
 debian/libproxmox-backup-client-perl.install |  1 +
 debian/libpve-common-perl.install| 28 
 4 files changed, 42 insertions(+)
 create mode 100644 debian/libproxmox-backup-client-perl.install
 create mode 100644 debian/libpve-common-perl.install

diff --git a/Makefile b/Makefile
index c2969aa..28d7214 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ SOURCE_PACKAGE=libpve-common-perl
 
 PACKAGES = \
$(SOURCE_PACKAGE) \
+   libproxmox-backup-client-perl \
 
 
 ARCH=all
diff --git a/debian/control b/debian/control
index ac4cd66..11a1e17 100644
--- a/debian/control
+++ b/debian/control
@@ -52,3 +52,15 @@ Breaks: ifupdown2 (<< 2.0.1-1+pve5),
 qemu-server (<< 8.0.1),
 Description: Proxmox VE base library
  This package contains the base library used by other Proxmox VE components.
+
+Package: libproxmox-backup-client-perl
+Architecture: all
+Depends: libpve-common-perl (>= X.Y.Z),
+proxmox-backup-client (>= 2.1.10~),
+proxmox-backup-file-restore,
+${perl:Depends}
+Replaces: libpve-common-perl (<< X.Y.Z)
+Breaks: libpve-common-perl (<< X.Y.Z)
+Description: Proxmox Backup Client Perl Library
+ This package contains utilities that wrap common Proxmox Backup client CLI
+ operations.
diff --git a/debian/libproxmox-backup-client-perl.install 
b/debian/libproxmox-backup-client-perl.install
new file mode 100644
index 000..a54f59b
--- /dev/null
+++ b/debian/libproxmox-backup-client-perl.install
@@ -0,0 +1 @@
+/usr/share/perl5/PVE/PBSClient.pm
diff --git a/debian/libpve-common-perl.install 
b/debian/libpve-common-perl.install
new file mode 100644
index 000..bf42f42
--- /dev/null
+++ b/debian/libpve-common-perl.install
@@ -0,0 +1,28 @@
+/usr/share/perl5/PVE/AtomicFile.pm
+/usr/share/perl5/PVE/CalendarEvent.pm
+/usr/share/perl5/PVE/Certificate.pm
+/usr/share/perl5/PVE/CGroup.pm
+/usr/share/perl5/PVE/CLIFormatter.pm
+/usr/share/perl5/PVE/CLIHandler.pm
+/usr/share/perl5/PVE/CpuSet.pm
+/usr/share/perl5/PVE/Daemon.pm
+/usr/share/perl5/PVE/Exception.pm
+/usr/share/perl5/PVE/Format.pm
+/usr/share/perl5/PVE/INotify.pm
+/usr/share/perl5/PVE/Job/
+/usr/share/perl5/PVE/Job/Registry.pm
+/usr/share/perl5/PVE/JSONSchema.pm
+/usr/share/perl5/PVE/LDAP.pm
+/usr/share/perl5/PVE/Network.pm
+/usr/share/perl5/PVE/OTP.pm
+/usr/share/perl5/PVE/ProcFSTools.pm
+/usr/share/perl5/PVE/PTY.pm
+/usr/share/perl5/PVE/RESTEnvironment.pm
+/usr/share/perl5/PVE/RESTHandler.pm
+/usr/share/perl5/PVE/SafeSyslog.pm
+/usr/share/perl5/PVE/SectionConfig.pm
+/usr/share/perl5/PVE/Syscall.pm
+/usr/share/perl5/PVE/SysFSTools.pm
+/usr/share/perl5/PVE/Systemd.pm
+/usr/share/perl5/PVE/Ticket.pm
+/usr/share/perl5/PVE/Tools.pm
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 16/18] pbsclient: don't return anything in `forget_snapshot`

[Max Carrara]

The actual `proxmox-backup-client forget` command doesn't return
anything (despite the `--output-format json` flag), which is why this
method ends up returning `undef` on successful calls (due to the JSON
parse in `run_client_cmd` returning `undef`).

Therefore, make it explicit that nothing is returned to make it more
clear what the code does.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 4adb04c..aeceed3 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -681,7 +681,8 @@ sub forget_snapshot {
 
 (my $namespace, $snapshot) = split_namespaced_parameter($self, $snapshot);
 
-return run_client_cmd($self, 'forget', [ "$snapshot" ], 1, undef, 
$namespace)
+run_client_cmd($self, 'forget', [ "$snapshot" ], 1, undef, $namespace);
+return;
 };
 
 =pod
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 15/18] pbsclient: don't return anything in PXAR methods

[Max Carrara]

Due to the previously removed implicit return in `do_raw_client_cmd`,
the `run_raw_client_cmd` and consequently also the `backup_fs_tree`
and `restore_pxar` methods would return `0` on success if they didn't
`die` otherwise.

Therefore, because now nothing is actually returned, explicitly don't
return anything in order to prevent returning something implicitly in
the future (if e.g. `run_raw_client_cmd` is made to return something).

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 4ffdd04..4adb04c 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -605,7 +605,8 @@ sub backup_fs_tree {
$cmd_opts->{namespace} = $namespace;
 }
 
-return run_raw_client_cmd($self, 'backup', $param, %$cmd_opts);
+run_raw_client_cmd($self, 'backup', $param, %$cmd_opts);
+return;
 };
 
 =pod
@@ -653,7 +654,8 @@ sub restore_pxar {
 
 $cmd_opts->{namespace} = $namespace;
 
-return run_raw_client_cmd($self, 'restore', $param, %$cmd_opts);
+run_raw_client_cmd($self, 'restore', $param, %$cmd_opts);
+return;
 };
 
 =pod
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 12/18] pbsclient: throw exception if username of client has no realm

[Max Carrara]

.. in order to catch that error earlier, rather than having
`proxmox-backup-client` err and complain that the "repository value
doesn't match the regex pattern".

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 26f73ef..e7e9a79 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -108,6 +108,8 @@ sub get_repository {
 
 my $username = $scfg->{username} // 'root@pam';
 
+die "no realm given for username '$username'" if $username !~ m/@/;
+
 return "$username\@$server:$datastore";
 }
 
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 17/18] make: support building multiple packages from the same source

[Max Carrara]

This also makes sure that `lintian` actually lints any new packages.

Signed-off-by: Max Carrara 
---
 Makefile | 26 +++---
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 637cd49..c2969aa 100644
--- a/Makefile
+++ b/Makefile
@@ -1,20 +1,24 @@
 include /usr/share/dpkg/pkg-info.mk
 
-PACKAGE=libpve-common-perl
+SOURCE_PACKAGE=libpve-common-perl
+
+PACKAGES = \
+   $(SOURCE_PACKAGE) \
+
 
 ARCH=all
 
-BUILDDIR ?= $(PACKAGE)-$(DEB_VERSION_UPSTREAM)
+BUILDDIR ?= $(SOURCE_PACKAGE)-$(DEB_VERSION_UPSTREAM)
 
-DEB=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION)_$(ARCH).deb
-DSC=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION).dsc
+DEBS = $(addsuffix _$(DEB_VERSION_UPSTREAM_REVISION)_$(ARCH).deb,$(PACKAGES))
+DSC=$(SOURCE_PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION).dsc
 
 all:
$(MAKE) -C src
 
 .PHONY: dinstall
 dinstall: deb
-   dpkg -i $(DEB)
+   dpkg -i $(DEBS)
 
 $(BUILDDIR): src debian test
rm -rf $(BUILDDIR) $(BUILDDIR).tmp; mkdir $(BUILDDIR).tmp
@@ -23,10 +27,10 @@ $(BUILDDIR): src debian test
mv $(BUILDDIR).tmp $(BUILDDIR)
 
 .PHONY: deb
-deb: $(DEB)
-$(DEB): $(BUILDDIR)
+deb: $(DEBS)
+$(DEBS): $(BUILDDIR)
cd $(BUILDDIR); dpkg-buildpackage -b -us -uc
-   lintian $(DEB)
+   lintian $(DEBS)
 
 .PHONY: dsc
 dsc: $(DSC)
@@ -40,7 +44,7 @@ sbuild: $(DSC)
 .PHONY: clean distclean
 distclean: clean
 clean:
-   rm -rf *~ *.deb *.changes $(PACKAGE)-[0-9]*/ *.buildinfo *.build *.dsc 
*.tar.?z
+   rm -rf *~ *.deb *.changes $(SOURCE_PACKAGE)-[0-9]*/ *.buildinfo *.build 
*.dsc *.tar.?z
 
 .PHONY: check
 check:
@@ -52,5 +56,5 @@ install:
 
 .PHONY: upload
 upload: UPLOAD_DIST ?= $(DEB_DISTRIBUTION)
-upload: $(DEB)
-   tar cf - $(DEB)|ssh -X repo...@repo.proxmox.com -- upload --product 
pve,pmg --dist $(UPLOAD_DIST)
+upload: $(DEBS)
+   tar cf - $(DEBS)|ssh -X repo...@repo.proxmox.com -- upload --product 
pve,pmg --dist $(UPLOAD_DIST)
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 10/18] pbsclient: use `File::Spec->catfile` to concatenate file paths

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 19 +++
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 2084bb5..69b4e40 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -5,6 +5,7 @@ use warnings;
 
 use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC);
 use File::Path;
+use File::Spec;
 use File::Temp qw(tempdir);
 use IO::File;
 use JSON;
@@ -181,7 +182,7 @@ sub new {
 my sub password_file_name {
 my ($self) = @_;
 
-return "$self->{secret_dir}/$self->{storeid}.pw";
+return File::Spec->catfile($self->{secret_dir}, "$self->{storeid}.pw");
 }
 
 =pod
@@ -267,7 +268,7 @@ I it is located in.
 sub encryption_key_file_name {
 my ($self) = @_;
 
-return "$self->{secret_dir}/$self->{storeid}.enc";
+return File::Spec->catfile($self->{secret_dir}, "$self->{storeid}.enc");
 };
 
 =pod
@@ -354,7 +355,7 @@ my sub do_raw_client_cmd {
 my $client_bin = delete($opts{binary}) || 'proxmox-backup-client';
 my $use_crypto = $USE_CRYPT_PARAMS->{$client_bin}->{$client_cmd} // 0;
 
-my $client_exe = "/usr/bin/$client_bin";
+my $client_exe = File::Spec->catfile("/usr/bin", $client_bin);
 die "executable not found '$client_exe'! $client_bin not installed?\n" if 
! -x $client_exe;
 
 my $scfg = $self->{scfg};
@@ -865,18 +866,20 @@ sub file_restore_extract_prepare {
 my ($self) = @_;
 
 my $tmpdir = tempdir();
-mkfifo("$tmpdir/fifo", 0600)
-   or die "creating file download fifo '$tmpdir/fifo' failed: $!\n";
+my $fifo_path = File::Spec->catfile($tmpdir, "fifo");
+
+mkfifo($fifo_path, 0600)
+   or die "creating file download fifo '$fifo_path' failed: $!\n";
 
 # allow reading data for proxy user
 my $wwwid = getpwnam('www-data') ||
die "getpwnam failed";
 chown($wwwid, -1, "$tmpdir")
or die "changing permission on fifo dir '$tmpdir' failed: $!\n";
-chown($wwwid, -1, "$tmpdir/fifo")
-   or die "changing permission on fifo '$tmpdir/fifo' failed: $!\n";
+chown($wwwid, -1, $fifo_path)
+   or die "changing permission on fifo '$fifo_path' failed: $!\n";
 
-return "$tmpdir/fifo";
+return $fifo_path;
 }
 
 =pod
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 09/18] pbsclient: create secret dir with `mkdir -p` and mode `700`

[Max Carrara]

.. instead of using a regular `mkdir` call.

The `File::Path::make_path` subroutine is used for this purpose, which
recursively creates all directories if they didn't exist before. Upon
creation of those directories, the mode is also set to `700`.

This means that (like before), directory permissions are left
untouched if the directory existed already.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 15 +++
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index e0468d3..2084bb5 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -4,6 +4,7 @@ use strict;
 use warnings;
 
 use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC);
+use File::Path;
 use File::Temp qw(tempdir);
 use IO::File;
 use JSON;
@@ -191,7 +192,8 @@ my sub password_file_name {
 
 Updates or creates the I file, storing the given C<$password>.
 
-If the I does not exist, it is created beforehand.
+If the I does not exist, it is recursively created with the
+permissions C<700> beforehand.
 
 If the I file does not exist, a new one with the permissions C<600>
 is created.
@@ -202,7 +204,9 @@ sub set_password {
 my ($self, $password) = @_;
 
 my $pwfile = password_file_name($self);
-mkdir($self->{secret_dir});
+File::Path::make_path($self->{secret_dir}, {
+   mode => 0700,
+});
 
 PVE::Tools::file_set_contents($pwfile, "$password\n", 0600);
 };
@@ -274,7 +278,8 @@ sub encryption_key_file_name {
 
 Updates or creates the I file, storing the given C<$key>.
 
-If the I does not exist, it is created beforehand.
+If the I does not exist, it is recursively created with the
+permissions C<700> beforehand.
 
 If the I file does not exist, a new one with the permissions 
C<600>
 is created.
@@ -285,7 +290,9 @@ sub set_encryption_key {
 my ($self, $key) = @_;
 
 my $encfile = $self->encryption_key_file_name();
-mkdir($self->{secret_dir});
+File::Path::make_path($self->{secret_dir}, {
+   mode => 0700,
+});
 
 PVE::Tools::file_set_contents($encfile, "$key\n", 0600);
 };
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 08/18] pbsclient: document package and its public functions & methods

[Max Carrara]

This commit adds a brief overview for the `PVE::PBSClient` package and
documents its public functions and methods. Examples are added where
deemed appropriate.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 526 +--
 1 file changed, 511 insertions(+), 15 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 231406a..e0468d3 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -1,5 +1,4 @@
 package PVE::PBSClient;
-# utility functions for interaction with Proxmox Backup client CLI executable
 
 use strict;
 use warnings;
@@ -13,14 +12,83 @@ use POSIX qw(mkfifo strftime ENOENT);
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::Tools qw(run_command file_set_contents file_get_contents 
file_read_firstline $IPV6RE);
 
-# returns a repository string suitable for proxmox-backup-client, pbs-restore, 
etc.
-# $scfg must have the following structure:
-# {
-# datastore
-# server
-# port(optional defaults to 8007)
-# username(optional defaults to 'root@pam')
-# }
+=pod
+
+=head1 NAME
+
+PVE::PBSClient - Proxmox Backup Client Library
+
+=head1 DESCRIPTION
+
+This package contains utilities that wrap common Proxmox Backup client CLI
+operations.
+
+=head2 THE CLIENT OBJECT
+
+While the C> package contains regular 
L,
+the majority is done via the C> object. This object 
represents
+a client that is used to connect to a Proxmox Backup Server:
+
+use strict;
+use warnings;
+
+use Data::Dumper;
+
+$Data::Dumper::Indent = 1;
+$Data::Dumper::Quotekeys = 0;
+$Data::Dumper::Sortkeys = 1;
+$Data::Dumper::Terse = 1;
+
+use PVE::PBSClient;
+
+my $scfg = {
+   server => 'example.tld',
+   username => 'alice@pam',
+   datastore => 'foo-store',
+   fingerprint => '...',
+};
+
+my $client = PVE::PBSClient->new($scfg, "pbs-main");
+
+my ($total, $free, $used) = $client->status();
+
+print "Datastore has a total capacity of $total bytes, of which $used 
bytes are used"
+   . "and $free bytes are still available.\n";
+
+my $snapshot = "vm/1337/2024-07-30T10:01:57Z";
+my $filepath = "/";
+
+my $file_list = $client->file_restore_list($snapshot, $filepath);
+
+print "The snapshot '$snapshot' contains the following restorable 
files:\n";
+print Dumper($file_list);
+print "\n";
+
+
+=head1 FUNCTIONS
+
+=cut
+
+=pod
+
+=head3 get_repository
+
+$repository = get_repository($scfg)
+
+Returns a repository string suitable for the C and
+C executables.
+
+The C<$scfg> hash must have the following structure:
+
+{
+   datastore => 'my-datastore-name',
+   server => 'example.tld',
+   port => 8007, # optional, defaults to 8007
+   username => 'user@realm', # optional, defaults to 'root@pam'
+}
+
+=cut
+
 sub get_repository {
 my ($scfg) = @_;
 
@@ -41,6 +109,58 @@ sub get_repository {
 return "$username\@$server:$datastore";
 }
 
+=pod
+
+=head1 METHODS
+
+=cut
+
+=pod
+
+=head3 new
+
+$client = PVE::PBSClient->new($scfg, $storeid)
+$client = PVE::PBSClient->new($scfg, $storeid, $secret_dir)
+
+Creates a new instance of a C>.
+
+Throws an exception if no C<$scfg> hash is provided or if C<$storeid> is 
C.
+
+=over
+
+=item C<$scfg>
+
+The I hash that the client should use.
+
+This hash is expected to have the following structure:
+
+{
+   datastore => 'my-datastore-name',
+   namespace => 'my-namespace',
+   server => 'example.tld',
+   fingerprint => '...',
+   port => 8007, # optional, defaults to 8007
+   username => 'user@realm', # optional, defaults to 'root@pam'
+}
+
+=item C<$storeid>
+
+The I of the storage corresponding to C<$scfg>. This ID is used for 
operations
+concerning the I and I, such as C> 
and
+C>.
+
+=item C<$secret_dir> (optional)
+
+The name of the I in which the I and I
+files are stored. Defaults to C.
+
+Note that the I and I files are expected to be named
+C and C respectively, if, for example, C<$storeid> is 
C<"foo">.
+
+=back
+
+=cut
+
 sub new {
 my ($class, $scfg, $storeid, $secret_dir) = @_;
 
@@ -63,6 +183,21 @@ my sub password_file_name {
 return "$self->{secret_dir}/$self->{storeid}.pw";
 }
 
+=pod
+
+=head3 set_password
+
+$client->set_password($password)
+
+Updates or creates the I file, storing the given C<$password>.
+
+If the I does not exist, it is created beforehand.
+
+If the I file does not exist, a new one with the permissions C<600>
+is created.
+
+=cut
+
 sub set_password {
 my

[pve-devel] [PATCH v1 pve-common 14/18] pbsclient: prohibit implicit return

[Max Carrara]

.. for the `set_password`, `set_encryption_key` and
`do_raw_client_cmd` methods.

This makes it very clear that these methods aren't supposed to return
anything. In the case of `do_raw_client_cmd` in particular, the
implicit return of `0` on success (would be `undef` if the command
failed) was never depended on either, so prevent future code from
doing so.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index a701542..4ffdd04 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -223,6 +223,7 @@ sub set_password {
 });
 
 PVE::Tools::file_set_contents($pwfile, "$password\n", 0600);
+return;
 };
 
 =pod
@@ -309,6 +310,7 @@ sub set_encryption_key {
 });
 
 PVE::Tools::file_set_contents($encfile, "$key\n", 0600);
+return;
 };
 
 =pod
@@ -416,6 +418,7 @@ my sub do_raw_client_cmd {
 }
 
 run_command($cmd, %opts);
+return;
 }
 
 my sub run_raw_client_cmd : prototype($$$%) {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 06/18] pbsclient: use spaces around list braces and parens around ternaries

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index d707971..d3daf41 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -216,7 +216,7 @@ my sub run_client_cmd : prototype($$;) {
 $param = [] if !defined($param);
 $param = [ $param ] if !ref($param);
 
-$param = [@$param, '--output-format=json'] if !$no_output;
+$param = [ @$param, '--output-format=json' ] if !$no_output;
 
 do_raw_client_cmd(
$self,
@@ -239,7 +239,7 @@ sub autogen_encryption_key {
 my ($self) = @_;
 my $encfile = $self->encryption_key_file_name();
 run_command(
-['proxmox-backup-client', 'key', 'create', '--kdf', 'none', $encfile],
+[ 'proxmox-backup-client', 'key', 'create', '--kdf', 'none', $encfile 
],
 errmsg => 'failed to create encryption key'
 );
 return file_get_contents($encfile);
@@ -327,7 +327,7 @@ sub forget_snapshot {
 
 (my $namespace, $snapshot) = split_namespaced_parameter($self, $snapshot);
 
-return run_client_cmd($self, 'forget', ["$snapshot"], 1, undef, $namespace)
+return run_client_cmd($self, 'forget', [ "$snapshot" ], 1, undef, 
$namespace)
 };
 
 sub prune_group {
@@ -383,7 +383,7 @@ sub file_restore_list {
 my ($self, $snapshot, $filepath, $base64, $extra_params) = @_;
 
 (my $namespace, $snapshot) = split_namespaced_parameter($self, $snapshot);
-my $cmd = [ $snapshot, $filepath, "--base64", $base64 ? 1 : 0];
+my $cmd = [ $snapshot, $filepath, "--base64", ($base64 ? 1 : 0) ];
 
 if (my $timeout = $extra_params->{timeout}) {
push($cmd->@*, '--timeout', $timeout);
@@ -435,7 +435,7 @@ sub file_restore_extract {
my $fn = fileno($fh);
my $errfunc = sub { print $_[0], "\n"; };
 
-   my $cmd = [ $snapshot, $filepath, "-", "--base64", $base64 ? 1 : 0];
+   my $cmd = [ $snapshot, $filepath, "-", "--base64", ($base64 ? 1 : 0) ];
if ($tar) {
push(@$cmd, '--format', 'tar', '--zstd', 1);
}
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 13/18] pbsclient: make method `password_file_name` public

[Max Carrara]

.. in order to allow users to only store a password if the password
file doesn't exist yet, for example.

Also adds corresponding documentation.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index e7e9a79..a701542 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -181,7 +181,18 @@ sub new {
 return $self;
 }
 
-my sub password_file_name {
+=pod
+
+=head3 password_file_name
+
+$file_name = $client->password_file_name()
+
+Returns the full name of the I file, including the path of the
+I it is located in.
+
+=cut
+
+sub password_file_name {
 my ($self) = @_;
 
 return File::Spec->catfile($self->{secret_dir}, "$self->{storeid}.pw");
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 11/18] pbsclient: let `status` method return a hash instead of an array

[Max Carrara]

Instead of returning an (arguably awkward) array with four elements,
where each element has a special meaning, return the hash that is
constructed from the `proxmox-backup-client status` command's JSON
output.

The documentation is updated accordingly.

This method isn't used anywhere in our code base at the moment, so I
assume it is safe to change it. Checked with ripgrep.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 32 ++--
 1 file changed, 10 insertions(+), 22 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 69b4e40..26f73ef 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -731,36 +731,24 @@ sub prune_group {
 
 =head3 status
 
-($total, $free, $used, $active) = $client->status()
+$status = $client->status()
 
-Return the I of the client's repository as an array.
+Return the I of the client's repository as a hash.
 
-The array contains the C<$total>, C<$free> and C<$used> size of the repository
-as bytes, as well as whether the repository is C<$active> or not.
+The returned hash has the following structure:
+
+{
+   avail => 41735159808,  # sizes are in bytes!
+   total => 264240103424,
+   used => 41735159808,
+}
 
 =cut
 
 sub status {
 my ($self) = @_;
 
-my $total = 0;
-my $free = 0;
-my $used = 0;
-my $active = 0;
-
-eval {
-   my $res = run_client_cmd($self, "status");
-
-   $active = 1;
-   $total = $res->{total};
-   $used = $res->{used};
-   $free = $res->{avail};
-};
-if (my $err = $@) {
-   warn $err;
-}
-
-return ($total, $free, $used, $active);
+return run_client_cmd($self, "status");
 };
 
 =pod
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 05/18] pbsclient: use cond. statements instead of chained 'or' operators

[Max Carrara]

.. like in the `delete_encryption_key` subroutine below, as it's more
readable at a glance.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index ab1fa62..d707971 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -77,7 +77,10 @@ sub delete_password {
 
 my $pwfile = password_file_name($self);
 
-unlink $pwfile or $! == ENOENT or die "deleting password file failed - 
$!\n";
+if (!unlink($pwfile)) {
+   return if $! == ENOENT;
+   die "deleting password file failed - $!\n";
+}
 };
 
 sub get_password {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 04/18] pbsclient: pull variable out of long post-if definedness check

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 525b37f..ab1fa62 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -288,7 +288,9 @@ sub backup_fs_tree {
 
 $cmd_opts = {} if !defined($cmd_opts);
 
-$cmd_opts->{namespace} = $self->{scfg}->{namespace} if 
defined($self->{scfg}->{namespace});
+if (defined(my $namespace = $self->{scfg}->{namespace})) {
+   $cmd_opts->{namespace} = $namespace;
+}
 
 return run_raw_client_cmd($self, 'backup', $param, %$cmd_opts);
 };
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 02/18] pbsclient: use parentheses when calling most inbuilts

[Max Carrara]

.. except for really common cases like `die`, `warn`, `keys`.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 46 ++--
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index a1536e6..3e98dd1 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -49,11 +49,11 @@ sub new {
 
 $secret_dir = '/etc/pve/priv/storage' if !defined($secret_dir);
 
-my $self = bless {
+my $self = bless({
scfg => $scfg,
storeid => $storeid,
secret_dir => $secret_dir
-}, $class;
+}, $class);
 return $self;
 }
 
@@ -67,7 +67,7 @@ sub set_password {
 my ($self, $password) = @_;
 
 my $pwfile = password_file_name($self);
-mkdir $self->{secret_dir};
+mkdir($self->{secret_dir});
 
 PVE::Tools::file_set_contents($pwfile, "$password\n", 0600);
 };
@@ -98,7 +98,7 @@ sub set_encryption_key {
 my ($self, $key) = @_;
 
 my $encfile = $self->encryption_key_file_name();
-mkdir $self->{secret_dir};
+mkdir($self->{secret_dir});
 
 PVE::Tools::file_set_contents($encfile, "$key\n", 0600);
 };
@@ -108,7 +108,7 @@ sub delete_encryption_key {
 
 my $encfile = $self->encryption_key_file_name();
 
-if (!unlink $encfile) {
+if (!unlink($encfile)) {
return if $! == ENOENT;
die "failed to delete encryption key! $!\n";
 }
@@ -144,7 +144,7 @@ my $USE_CRYPT_PARAMS = {
 my sub do_raw_client_cmd {
 my ($self, $client_cmd, $param, %opts) = @_;
 
-my $client_bin = (delete $opts{binary}) || 'proxmox-backup-client';
+my $client_bin = delete($opts{binary}) || 'proxmox-backup-client';
 my $use_crypto = $USE_CRYPT_PARAMS->{$client_bin}->{$client_cmd} // 0;
 
 my $client_exe = "/usr/bin/$client_bin";
@@ -153,13 +153,13 @@ my sub do_raw_client_cmd {
 my $scfg = $self->{scfg};
 my $repo = get_repository($scfg);
 
-my $userns_cmd = delete $opts{userns_cmd};
+my $userns_cmd = delete($opts{userns_cmd});
 
 my $cmd = [];
 
-push @$cmd, @$userns_cmd if defined($userns_cmd);
+push(@$cmd, @$userns_cmd) if defined($userns_cmd);
 
-push @$cmd, $client_exe, $client_cmd;
+push(@$cmd, $client_exe, $client_cmd);
 
 # This must live in the top scope to not get closed before the 
`run_command`
 my $keyfd;
@@ -169,17 +169,17 @@ my sub do_raw_client_cmd {
// die "failed to get file descriptor flags: $!\n";
fcntl($keyfd, F_SETFD, $flags & ~FD_CLOEXEC)
or die "failed to remove FD_CLOEXEC from encryption key file 
descriptor\n";
-   push @$cmd, '--crypt-mode=encrypt', '--keyfd='.fileno($keyfd);
+   push(@$cmd, '--crypt-mode=encrypt', '--keyfd='.fileno($keyfd));
} else {
-   push @$cmd, '--crypt-mode=none';
+   push(@$cmd, '--crypt-mode=none');
}
 }
 
-push @$cmd, @$param if defined($param);
+push(@$cmd, @$param) if defined($param);
 
-push @$cmd, "--repository", $repo;
+push(@$cmd, "--repository", $repo);
 if (defined(my $ns = delete($opts{namespace}))) {
-   push @$cmd, '--ns', $ns;
+   push(@$cmd, '--ns', $ns);
 }
 
 local $ENV{PBS_PASSWORD} = $self->get_password();
@@ -266,7 +266,7 @@ sub get_snapshots {
 }
 
 my $param = [];
-push @$param, $group if defined($group);
+push(@$param, $group) if defined($group);
 
 return run_client_cmd($self, "snapshots", $param, undef, undef, 
$namespace);
 };
@@ -337,16 +337,16 @@ sub prune_group {
 
 my $param = [];
 
-push @$param, "--quiet";
+push(@$param, "--quiet");
 
 if (defined($opts->{'dry-run'}) && $opts->{'dry-run'}) {
-   push @$param, "--dry-run", $opts->{'dry-run'};
+   push(@$param, "--dry-run", $opts->{'dry-run'});
 }
 
 foreach my $keep_opt (keys %$prune_opts) {
-   push @$param, "--$keep_opt", $prune_opts->{$keep_opt};
+   push(@$param, "--$keep_opt", $prune_opts->{$keep_opt});
 }
-push @$param, "$group";
+push(@$param, "$group");
 
 return run_client_cmd($self, 'prune', $param, undef, undef, $namespace);
 };
@@ -381,7 +381,7 @@ sub file_restore_list {
 my $cmd = [ $snapshot, $filepath, "--base64", $base64 ? 1 : 0];
 
 if (my $timeout = $extra_params->{timeout}) {
-   push $cmd->@*, '--timeout', $timeout;
+   push($cmd->@*, '--timeout', $timeout);
 }
 
 return run_client_cmd(
@@ -406,9 +406,9 @@ sub file_restore_extract_prepare {
 # allow reading data for proxy user
 my $wwwid

[pve-devel] [PATCH v1 pve-common 07/18] pbsclient: s/foreach/for

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index d3daf41..231406a 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -348,7 +348,7 @@ sub prune_group {
push(@$param, "--dry-run", $opts->{'dry-run'});
 }
 
-foreach my $keep_opt (keys %$prune_opts) {
+for my $keep_opt (keys %$prune_opts) {
push(@$param, "--$keep_opt", $prune_opts->{$keep_opt});
 }
 push(@$param, "$group");
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 03/18] pbsclient: use post-if definedness checks instead of '//=' operator

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index 3e98dd1..525b37f 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -208,7 +208,7 @@ my sub run_client_cmd : prototype($$;) {
 my $json_str = '';
 my $outfunc = sub { $json_str .= "$_[0]\n" };
 
-$binary //= 'proxmox-backup-client';
+$binary = 'proxmox-backup-client' if !defined($binary);
 
 $param = [] if !defined($param);
 $param = [ $param ] if !ref($param);
@@ -286,7 +286,7 @@ sub backup_fs_tree {
'--backup-id', $id,
 ];
 
-$cmd_opts //= {};
+$cmd_opts = {} if !defined($cmd_opts);
 
 $cmd_opts->{namespace} = $self->{scfg}->{namespace} if 
defined($self->{scfg}->{namespace});
 
@@ -308,7 +308,7 @@ sub restore_pxar {
"$target",
"--allow-existing-dirs", 0,
 ];
-$cmd_opts //= {};
+$cmd_opts = {} if !defined($cmd_opts);
 
 $cmd_opts->{namespace} = $namespace;
 
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 01/18] pbsclient: rename 'sdir' parameter of constructor to 'secret_dir'

[Max Carrara]

.. so that it's less ambiguous for what the parameter stands for at a
glance.

Signed-off-by: Max Carrara 
---
 src/PVE/PBSClient.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/PVE/PBSClient.pm b/src/PVE/PBSClient.pm
index e63af03..a1536e6 100644
--- a/src/PVE/PBSClient.pm
+++ b/src/PVE/PBSClient.pm
@@ -42,12 +42,12 @@ sub get_repository {
 }
 
 sub new {
-my ($class, $scfg, $storeid, $sdir) = @_;
+my ($class, $scfg, $storeid, $secret_dir) = @_;
 
 die "no section config provided\n" if ref($scfg) eq '';
 die "undefined store id\n" if !defined($storeid);
 
-my $secret_dir = $sdir // '/etc/pve/priv/storage';
+$secret_dir = '/etc/pve/priv/storage' if !defined($secret_dir);
 
 my $self = bless {
scfg => $scfg,
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 pve-common 00/18] Introduction of libproxmox-backup-client-perl

[Max Carrara]

 feedback or suggestions in
that regard.

Testing
---

Module Methods
**

The module's methods were smoke-tested with a local script that I
executed on my dev VM. I created a new user on my PBS VM that the
script uses.

What was tested:
  * Creating new password and encryption key files in a separate secret
directory (in order to not mess with my VM's installation)
  * Querying the datastore's status
  * Listing snapshots
  * Creating and uploading a new PXAR backup from a random directory
I had laying around
  * Querying the snapshots again and filtering out the newly created
snapshot of the PXAR backup, then extracting the backed up archive
to a new directory again
  * Forgetting the newly snapshot after its extraction
  * Downloading files from specific backups via the
   `file_restore_extract_prepare` and `file_restore_extract` methods

This does however not cover all methods / cases, so I'd be grateful for
anyone testing this series. The docs should hopefully be enough to get
you started - if they're not, please let me know, because that means
they need improvements ;)

Package Split
*

The package split was tested by substituting the `X.Y.Z` placeholder in
`debian/control` with a higher minor version and adding a corresponding
bump in `debian/changelog` with `dch -i`. The two packages built
successfully and were installed on my dev VM.

* Trying to install only the `libproxmox-backup-client-perl` package
  results in `apt` refusing to perform the installation, as expected.
* Installing both packages succeeds, as expected, and did not caus any
  noticable issues.
* Downgrading to the previous version of `libpve-common-perl` also
  causes `libproxmox-backup-client-perl` to be uninstalled, as expected.

For good measure I also double-checked if the Perl module was correctly
moved to the new package by comparing both `.deb` files via `dpkg -c`
and running `dpkg -S /usr/share/perl5/PVE/PBSClient.pm` on my dev VM.

References
--

[1]: 
https://git.proxmox.com/?p=pve-storage.git;a=blob;f=src/PVE/Storage/PBSPlugin.pm;h=0808bccd79a75a6ace1c7194b30e894f5db54659;hb=refs/heads/master#l75
[2]: https://bugzilla.proxmox.com/show_bug.cgi?id=3186

Summary of Changes
--

Max Carrara (18):
  pbsclient: rename 'sdir' parameter of constructor to 'secret_dir'
  pbsclient: use parentheses when calling most inbuilts
  pbsclient: use post-if definedness checks instead of '//=' operator
  pbsclient: pull variable out of long post-if definedness check
  pbsclient: use cond. statements instead of chained 'or' operators
  pbsclient: use spaces around list braces and parens around ternaries
  pbsclient: s/foreach/for
  pbsclient: document package and its public functions & methods
  pbsclient: create secret dir with `mkdir -p` and mode `700`
  pbsclient: use `File::Spec->catfile` to concatenate file paths
  pbsclient: let `status` method return a hash instead of an array
  pbsclient: throw exception if username of client has no realm
  pbsclient: make method `password_file_name` public
  pbsclient: prohibit implicit return
  pbsclient: don't return anything in PXAR methods
  pbsclient: don't return anything in `forget_snapshot`
  make: support building multiple packages from the same source
  deb: split PBSClient.pm into new package libproxmox-backup-client-perl

 Makefile |  27 +-
 debian/control   |  12 +
 debian/libproxmox-backup-client-perl.install |   1 +
 debian/libpve-common-perl.install|  28 +
 src/PVE/PBSClient.pm | 670 ---
 5 files changed, 651 insertions(+), 87 deletions(-)
 create mode 100644 debian/libproxmox-backup-client-perl.install
 create mode 100644 debian/libpve-common-perl.install

-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [RFC storage 10/23] plugin: introduce new_backup_provider() method

[Max Carrara]

On Fri Jul 26, 2024 at 11:52 AM CEST, Fiona Ebner wrote:
> Am 25.07.24 um 17:32 schrieb Max Carrara:
> > On Thu Jul 25, 2024 at 3:11 PM CEST, Fiona Ebner wrote:
> >> Am 25.07.24 um 11:48 schrieb Max Carrara:
> >>> The same goes for backup provider plugins - IMO namespacing them
> >>> like e.g. `PVE::Backup::Provider::Plugin::Foo` where `Foo` is a
> >>> (concrete) plugin.
> >>>
> >>
> >> The BackupProvider namespace is already intended for the plugins, adding
> >> an extra level with "Plugin" would just bloat the module names,
> >> especially if we decide to go the same route as for storage plugins and
> >> have a "Custom" sub-namespace.
> > 
> > I understand what you mean, yeah. Would perhaps something like
> > `PVE::BackupProvider::Plugin::*` be better?
> > 
> > The reason why I'm suggesting this is because in `PVE::Storage::*`,
> > every plugin lives alongside `Plugin.pm`, even though the extra
> > directory wouldn't really hurt IMO. E.g. `PVE::Storage::DirPlugin` would
> > then be `PVE::Storage::Plugin::Dir`.
> > 
>
> I think it's fine to live alongside the base plugin (I'd prefer
> Plugin::Base if going for a dedicated directory). I agree, if we ever
> want to add something other than plugins to the top namespace, it is
> much nicer to have the dedicated directory. And it is made more explicit
> that things in there are plugins (and not having to name each one
> FooPlugin). However, I still feel like
> PVE::BackupProvider::Plugin::Custom::Bar is rather lengthy (should we go
> with the Custom directory again), but I'm not really opposed to doing it
> like this.
>
> >>
> >>> The above two methods - `backup_nbd` and `backup_directory` - is there
> >>> perhaps a way to merge them? I'm not sure if what I'm having in mind
> >>> here is actually feasible, but what I mean is "making the method
> >>> agnostic to the type of backup". As in, perhaps pass a hash that
> >>> contains a `type` key for the type of backup being made, and instead of
> >>> having long method signatures, include the remaining parameters as the
> >>> remaining keys. For example:
> >>>
> >>> {
> >>> 'type' => 'lxc-dir',  # type names are just examples here
> >>> 'directory' => '/foo/bar/baz',
> >>> 'bandwidth_limit' => 42,
> >>> ...
> >>> }
> >>>
> >>> {
> >>> 'type' => 'vm-nbd',
> >>> 'device_name' => '...',
> >>> 'nbd_path' => '...',
> >>> ...
> >>> }
> >>>
> >>> You get the point :P
> >>>
> >>> IMO it would make it easier to extend later, and also make it more
> >>> straightforward to introduce new parameters / deprecate old ones, while
> >>> the method signature stays stable otherwise.
> >>>
> >>> The same goes for the different cleanup methods further down below;
> >>> instead of having a separate method for each "type of cleanup being
> >>> performed", let the implementor handle it according to the data the
> >>> method receives.
> >>>
> >>> IMHO I think it's best to be completely agnostic over VM / LXC backups
> >>> (and their specific types) wherever possible and let the data describe
> >>> what's going on instead.
> >>>
> >>
> >> The point about extensibility is a good one. The API wouldn't need to
> >> change even if we implement new mechanisms. But thinking about it some
> >> more, is there anything really gained? Because we will not force plugins
> >> to implement the methods for new mechanisms of course, they can just
> >> continue supporting what they support. Each mechanism will have its own
> >> specific set of parameters, so throwing everything into a catch-all
> >> method and hash might make it too generic.
> > 
> > The main point is indeed extensibility, but it also makes maintaining
> > the API a bit easier. Should we (in the future) decide to add or remove
> > any parameters, we don't need to touch the signature - and in turn, we
> > don't need to tediously `grep` for every call site to ensure that
> > they're updated accordingly.
> > 
> > With hashes one could instead always just check i

Re: [pve-devel] [RFC storage 10/23] plugin: introduce new_backup_provider() method

[Max Carrara]

On Thu Jul 25, 2024 at 3:11 PM CEST, Fiona Ebner wrote:
> Am 25.07.24 um 11:48 schrieb Max Carrara:
> > On Tue Jul 23, 2024 at 11:56 AM CEST, Fiona Ebner wrote:
> >> Signed-off-by: Fiona Ebner 
> > 
> > Some overall thoughts:
> > 
> > 1.  I'm really, really happy to see documentation in this module here,
> > that's fantastic! :)
> > 
> > While the contents of the docs seem fine, I would suggest you used
> > POD instead. You can find an example in one of my recent series. [1]
> > I mainly prefer POD solely because it's what Perl uses; it also
> > indirectly makes sure we all use the same kind of format for
> > documenting our Perl code.
> > 
> > Of course, we've currently not decided on any particular format, but
> > because the opportunity arose, I wanted to pitch POD here
> > nevertheless. ;)
> > 
>
> I'll look into it for v2. Agreed, following a standard for documenting
> an API module has its merits.

Sweet!

>
> > 2.  I would personally prefer a namespace like `PVE::Backup::Provider`
> > instead of `PVE::BackupProvider`, simply because it leaves room for
> > further packages and reduces churn in the long term, IMO.
> > 
>
> There's a risk though that PVE::Backup::Provider and PVE::Backup::Foo
> are unrelated things that have no real business sharing a namespace.

Hmm, fair point - on second thought, `PVE::Backup` indeed seems a bit
too generic.

>
> > The same goes for backup provider plugins - IMO namespacing them
> > like e.g. `PVE::Backup::Provider::Plugin::Foo` where `Foo` is a
> > (concrete) plugin.
> > 
>
> The BackupProvider namespace is already intended for the plugins, adding
> an extra level with "Plugin" would just bloat the module names,
> especially if we decide to go the same route as for storage plugins and
> have a "Custom" sub-namespace.

I understand what you mean, yeah. Would perhaps something like
`PVE::BackupProvider::Plugin::*` be better?

The reason why I'm suggesting this is because in `PVE::Storage::*`,
every plugin lives alongside `Plugin.pm`, even though the extra
directory wouldn't really hurt IMO. E.g. `PVE::Storage::DirPlugin` would
then be `PVE::Storage::Plugin::Dir`.

The reason why I'm suggesting *something* like that here is to reduce
some clutter and simply keep related things grouped. Also, IMO it's
better to consider the overall package structure beforehand, simply to
avoid any churn in the future - something I've noticed while poking
around the storage API.

Maybe I'm being a bit pedantic here (tbh I probably am), but it's just
something that I wanted to mention anyhow.

>
> > While this seems long or somewhat excessive, I think it enforces a
> > clear package / module hierarchy and keeps things tidier in the long
> > term, and those couple extra keystrokes don't really hurt anyone.
> > 
>
> I get where you're coming from, I just feel like BackupProvider might be
> better as its own separate thing, containing the plugins for the
> specific purpose. But I don't have a strong opinion about it, and am
> fine making such changes if other developers prefer it too :)

I agree now that BackupProvider should remain on its own; I otherwise
don't have any strong opinions about it either (though I would like to
shove plugins one directory level deeper ;P). As I said, I'm probably
just a bit pedantic here; feel free to disregard these suggestions if
you think they're not applicable :)

>
> > The above two methods - `backup_nbd` and `backup_directory` - is there
> > perhaps a way to merge them? I'm not sure if what I'm having in mind
> > here is actually feasible, but what I mean is "making the method
> > agnostic to the type of backup". As in, perhaps pass a hash that
> > contains a `type` key for the type of backup being made, and instead of
> > having long method signatures, include the remaining parameters as the
> > remaining keys. For example:
> > 
> > {
> > 'type' => 'lxc-dir',  # type names are just examples here
> > 'directory' => '/foo/bar/baz',
> > 'bandwidth_limit' => 42,
> > ...
> > }
> > 
> > {
> > 'type' => 'vm-nbd',
> > 'device_name' => '...',
> > 'nbd_path' => '...',
> > ...
> > }
> > 
> > You get the point :P
> > 
> > IMO it would make it easier to extend later, and also make it more
> > straightforward to 

Re: [pve-devel] [RFC storage 10/23] plugin: introduce new_backup_provider() method

[Max Carrara]

On Tue Jul 23, 2024 at 11:56 AM CEST, Fiona Ebner wrote:
> The new_backup_provider() method can be used by storage plugins for
> external backup providers. If the method returns a provider, Proxmox
> VE will use callbacks to that provider for backups and restore instead
> of using its usual backup/restore mechanisms.
>
> API age and version are both bumped.
>
> The backup provider API is split into two parts, both of which again
> need different implementations for VM and LXC guests:
>
> 1. Backup API
>
> There hook callbacks for the start/end/abort phases of guest backups
> as well as for start/end/abort phases of a whole backup job.
>
> The backup_get_mechanism() method is used to decide on the backup
> mechanism. Currently only 'nbd' for VMs and 'directory' for containers
> are possible. It also let's the plugin decide whether to use a bitmap
> for incremental VM backup or not.
>
> Next, there are methods for backing up guest and firewall
> configuration as well as for the backup mechanisms:
>
> - a container filesystem using a provided directory. The directory
>   contains an unchanging copy of the container's file system.
>
> - a VM disk using a provided NBD export. The export is an unchanging
>   copy of the VM's disk. Either the full image, or in case a bitmap is
>   used, the dirty parts of the image since the last time the bitmap
>   was used for a successful backup. Reading outside of the dirty parts
>   will result in an error. After backing up each part of the disk, it
>   should be discarded in the export to avoid unnecessary space usage
>   on the Proxmox VE side (there is an associated fleecing image).
>
> Finally, some helpers like getting the provider name or volume ID for
> the backup target, as well as for handling the backup log.
>
> 2. Restore API
>
> The restore_get_mechanism() method is used to decide on the restore
> mechanism. Currently, only 'qemu-img' for VMs and 'directory' and
> 'tar' for containers are possible.
>
> Next, methods for extracting the guest and firewall configuration and
> the implementations of the restore mechanism. It is enough to
> implement one restore mechanism per guest type of course:
>
> - for VMs, with the 'qemu-img' mechanism, the backup provider gives a
>   path to the disk image that will be restore. The path should be
>   something qemu-img can deal with, e.g. can also be an NBD URI.
>
> - for containers, with the 'directory' mechanism, the backup provider
>   gives the path to a directory with the full filesystem structure of
>   the container.
>
> - for containers, with the 'tar' mechanism, the backup provider gives
>   the path to a (potentially compressed) tar archive with the full
>   filesystem structure of the container.
>
> For VMs, there also is a restore_qemu_get_device_info() helper
> required, to get the disks included in the backup and their sizes.
>
> See the PVE::BackupProvider::Plugin module for the full API
> documentation.
>
> Signed-off-by: Fiona Ebner 

Some overall thoughts:

1.  I'm really, really happy to see documentation in this module here,
that's fantastic! :)

While the contents of the docs seem fine, I would suggest you used
POD instead. You can find an example in one of my recent series. [1]
I mainly prefer POD solely because it's what Perl uses; it also
indirectly makes sure we all use the same kind of format for
documenting our Perl code.

Of course, we've currently not decided on any particular format, but
because the opportunity arose, I wanted to pitch POD here
nevertheless. ;)

2.  I would personally prefer a namespace like `PVE::Backup::Provider`
instead of `PVE::BackupProvider`, simply because it leaves room for
further packages and reduces churn in the long term, IMO.

The same goes for backup provider plugins - IMO namespacing them
like e.g. `PVE::Backup::Provider::Plugin::Foo` where `Foo` is a
(concrete) plugin.

While this seems long or somewhat excessive, I think it enforces a
clear package / module hierarchy and keeps things tidier in the long
term, and those couple extra keystrokes don't really hurt anyone.

There are some more comments inline, though I want to mention that I
really like your work so far! :)

[1]: https://lists.proxmox.com/pipermail/pve-devel/2024-July/064703.html

> ---
>  src/PVE/BackupProvider/Makefile  |   6 +
>  src/PVE/BackupProvider/Plugin.pm | 343 +++
>  src/PVE/Makefile |   1 +
>  src/PVE/Storage.pm   |  12 +-
>  src/PVE/Storage/Plugin.pm|  15 ++
>  5 files changed, 375 insertions(+), 2 deletions(-)
>  create mode 100644 src/PVE/BackupProvider/Makefile
>  create mode 100644 src/PVE/BackupProvider/Plugin.pm
>
> diff --git a/src/PVE/BackupProvider/Makefile b/src/PVE/BackupProvider/Makefile
> new file mode 100644
> index 000..53cccac
> --- /dev/null
> +++ b/src/PVE/BackupProvider/Makefile
> @@ -0,0 +1,6 @@
> +SOURCES = Plugin.pm
> +
> +.PHONY: insta

[pve-devel] [PATCH v3 pve-manager 5/5] fix #5366: api: ceph: change version format in OSD metadata endpoint

[Max Carrara]

.. in order to include Ceph's build commit. Instead of e.g.

  18.2.2 (reef)

the string will now contain:

  18.2.2 (build: e9fe820e7) reef

This format is used in the OSD detail view; the build commit will
therefore also be shown there.

Signed-off-by: Max Carrara 
Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=5366
---
Changes v2 --> v3:
  * put 'build: ' in front of build commit (thanks Thomas!)
  * reword commit title
  * add 'Fixes' trailer

Changes v1 --> v2:
  * NEW

 PVE/API2/Ceph/OSD.pm | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/PVE/API2/Ceph/OSD.pm b/PVE/API2/Ceph/OSD.pm
index 1ff51fbd..88b7142b 100644
--- a/PVE/API2/Ceph/OSD.pm
+++ b/PVE/API2/Ceph/OSD.pm
@@ -747,6 +747,12 @@ __PACKAGE__->register_method ({
my $osd_pss_memory = eval { get_proc_pss_from_pid($pid) } // 0;
warn $@ if $@;
 
+   my ($ceph_version, $ceph_buildcommit) = 
PVE::Ceph::Tools::parse_ceph_version(
+   $metadata->{ceph_version}
+   );
+
+   $ceph_buildcommit = substr($ceph_buildcommit, 0, 9);
+
my $data = {
osd => {
hostname => $metadata->{hostname},
@@ -755,7 +761,7 @@ __PACKAGE__->register_method ({
osd_data => $metadata->{osd_data},
osd_objectstore => $metadata->{osd_objectstore},
pid => $pid,
-   version => "$metadata->{ceph_version_short} 
($metadata->{ceph_release})",
+   version => "$ceph_version (build: $ceph_buildcommit) 
$metadata->{ceph_release}",
front_addr => $metadata->{front_addr},
back_addr => $metadata->{back_addr},
hb_front_addr => $metadata->{hb_front_addr},
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v3 pve-manager 4/5] ui: ceph: osd: increase width of version column

[Max Carrara]

.. so that the Ceph build commit as well as differing build commits
are shown properly.

Signed-off-by: Max Carrara 
---
Changes v2 --> v3:
  * increase the width even further to account for new changes

Changes v1 --> v2:
  * NEW

 www/manager6/ceph/OSD.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/www/manager6/ceph/OSD.js b/www/manager6/ceph/OSD.js
index 11ec7670..72fea489 100644
--- a/www/manager6/ceph/OSD.js
+++ b/www/manager6/ceph/OSD.js
@@ -802,6 +802,7 @@ Ext.define('PVE.node.CephOsdTree', {
dataIndex: 'version',
align: 'right',
renderer: 'render_version',
+   width: 270,
},
{
text: 'weight',
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v3 pve-manager 3/5] fix #5366: ui: ceph: osd: rework version field rendering

[Max Carrara]

.. and show the build commit next to the OSD version.

The logic of the `render_version` function is split up in order to
handle how the version is displayed depending on the type of the row.

If the parsed version is `undefined` or the row marks the beginning of
the tree, an empty string is now returned. This behaviour is
equivalent to before, it just makes the overall logic easier.

If the row belongs to a node, the build commit is now additionally
displayed in parentheses next to the installed Ceph version:

  18.2.2 (build: abcd1234)

If the row belongs to an OSD, the build commit is also additionally
displayed in parentheses next to the installed Ceph version.
Furthermore, should the build commit of the running OSD differ from
the one that's installed on the host, the new hash will also be shown
in parentheses:

  18.2.2 (build: abcd1234 -> 5678fedc)

Additionally, the icon displayed for running an OSD with an outdated
build is now the same as for running an outdated version. The
conditional display of cluster health-related icons remains the same
otherwise.

Signed-off-by: Max Carrara 
Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=5366
---
Changes v2 --> v3:
  * use camelCase instead of nocaseatall for certain variables
  * use less ambiguous variable names (thanks Thomas!)
  * use new `renderCephBuildCommit` helper instead of duplicating logic
(thanks Thomas!)
  * put 'build: ' in front of build commit when rendering (thanks Thomas!)
  * reword commit title
  * add 'Fixes' trailer
  * remove outdated R-b and T-b trailers

Changes v1 --> v2:
  * use camelCase instead of snake_case (thanks Lukas!)
  * use more descriptive variable names (thanks Lukas!)
  * use `let` instead of `const` for variables where applicable (thanks Lukas!)

 www/manager6/ceph/OSD.js | 53 +++-
 1 file changed, 41 insertions(+), 12 deletions(-)

diff --git a/www/manager6/ceph/OSD.js b/www/manager6/ceph/OSD.js
index d2caafa4..11ec7670 100644
--- a/www/manager6/ceph/OSD.js
+++ b/www/manager6/ceph/OSD.js
@@ -642,23 +642,52 @@ Ext.define('PVE.node.CephOsdTree', {
},
 
render_version: function(value, metadata, rec) {
+   if (value === undefined || rec.data.type === 'root') {
+   return '';
+   }
+
let vm = this.getViewModel();
-   let versions = vm.get('versions');
-   let icon = "";
-   let version = value || "";
-   let maxversion = vm.get('maxversion');
-   if (value && PVE.Utils.compare_ceph_versions(value, maxversion) !== 
0) {
-   let host_version = rec.parentNode?.data?.version || 
versions[rec.data.host] || "";
-   if (rec.data.type === 'host' || 
PVE.Utils.compare_ceph_versions(host_version, maxversion) !== 0) {
+   let maxVersion = vm.get('maxversion');
+   let hasMixedVersions = vm.get('mixedversions');
+
+   if (rec.data.type === 'host') {
+   let icon = "";
+   let installedBuildCommit = rec.data.buildcommit ?? '';
+
+   if (PVE.Utils.compare_ceph_versions(maxVersion, value) > 0) {
icon = PVE.Utils.get_ceph_icon_html('HEALTH_UPGRADE');
-   } else {
-   icon = PVE.Utils.get_ceph_icon_html('HEALTH_OLD');
+   } else if (hasMixedVersions) {
+   icon = PVE.Utils.get_ceph_icon_html('HEALTH_OK');
+   }
+
+   if (installedBuildCommit === '') {
+   return `${icon}${value}`;
}
-   } else if (value && vm.get('mixedversions')) {
-   icon = PVE.Utils.get_ceph_icon_html('HEALTH_OK');
+
+   return `${icon}${value} (build: 
${installedBuildCommit.substring(0, 9)})`;
+   }
+
+   let installedVersion = rec.parentNode?.data?.version ?? '';
+
+   let runningBuildCommit = PVE.Utils.parseCephBuildCommit(rec.data) 
?? '';
+   let installedBuildCommit = rec.parentNode?.data?.buildcommit ?? '';
+
+   let versionInfo = {
+   runningVersion: value,
+   installedVersion: installedVersion,
+   runningBuildCommit: runningBuildCommit,
+   installedBuildCommit: installedBuildCommit,
+   maxVersion: maxVersion,
+   hasMixedVersions: hasMixedVersions,
+   };
+
+   let { icon, renderedBuildCommit } = 
PVE.Utils.renderCephBuildCommit(versionInfo, 9);
+
+   if (renderedBuildCommit) {
+   return `${icon}${value} (build: ${renderedBuildCommit})`;
}
 
-   return icon + version;
+   return `${icon}${value}`;
},
 
render_osd_val: function(value, metaData, rec) {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v3 pve-manager 2/5] fix #5366: api: ceph: add host build commit to Ceph OSD index data

[Max Carrara]

This is required in order to avoid making multiple API calls in the
following commit.

Signed-off-by: Max Carrara 
Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=5366
Tested-by: Lukas Wagner 
Reviewed-by: Lukas Wagner 
---
Changes v2 --> v3:
  * reword commit title
  * add 'Fixes' trailer

Changes v1 --> v2:
  * none

 PVE/API2/Ceph/OSD.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/PVE/API2/Ceph/OSD.pm b/PVE/API2/Ceph/OSD.pm
index 5e39eed7..1ff51fbd 100644
--- a/PVE/API2/Ceph/OSD.pm
+++ b/PVE/API2/Ceph/OSD.pm
@@ -206,6 +206,7 @@ __PACKAGE__->register_method ({
 
if ($name && $e->{type} eq 'host') {
$new->{version} = $hostversions->{$name}->{version}->{str};
+   $new->{buildcommit} = $hostversions->{$name}->{buildcommit};
}
}
 
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v3 pve-manager 0/5] Fix #5366: Ceph Build Commit in UI

[Max Carrara]

Ceph Build Commit in UI - Version 3
===

Notable Changes Since v2


  * Rebase on master branch as v2 was partially applied (thanks!)
  * Factor duplicate build commit rendering code into separate helper
as suggested [1] (thanks Thomas!)
  * Make variable names more clear / less ambiguous (thanks Thomas!)
  * Increase default width of version column in OSD tree view ever so
slightlymore
  * Reword commit titles and messages to reflect the changes made

For a detailed list of changes please see the comments in the individual
patches.

NOTE: Any T-b and R-b trailers on patches that received changes are
considered outdated and are thus removed.

Older Versions
--

v1: https://lists.proxmox.com/pipermail/pve-devel/2024-April/063772.html
v2: https://lists.proxmox.com/pipermail/pve-devel/2024-July/064349.html

References
--

[1]: https://lists.proxmox.com/pipermail/pve-devel/2024-July/064789.html

Summary of Changes
--

Max Carrara (5):
  fix #5366: ui: ceph: services: parse and display build commit
  fix #5366: api: ceph: add host build commit to Ceph OSD index data
  fix #5366: ui: ceph: osd: rework version field rendering
  ui: ceph: osd: increase width of version column
  fix #5366: api: ceph: change version format in OSD metadata endpoint

 PVE/API2/Ceph/OSD.pm |   9 ++-
 www/manager6/Utils.js| 107 +++
 www/manager6/ceph/OSD.js |  54 
 www/manager6/ceph/ServiceList.js |  33 +++---
 www/manager6/ceph/Services.js|  19 +-
 5 files changed, 198 insertions(+), 24 deletions(-)

-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v3 pve-manager 1/5] fix #5366: ui: ceph: services: parse and display build commit

[Max Carrara]

The build commit is displayed and taken into account when comparing
monitor and manager versions in the client. Specifically, the
shortened build commit is now displayed in parentheses next to the
version for both monitors and managers like so:

  18.2.2 (build: abcd1234)

Should the build commit of the running service differ from the one
that's installed on the host, the newer build commit will also be
shown in parentheses:

  18.2.2 (build: abcd1234 -> 5678fedc)

The icon displayed for running a service with an outdated build is the
same as for running an outdated version. The conditional display of
icons related to cluster health remains the same otherwise.

The Ceph summary view also displays the hash and will show a warning
if a service is running with a build commit that doesn't match the one
that's advertised by the host.

This requires the introduction of two helper functions:

1. `PVE.Utils.parseCephBuildCommit`, which can be used to get the full
  hash "eccf199d..." in parentheses from a string like the following:

  ceph version 17.2.7 (eccf199d63457659c09677399928203b7903c888) quincy (stable)

2. `PVE.Utils.renderCephBuildCommit`, which is used to determine how
   to render a service's current build commit and which accompanying
   icon to choose.

Signed-off-by: Max Carrara 
Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=5366
---
Changes v2 --> v3:
  * add new `renderCephBuildCommit` helper function (thanks Thomas!)
  * add docstrings for helpers
  * use less ambiguous variable names (thanks Thomas!)
  * put 'build: ' in front of build commit when rendering (thanks Thomas!)
  * handle no rendered build commit being available in MGR / MON lists,
returning the icon + version without the commit instead
  * make the modified logic in Services.js more readable
  * reword message about differing builds in the overview
  * reword commit title & message
  * add 'Fixes' trailer
  * remove outdated R-b and T-b trailers

Changes v1 --> v2:
  * use camelCase instead of snake_case (thanks Lukas!)
  * use more descriptive variable names (thanks Lukas!)
  * use `let` instead of `const` for variables where applicable (thanks Lukas!)

 www/manager6/Utils.js| 107 +++
 www/manager6/ceph/ServiceList.js |  33 +++---
 www/manager6/ceph/Services.js|  19 +-
 3 files changed, 148 insertions(+), 11 deletions(-)

diff --git a/www/manager6/Utils.js b/www/manager6/Utils.js
index db86fa9a..1d42be34 100644
--- a/www/manager6/Utils.js
+++ b/www/manager6/Utils.js
@@ -128,6 +128,113 @@ Ext.define('PVE.Utils', {
return undefined;
 },
 
+/**
+ * Parses a Ceph build commit from its version string.
+ *
+ * @param {object} service
+ * @param {string} service.ceph_version - The long form of a Ceph version 
string
+ * @returns {(undefined|string)} The matched build commit or `undefined`
+ */
+parseCephBuildCommit: function(service) {
+   if (service.ceph_version) {
+   // See PVE/Ceph/Tools.pm - get_local_version
+   const match = service.ceph_version.match(
+   /^ceph.*\sv?(?:\d+(?:\.\d+)+)\s+(?:\(([a-zA-Z0-9]+)\))/,
+   );
+   if (match) {
+   return match[1];
+   }
+   }
+
+   return undefined;
+},
+
+/**
+ * Determines how Ceph build commits should be rendered and which icon
+ * should be chosen depending on the Ceph versions and build commits given.
+ *
+ * Should the two build commits be identical, the commit is shortened.
+ * Should they differ, they're rendered as e.g.:
+ *
+ * 0558a7146 -> b72f4fd04
+ *
+ * Note that the arrow `->` is rendered as a Font Awesome icon.
+ *
+ * @param {Object} versionInfo
+ * @param {string} versionInfo.runningVersion - The version of the 
currently
+ * running Ceph service
+ * @param {string} versionInfo.installedVersion - The version of the Ceph
+ * service (or its executable) as reported by the node
+ * @param {string} versionInfo.runningBuildCommit - The build commit of the
+ * currently running Ceph service
+ * @param {string} versionInfo.installedBuildCommit - The build commit of 
the
+ * Ceph service (or its executable) as reported by the node
+ * @param {string} versionInfo.maxVersion - The highest version of the
+ * Ceph service in the cluster
+ * @param {boolean} versionInfo.hasMixedVersions - Whether different
+ * versions of the service are currently running in the cluster
+ * @param {number} [buildCommitLength] - (Optional) The length to which to
+ * shorten the rendered build commit(s), `9` by default
+ * @returns {Object} - An object containing the selected `icon`, whether 
the
+ * current build commit has `changed` and the build commit (or build
+ * co

Re: [pve-devel] [PATCH v2 pve-manager 06/10] ui: ceph: services: parse and display build commit

[Max Carrara]

On Mon Jul 22, 2024 at 5:38 PM CEST, Thomas Lamprecht wrote:
> Am 01/07/2024 um 16:10 schrieb Max Carrara:
> > This commit adds `PVE.Utils.parseCephBuildCommit`, which can be used
> > to get the full hash "eccf199d..." in parentheses from a string like
> > the following:
> > 
> >   ceph version 17.2.7 (eccf199d63457659c09677399928203b7903c888) quincy 
> > (stable)
> > 
> > This hash is displayed and taken into account when comparing monitor
> > and manager versions in the client. Specifically, the shortened build
> > commit is now displayed in parentheses next to the version for both
> > monitors and managers like so:
> > 
> >   18.2.2 (abcd1234)
> > 
> > Should the build commit of the running service differ from the one
> > that's installed on the host, the newer build commit will also be
> > shown in parentheses:
> > 
> >   18.2.2 (abcd1234 -> 5678fedc)
> > 
> > The icon displayed for running a service with an outdated build is the
> > same as for running an outdated version. The conditional display of
> > cluster health-related icons remains the same otherwise.
> > 
> > The Ceph summary view also displays the hash and will show a warning
> > if a service is running with a build commit that doesn't match the one
> > that's advertised by the host.
> > 
> > Signed-off-by: Max Carrara 
> > Tested-by: Lukas Wagner 
> > Reviewed-by: Lukas Wagner 
> > ---
> > Changes v1 --> v2:
> >   * use camelCase instead of snake_case (thanks Lukas!)
> >   * use more descriptive variable names (thanks Lukas!)
> >   * use `let` instead of `const` for variables where applicable (thanks 
> > Lukas!)
> > 
> >  www/manager6/Utils.js| 14 ++
> >  www/manager6/ceph/ServiceList.js | 32 ++--
> >  www/manager6/ceph/Services.js| 14 +-
> >  3 files changed, 53 insertions(+), 7 deletions(-)
> > 
> > diff --git a/www/manager6/Utils.js b/www/manager6/Utils.js
> > index 74e46694..f2fd0f7e 100644
> > --- a/www/manager6/Utils.js
> > +++ b/www/manager6/Utils.js
> > @@ -128,6 +128,20 @@ Ext.define('PVE.Utils', {
> > return undefined;
> >  },
> >  
> > +parseCephBuildCommit: function(service) {
> > +   if (service.ceph_version) {
> > +   // See PVE/Ceph/Tools.pm - get_local_version
> > +   const match = service.ceph_version.match(
> > +   /^ceph.*\sv?(?:\d+(?:\.\d+)+)\s+(?:\(([a-zA-Z0-9]+)\))/,
> > +   );
> > +   if (match) {
> > +   return match[1];
> > +   }
> > +   }
> > +
> > +   return undefined;
> > +},
> > +
> >  compare_ceph_versions: function(a, b) {
> > let avers = [];
> > let bvers = [];
> > diff --git a/www/manager6/ceph/ServiceList.js 
> > b/www/manager6/ceph/ServiceList.js
> > index 76710146..d994aa4e 100644
> > --- a/www/manager6/ceph/ServiceList.js
> > +++ b/www/manager6/ceph/ServiceList.js
> > @@ -102,21 +102,41 @@ Ext.define('PVE.node.CephServiceController', {
> > if (value === undefined) {
> > return '';
> > }
> > +
> > +   let buildCommit = PVE.Utils.parseCephBuildCommit(rec.data) ?? '';
> > +
> > let view = this.getView();
> > -   let host = rec.data.host, nodev = [0];
> > +   let host = rec.data.host;
> > +
> > +   let versionNode = [0];
> > +   let buildCommitNode = '';
> > if (view.nodeversions[host] !== undefined) {
> > -   nodev = view.nodeversions[host].version.parts;
> > +   versionNode = view.nodeversions[host].version.parts;
> > +   buildCommitNode = view.nodeversions[host].buildcommit;
> > }
> >  
> > +   let bcChanged =
>
> I'd prefer the more telling `buildCommitChanged` variable name here.
>
> > +   buildCommit !== '' &&
> > +   buildCommitNode !== '' &&
> > +   buildCommit !== buildCommitNode;
>
> above hunk and... 
>
> > +
> > let icon = '';
> > -   if (PVE.Utils.compare_ceph_versions(view.maxversion, nodev) > 0) {
> > +   if (PVE.Utils.compare_ceph_versions(view.maxversion, versionNode) > 0) {
> > icon = PVE.Utils.get_ceph_icon_html('HEALTH_UPGRADE');
> > -   } else if (PVE.Utils.compare_ceph_versions(nodev, value) > 0) {
> > +   } else if (PVE.Utils.compare_ceph_versions(versionNode, value) > 0) {
> > icon = PVE.Utils.

Re: [pve-devel] [pbs-devel] [PATCH manager v2 07/12] api: add routes for webhook notification endpoints

[Max Carrara]

On Mon Jul 22, 2024 at 9:37 AM CEST, Lukas Wagner wrote:
>
>
> On  2024-07-17 17:36, Max Carrara wrote:
> > On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> >> These just call the API implementation via the perl-rs bindings.
> >>
> >> Signed-off-by: Lukas Wagner 
> >> ---
> >>  PVE/API2/Cluster/Notifications.pm | 263 +-
> >>  1 file changed, 262 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/PVE/API2/Cluster/Notifications.pm 
> >> b/PVE/API2/Cluster/Notifications.pm
> >> index 10b611c9..eae2d436 100644
> >> --- a/PVE/API2/Cluster/Notifications.pm
> >> +++ b/PVE/API2/Cluster/Notifications.pm
> >> @@ -108,6 +108,7 @@ __PACKAGE__->register_method ({
> >>{ name => 'gotify' },
> >>{ name => 'sendmail' },
> >>{ name => 'smtp' },
> >> +  { name => 'webhook' },
> >>];
> >>  
> >>return $result;
> >> @@ -144,7 +145,7 @@ __PACKAGE__->register_method ({
> >>'type' => {
> >>description => 'Type of the target.',
> >>type  => 'string',
> >> -  enum => [qw(sendmail gotify smtp)],
> >> +  enum => [qw(sendmail gotify smtp webhook)],
> >>},
> >>'comment' => {
> >>description => 'Comment',
> >> @@ -1094,6 +1095,266 @@ __PACKAGE__->register_method ({
> >>  }
> >>  });
> >>  
> >> +my $webhook_properties = {
> >> +name => {
> >> +  description => 'The name of the endpoint.',
> >> +  type => 'string',
> >> +  format => 'pve-configid',
> >> +},
> >> +url => {
> >> +  description => 'Server URL',
> >> +  type => 'string',
> >> +},
> >> +method => {
> >> +  description => 'HTTP method',
> >> +  type => 'string',
> >> +  enum => [qw(post put get)],
> >> +},
> >> +header => {
> >> +  description => 'HTTP headers to set. These have to be formatted as'
> >> +. ' a property string in the format name=,value= >> value>',
> >> +  type => 'array',
> >> +  items => {
> >> +  type => 'string',
> >> +  },
> >> +  optional => 1,
> >> +},
> >> +body => {
> >> +  description => 'HTTP body, base64 encoded',
> >> +  type => 'string',
> >> +  optional => 1,
> >> +},
> >> +secret => {
> >> +  description => 'Secrets to set. These have to be formatted as'
> >> +. ' a property string in the format name=,value= >> value>',
> >> +  type => 'array',
> >> +  items => {
> >> +  type => 'string',
> >> +  },
> >> +  optional => 1,
> >> +},
> >> +comment => {
> >> +  description => 'Comment',
> >> +  type => 'string',
> >> +  optional => 1,
> >> +},
> >> +disable => {
> >> +  description => 'Disable this target',
> >> +  type => 'boolean',
> >> +  optional => 1,
> >> +  default => 0,
> >> +},
> >> +};
> >> +
> >> +__PACKAGE__->register_method ({
> >> +name => 'get_webhook_endpoints',
> >> +path => 'endpoints/webhook',
> >> +method => 'GET',
> >> +description => 'Returns a list of all webhook endpoints',
> >> +protected => 1,
> >> +permissions => {
> >> +  check => ['perm', '/mapping/notifications', ['Mapping.Modify']],
> >> +  check => ['perm', '/mapping/notifications', ['Mapping.Audit']],
> >> +},
> >> +parameters => {
> >> +  additionalProperties => 0,
> >> +  properties => {},
> >> +},
> >> +returns => {
> >> +  type => 'array',
> >> +  items => {
> >> +  type => 'object',
> >> +  properties => {
> >> +  %$webhook_properties,
> > 
> > Would prefer `$webhook_properties->%*` here (postfix dereferencing) -
> > even though not explicitly stated in our style guide, we use that kind
> > of syntax for calling subroutines behind a reference, e.g.
> > `$foo->($arg)` instead of `&$foo($arg)`.
> > 
>
> I kinda prefer the brevity of the prefix variant in this case. Are there
> any pitfalls/problems with the prefix that I'm not aware of? If not, I'd 
> prefer
> to keep this as is, I used the syntax in many other spots in this file ;)

I personally have no hard feelings if you keep it tbh. Postfix
dereference is mainly useful if you have e.g. a nested hash (or rather,
makes more sense) because of how the code is usually read. For example,

%$foo->{bar}->{baz}

vs

$foo->{bar}->{baz}->%*

I'd argue that the main benefit is that it's easier to read for people
who aren't as familiar with Perl, but before this gets too bikesheddy,
I'm personally fine if you keep it as-is for simple cases like the above
:P


___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [pbs-devel] [PATCH proxmox v2 01/12] notify: implement webhook targets

[Max Carrara]

On Mon Jul 22, 2024 at 9:30 AM CEST, Lukas Wagner wrote:
>
>
> On  2024-07-17 17:35, Max Carrara wrote:
> >> +let handlebars = setup_handlebars();
> >> +let body_template = 
> >> self.base_64_decode(self.config.body.as_deref().unwrap_or_default())?;
> >> +
> >> +let body = handlebars
> >> +.render_template(&body_template, &data)
> >> +.map_err(|err| {
> >> +// TODO: Cleanup error types, they have become a bit 
> >> messy.
> >> +// No user of the notify crate distinguish between the 
> >> error types any way, so
> >> +// we can refactor without any issues
> >> +Error::Generic(format!("failed to render webhook body: 
> >> {err}"))
> > 
> > I'm curious, how would you clean up the error types in particular?
> > 
>
> Right now, error handling is a bit messy... Some endpoints primarily use
> the `NotifyFailed` variant, which wraps another error, while in some places
> where I need a leaf error type that does not wrap any error I use the
> `Generic` variant, which only stores a string.
> I could have used the `NotifyFailed` variant here, but that one would
> not have allowed me to add additional context ("failed to render webhook 
> ..."),
> unless wrapping another `Generic` variant...
>
> I don't have yet made any detailed plans on how to clean that up though.

Hmm, I see - thanks for elaborating! I'm not really sure how to clean
them up either, but if I can think of something, I'll let you know.



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH v2 pve-manager 00/10] Ceph Build Commit in UI

[Max Carrara]

On Fri Jul 19, 2024 at 2:17 PM CEST, Igor Moritz Thaller wrote:
> I have tested the new ceph GUI feature where if a cluster has an outdated 
> ceph version running, it will inform the user.
>
> My setup consisted of a ceph cluster with three monitors and four nodes, each 
> having their own two osds. Since I didn't want to rebuild ceph I instead 
> modified the 'CEPH_GIT_VER' variable in the python file '/usr/bin/ceph'. I 
> changed the ceph version multiple times to different versions/non-versions, 
> and it correctly updated the GUI with a warning that the currently running 
> version was outdated.
>
> Overall, from what I have tested, it works great!
>
> Tested-by: Igor Thaller 

Thanks a lot, appreciate it!

> ____________
> Von: pve-devel  im Auftrag von Max 
> Carrara 
> Gesendet: Montag, 1. Juli 2024 16:10
> An: pve-devel@lists.proxmox.com
> Cc: Lukas Wagner
> Betreff: [pve-devel] [PATCH v2 pve-manager 00/10] Ceph Build Commit in UI
>
> Ceph Build Commit in UI - Version 2
> ===
>
> Notable Changes since v1
> 
>
>   * Use camelCase instead of snake_case for new functions / variables
> as per our style guide [0] (thanks Lukas!)
>   * Refrain from using `const` for things that aren't actual constants
> as per our style guide [1] (thanks Lukas!)
>   * NEW: Patch 09: Increase the default width of the version field in
> the OSD tree so that longer strings are immediately readable without
> needing to adjust the column widths manually
> --> e.g. "18.2.2 (e9fe820e7 -> 69ce99eba)" takes up a lot of space
> in the column
>   * NEW: Patch 10: Include Ceph build commit in the version string
> which is part of the object of the `ceph/osd/{osdid}/metadata` call
>
> For a detailed list of changes, please see the comments in the
> individual patches.
>
> NOTE: I added Lukas's T-b and R-b tags to all patches except the new
> ones, as mentioned in a reply to v1 [2].
>
> Older Versions
> --
>
> v1: https://lists.proxmox.com/pipermail/pve-devel/2024-April/063772.html
>
> References
> --
>
> [0]: https://pve.proxmox.com/wiki/Javascript_Style_Guide#Casing
> [1]: https://pve.proxmox.com/wiki/Javascript_Style_Guide#Variables
> [2]: https://lists.proxmox.com/pipermail/pve-devel/2024-June/064084.html
>
> Summary of Changes
> --
>
> Max Carrara (10):
>   ceph: tools: refactor installation check as guard clause
>   ceph: tools: parse Ceph version in separate sub and update regex
>   ceph: services: remove old cluster broadcast
>   ceph: services: refactor version existence check as guard clause
>   utils: align regex of parse_ceph_version with Perl equivalent
>   ui: ceph: services: parse and display build commit
>   api: ceph: add build commit of host to Ceph osd index endpoint data
>   ui: ceph: osd: rework rendering of version field & show build commit
>   ui: ceph: osd: increase width of version column
>   api: ceph: change version format in OSD metadata endpoint
>
>  PVE/API2/Ceph/OSD.pm |  9 -
>  PVE/Ceph/Services.pm | 38 ++--
>  PVE/Ceph/Tools.pm| 59 ++--
>  www/manager6/Utils.js| 17 -
>  www/manager6/ceph/OSD.js | 57 +-
>  www/manager6/ceph/ServiceList.js | 32 +
>  www/manager6/ceph/Services.js| 14 +++-
>  7 files changed, 170 insertions(+), 56 deletions(-)
>
> --
> 2.39.2
>
>
>
> ___
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
>
>
> ___
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [RFC pve-storage 01/36] plugin: base: remove old fixme comments

[Max Carrara]

On Wed Jul 17, 2024 at 6:02 PM CEST, Thomas Lamprecht wrote:
> Am 17/07/2024 um 11:39 schrieb Max Carrara:
> > These have been around since 2012 - suffice to say they're not needed
> > anymore.
>
> That's really not a good argument though? Just because nobody checked
> those closely for a long time it does not mean that they became
> magically irrelevant.
>
> Look, it can be (and probably _is_) fine to remove them, but stating
> that this is fine just because they were not touched since a while is a
> rather dangerous tactic. Someone had some thoughts when adding this,
> they might be still relevant or not, but it's definitively *not*
> "suffice to say" that they aren't just because they have been around
> since 2012, (iSCSI) portals and local storage still exist and are not
> working really different compared to 12 years ago.
>
> The node restriction FIXME comment can e.g. be removed as we delete any
> such restriction in "parse_config", mentioning that as a reason would
> make doing so fine, saying "it's old and unchanged" doesn't.
>
> The storage portal one could be argued with not being defined elsewhere
> and all use cases being covered by pve-storage-portal-dns, so removing
> it won't hurt, especially as we can always recover it from history.
>
> I think your intention quite surely matched those and meant well, but
> removing something just because it's old is on its own IMO a bit of a
> red flag, so one should get too used to that argumentation style even
> if it's for removing comments, or other stuff that won't change semantics.

I completely agree with you, I probably should've stated a better reason
there. IIRC I removed those two things for a valid reason, but because
the commit was made a while ago, I'm not actually sure anymore what they
were exactly. I guess this proves your point. ;)

In a future RFC / Series, this will definitely be updated. Thanks for
pointing that out.

>
> Anyhow, do not let this demotivate you from your clean-up efforts, they
> are still quite appreciated.
> While removing dead code is good, the argumentation behind should be
> sound, and I only write this long tirade (sorry) as we got bitten by
> some innocent looking changes stemming from a similar argumentation in
> the past.

No worries, no offense taken here - I really appreciate comment.
Sometimes these things do need to be pointed out, because e.g. for me
personally it just wasn't on my radar that a commit like this could
become a tough to debug issue in case things go south. That's probably
because I've never had to deal with debugging such a thing myself.

So again, no worries, I appreciate it!



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [pbs-devel] [PATCH manager v2 07/12] api: add routes for webhook notification endpoints

[Max Carrara]

On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> These just call the API implementation via the perl-rs bindings.
>
> Signed-off-by: Lukas Wagner 
> ---
>  PVE/API2/Cluster/Notifications.pm | 263 +-
>  1 file changed, 262 insertions(+), 1 deletion(-)
>
> diff --git a/PVE/API2/Cluster/Notifications.pm 
> b/PVE/API2/Cluster/Notifications.pm
> index 10b611c9..eae2d436 100644
> --- a/PVE/API2/Cluster/Notifications.pm
> +++ b/PVE/API2/Cluster/Notifications.pm
> @@ -108,6 +108,7 @@ __PACKAGE__->register_method ({
>   { name => 'gotify' },
>   { name => 'sendmail' },
>   { name => 'smtp' },
> + { name => 'webhook' },
>   ];
>  
>   return $result;
> @@ -144,7 +145,7 @@ __PACKAGE__->register_method ({
>   'type' => {
>   description => 'Type of the target.',
>   type  => 'string',
> - enum => [qw(sendmail gotify smtp)],
> + enum => [qw(sendmail gotify smtp webhook)],
>   },
>   'comment' => {
>   description => 'Comment',
> @@ -1094,6 +1095,266 @@ __PACKAGE__->register_method ({
>  }
>  });
>  
> +my $webhook_properties = {
> +name => {
> + description => 'The name of the endpoint.',
> + type => 'string',
> + format => 'pve-configid',
> +},
> +url => {
> + description => 'Server URL',
> + type => 'string',
> +},
> +method => {
> + description => 'HTTP method',
> + type => 'string',
> + enum => [qw(post put get)],
> +},
> +header => {
> + description => 'HTTP headers to set. These have to be formatted as'
> +   . ' a property string in the format name=,value= value>',
> + type => 'array',
> + items => {
> + type => 'string',
> + },
> + optional => 1,
> +},
> +body => {
> + description => 'HTTP body, base64 encoded',
> + type => 'string',
> + optional => 1,
> +},
> +secret => {
> + description => 'Secrets to set. These have to be formatted as'
> +   . ' a property string in the format name=,value= value>',
> + type => 'array',
> + items => {
> + type => 'string',
> + },
> + optional => 1,
> +},
> +comment => {
> + description => 'Comment',
> + type => 'string',
> + optional => 1,
> +},
> +disable => {
> + description => 'Disable this target',
> + type => 'boolean',
> + optional => 1,
> + default => 0,
> +},
> +};
> +
> +__PACKAGE__->register_method ({
> +name => 'get_webhook_endpoints',
> +path => 'endpoints/webhook',
> +method => 'GET',
> +description => 'Returns a list of all webhook endpoints',
> +protected => 1,
> +permissions => {
> + check => ['perm', '/mapping/notifications', ['Mapping.Modify']],
> + check => ['perm', '/mapping/notifications', ['Mapping.Audit']],
> +},
> +parameters => {
> + additionalProperties => 0,
> + properties => {},
> +},
> +returns => {
> + type => 'array',
> + items => {
> + type => 'object',
> + properties => {
> + %$webhook_properties,

Would prefer `$webhook_properties->%*` here (postfix dereferencing) -
even though not explicitly stated in our style guide, we use that kind
of syntax for calling subroutines behind a reference, e.g.
`$foo->($arg)` instead of `&$foo($arg)`.

> + 'origin' => {
> + description => 'Show if this entry was created by a user or 
> was built-in',
> + type  => 'string',
> + enum => [qw(user-created builtin modified-builtin)],
> + },
> + },
> + },
> + links => [ { rel => 'child', href => '{name}' } ],
> +},
> +code => sub {
> + my $config = PVE::Notify::read_config();
> + my $rpcenv = PVE::RPCEnvironment::get();
> +
> + my $entities = eval {
> + $config->get_webhook_endpoints();
> + };
> + raise_api_error($@) if $@;
> +
> + return $entities;
> +}
> +});
> +
> +__PACKAGE__->register_method ({
> +name => 'get_webhook_endpoint',
> +path => 'endpoints/webhook/{name}',
> +method => 'GET',
> +description => 'Return a specific webhook endpoint',
> +protected => 1,
> +permissions => {
> + check => ['or',
> + ['perm', '/mapping/notifications', ['Mapping.Modify']],
> + ['perm', '/mapping/notifications', ['Mapping.Audit']],
> + ],
> +},
> +parameters => {
> + additionalProperties => 0,
> + properties => {
> + name => {
> + type => 'string',
> + format => 'pve-configid',
> + description => 'Name of the endpoint.'
> + },
> + }
> +},
> +returns => {
> + type => 'object',
> + properties => {
> + %$webhook_properties,

Same here :)

> + digest => get_standard_option('pve-config-digest'),
> + }
> +},
> +

Re: [pve-devel] [pbs-devel] [PATCH proxmox-perl-rs v2 04/12] common: notify: add bindings for get_targets

[Max Carrara]

And missed `cargo fmt` here too ;)

On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> This allows us to drop the impl of that function on the perl side.
>
> Signed-off-by: Lukas Wagner 
> ---
>  common/src/notify.rs | 9 +
>  1 file changed, 9 insertions(+)
>
> diff --git a/common/src/notify.rs b/common/src/notify.rs
> index fe192d5..0f8a35d 100644
> --- a/common/src/notify.rs
> +++ b/common/src/notify.rs
> @@ -27,6 +27,7 @@ mod export {
>  MatcherConfigUpdater, SeverityMatcher,
>  };
>  use proxmox_notify::{api, Config, Notification, Severity};
> +use proxmox_notify::api::Target;
>  
>  pub struct NotificationConfig {
>  config: Mutex,
> @@ -112,6 +113,14 @@ mod export {
>  api::common::send(&config, ¬ification)
>  }
>  
> +#[export(serialize_error)]
> +fn get_targets(
> +#[try_from_ref] this: &NotificationConfig,
> +) -> Result, HttpError> {
> +let config = this.config.lock().unwrap();
> +api::get_targets(&config)
> +}
> +
>  #[export(serialize_error)]
>  fn test_target(
>  #[try_from_ref] this: &NotificationConfig,



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH proxmox-perl-rs v2 03/12] common: notify: add bindings for webhook API routes

[Max Carrara]

Missed a `cargo fmt` here as well ;)

On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> Signed-off-by: Lukas Wagner 
> ---
>  common/src/notify.rs | 63 
>  1 file changed, 63 insertions(+)
>
> diff --git a/common/src/notify.rs b/common/src/notify.rs
> index e1b006b..fe192d5 100644
> --- a/common/src/notify.rs
> +++ b/common/src/notify.rs
> @@ -19,6 +19,9 @@ mod export {
>  DeleteableSmtpProperty, SmtpConfig, SmtpConfigUpdater, SmtpMode, 
> SmtpPrivateConfig,
>  SmtpPrivateConfigUpdater,
>  };
> +use proxmox_notify::endpoints::webhook::{
> +DeleteableWebhookProperty, WebhookConfig, WebhookConfigUpdater,
> +};
>  use proxmox_notify::matcher::{
>  CalendarMatcher, DeleteableMatcherProperty, FieldMatcher, 
> MatchModeOperator, MatcherConfig,
>  MatcherConfigUpdater, SeverityMatcher,
> @@ -393,6 +396,66 @@ mod export {
>  api::smtp::delete_endpoint(&mut config, name)
>  }
>  
> +#[export(serialize_error)]
> +fn get_webhook_endpoints(
> +#[try_from_ref] this: &NotificationConfig,
> +) -> Result, HttpError> {
> +let config = this.config.lock().unwrap();
> +api::webhook::get_endpoints(&config)
> +}
> +
> +#[export(serialize_error)]
> +fn get_webhook_endpoint(
> +#[try_from_ref] this: &NotificationConfig,
> +id: &str,
> +) -> Result {
> +let config = this.config.lock().unwrap();
> +api::webhook::get_endpoint(&config, id)
> +}
> +
> +#[export(serialize_error)]
> +#[allow(clippy::too_many_arguments)]
> +fn add_webhook_endpoint(
> +#[try_from_ref] this: &NotificationConfig,
> +endpoint_config: WebhookConfig,
> +) -> Result<(), HttpError> {
> +let mut config = this.config.lock().unwrap();
> +api::webhook::add_endpoint(
> +&mut config,
> +endpoint_config,
> +)
> +}
> +
> +#[export(serialize_error)]
> +#[allow(clippy::too_many_arguments)]
> +fn update_webhook_endpoint(
> +#[try_from_ref] this: &NotificationConfig,
> +name: &str,
> +config_updater: WebhookConfigUpdater,
> +delete: Option>,
> +digest: Option<&str>,
> +) -> Result<(), HttpError> {
> +let mut config = this.config.lock().unwrap();
> +let digest = decode_digest(digest)?;
> +
> +api::webhook::update_endpoint(
> +&mut config,
> +name,
> +config_updater,
> +delete.as_deref(),
> +digest.as_deref(),
> +)
> +}
> +
> +#[export(serialize_error)]
> +fn delete_webhook_endpoint(
> +#[try_from_ref] this: &NotificationConfig,
> +name: &str,
> +) -> Result<(), HttpError> {
> +let mut config = this.config.lock().unwrap();
> +api::webhook::delete_endpoint(&mut config, name)
> +}
> +
>  #[export(serialize_error)]
>  fn get_matchers(
>  #[try_from_ref] this: &NotificationConfig,



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH proxmox v2 02/12] notify: add api for webhook targets

[Max Carrara]

On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> All in all pretty similar to other endpoint APIs.
> One thing worth noting is how secrets are handled. We never ever
> return the values of previously stored secrets in get_endpoint(s)
> calls, but only a list of the names of all secrets. This is needed
> to build the UI, where we display all secrets that were set before in
> a table.
>
> For update calls, one is supposed to send all secrets that should be
> kept and updated. If the value should be updated, the name and value
> is expected, and if the current value should preseved, only the name
> is sent. If a secret's name is not present in the updater, it will be
> dropped. If 'secret' is present in the 'delete' array, all secrets
> will be dropped, apart from those which are also set/preserved in the
> same update call.
>
> Signed-off-by: Lukas Wagner 
> ---
>  proxmox-notify/src/api/mod.rs |  20 ++
>  proxmox-notify/src/api/webhook.rs | 406 ++
>  2 files changed, 426 insertions(+)
>  create mode 100644 proxmox-notify/src/api/webhook.rs
>
> diff --git a/proxmox-notify/src/api/mod.rs b/proxmox-notify/src/api/mod.rs
> index a7f6261c..7f823bc7 100644
> --- a/proxmox-notify/src/api/mod.rs
> +++ b/proxmox-notify/src/api/mod.rs
> @@ -15,6 +15,8 @@ pub mod matcher;
>  pub mod sendmail;
>  #[cfg(feature = "smtp")]
>  pub mod smtp;
> +#[cfg(feature = "webhook")]
> +pub mod webhook;
>  
>  // We have our own, local versions of http_err and http_bail, because
>  // we don't want to wrap the error in anyhow::Error. If we were to do that,
> @@ -54,6 +56,9 @@ pub enum EndpointType {
>  /// Gotify endpoint
>  #[cfg(feature = "gotify")]
>  Gotify,
> +/// Webhook endpoint
> +#[cfg(feature = "webhook")]
> +Webhook,
>  }
>  
>  #[api]
> @@ -113,6 +118,17 @@ pub fn get_targets(config: &Config) -> 
> Result, HttpError> {
>  })
>  }
>  
> +#[cfg(feature = "webhook")]
> +for endpoint in webhook::get_endpoints(config)? {
> +targets.push(Target {
> +name: endpoint.name,
> +origin: endpoint.origin.unwrap_or(Origin::UserCreated),
> +endpoint_type: EndpointType::Webhook,
> +disable: endpoint.disable,
> +comment: endpoint.comment,
> +})
> +}
> +
>  Ok(targets)
>  }
>  
> @@ -145,6 +161,10 @@ fn ensure_endpoint_exists(#[allow(unused)] config: 
> &Config, name: &str) -> Resul
>  {
>  exists = exists || smtp::get_endpoint(config, name).is_ok();
>  }
> +#[cfg(feature = "webhook")]
> +{
> +exists = exists || webhook::get_endpoint(config, name).is_ok();
> +}
>  
>  if !exists {
>  http_bail!(NOT_FOUND, "endpoint '{name}' does not exist")
> diff --git a/proxmox-notify/src/api/webhook.rs 
> b/proxmox-notify/src/api/webhook.rs
> new file mode 100644
> index ..b7f17c55
> --- /dev/null
> +++ b/proxmox-notify/src/api/webhook.rs
> @@ -0,0 +1,406 @@
> +use proxmox_http_error::HttpError;
> +use proxmox_schema::property_string::PropertyString;
> +
> +use crate::api::http_err;
> +use crate::endpoints::webhook::{
> +DeleteableWebhookProperty, KeyAndBase64Val, WebhookConfig, 
> WebhookConfigUpdater,
> +WebhookPrivateConfig, WEBHOOK_TYPENAME,
> +};
> +use crate::{http_bail, Config};
> +
> +use super::remove_private_config_entry;
> +use super::set_private_config_entry;
> +
> +/// Get a list of all webhook endpoints.
> +///
> +/// The caller is responsible for any needed permission checks.
> +/// Returns a list of all webhook endpoints or a `HttpError` if the config is
> +/// erroneous (`500 Internal server error`).
> +pub fn get_endpoints(config: &Config) -> Result, 
> HttpError> {
> +let mut endpoints: Vec = config
> +.config
> +.convert_to_typed_array(WEBHOOK_TYPENAME)
> +.map_err(|e| http_err!(NOT_FOUND, "Could not fetch endpoints: 
> {e}"))?;
> +
> +for endpoint in &mut endpoints {
> +let priv_config: WebhookPrivateConfig = config
> +.private_config
> +.lookup(WEBHOOK_TYPENAME, &endpoint.name)
> +.unwrap_or_default();
> +
> +let mut secret_names = Vec::new();
> +for secret in priv_config.secret {
> +secret_names.push(
> +KeyAndBase64Val {
> +name: secret.name.clone(),
> +value: None,
> +}
> +.into(),
> +)
> +}
> +
> +endpoint.secret = secret_names;
> +}
> +
> +Ok(endpoints)
> +}
> +
> +/// Get webhook endpoint with given `name`
> +///
> +/// The caller is responsible for any needed permission checks.
> +/// Returns the endpoint or a `HttpError` if the endpoint was not found 
> (`404 Not found`).
> +pub fn get_endpoint(config: &Config, name: &str) -> Result HttpError> {
> +let mut endpoint: WebhookConfig = config
> +.config
> +.lookup(WEBHOOK_TYPENAME, name)
> +.map_err

Re: [pve-devel] [RFC many v2 00/12] notifications: add support for webhook endpoints

[Max Carrara]

On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> Sending as an RFC because I don't want this merged yet; that being
> said, the feature should be mostly finished at this point, I'd
> appreciate any reviews and feedback.
>
> This series adds support for webhook notification targets to PVE
> and PBS.
>
> A webhook is a HTTP API route provided by a third-party service that
> can be used to inform the third-party about an event. In our case,
> we can easily interact with various third-party notification/messaging
> systems and send PVE/PBS notifications via this service.
> The changes were tested against ntfy.sh, Discord and Slack.
>
> The configuration of webhook targets allows one to configure:
>   - The URL
>   - The HTTP method (GET/POST/PUT)
>   - HTTP Headers
>   - Body
>
> One can use handlebar templating to inject notification text and metadata
> in the url, headers and body.
>
> One challenge is the handling of sensitve tokens and other secrets.
> Since the endpoint is completely generic, we cannot know in advance
> whether the body/header/url contains sensitive values.
> Thus we add 'secrets' which are stored in the protected config only
> accessible by root (e.g. /etc/pve/priv/notifications.cfg). These
> secrets are accessible in URLs/headers/body via templating:
>
>   Url: https://example.com/{{ secrets.token }}
>
> Secrets can only be set and updated, but never retrieved via the API.
> In the UI, secrets are handled like other secret tokens/passwords.
>
> Bumps for PVE:
>   - libpve-rs-perl needs proxmox-notify bumped
>   - pve-manager needs bumped proxmox-widget-toolkit and libpve-rs-perl bumped
>   - proxmox-mail-forward needs proxmox-notify bumped
>
> Bumps for PBS:
>   - proxmox-backup needs proxmox-notify bumped
>   - proxmox-mail-forward needs proxmox-notify bumped

Since this is an RFC, I mainly just did some proofreading; I haven't
really spotted anything out of the ordinary, apart from a few *very
small* things I commented on inline.

I like the overall idea of adding webhooks, so this looks pretty solid
to me. At first I thought that this might be a bit of a niche use case,
but I feel like it might actually be quite interesting for orgs that are
e.g. on Slack: You could e.g. just "route" all notifications via a
webhook to Slack, and Slack then sends a push notification to one's
phone. The same can obviously done with other applications / services as
well. So, pretty cool stuff :)

Not sure if this has been discussed somewhere already (off list etc.),
but could you elaborate on why you don't want this merged yet? The
patches look pretty solid to me, IMHO. Then again, I haven't really
tested them yet due to all the required package bumps, so take this with
a grain of salt.

If you want to have this RFC tested, I can of course give it a shot - do
let me know if that's the case :)

>
>
> Changes v1 -> v2:
>   - Rebase proxmox-notify changes
>
> proxmox:
>
> Lukas Wagner (2):
>   notify: implement webhook targets
>   notify: add api for webhook targets
>
>  proxmox-notify/Cargo.toml   |   9 +-
>  proxmox-notify/src/api/mod.rs   |  20 +
>  proxmox-notify/src/api/webhook.rs   | 406 +++
>  proxmox-notify/src/config.rs|  23 ++
>  proxmox-notify/src/endpoints/mod.rs |   2 +
>  proxmox-notify/src/endpoints/webhook.rs | 509 
>  proxmox-notify/src/lib.rs   |  17 +
>  7 files changed, 983 insertions(+), 3 deletions(-)
>  create mode 100644 proxmox-notify/src/api/webhook.rs
>  create mode 100644 proxmox-notify/src/endpoints/webhook.rs
>
>
> proxmox-perl-rs:
>
> Lukas Wagner (2):
>   common: notify: add bindings for webhook API routes
>   common: notify: add bindings for get_targets
>
>  common/src/notify.rs | 72 
>  1 file changed, 72 insertions(+)
>
>
> proxmox-widget-toolkit:
>
> Lukas Wagner (1):
>   notification: add UI for adding/updating webhook targets
>
>  src/Makefile  |   1 +
>  src/Schema.js |   5 +
>  src/panel/WebhookEditPanel.js | 417 ++
>  3 files changed, 423 insertions(+)
>  create mode 100644 src/panel/WebhookEditPanel.js
>
>
> pve-manager:
>
> Lukas Wagner (2):
>   api: notifications: use get_targets impl from proxmox-notify
>   api: add routes for webhook notification endpoints
>
>  PVE/API2/Cluster/Notifications.pm | 297 ++
>  1 file changed, 263 insertions(+), 34 deletions(-)
>
>
> pve-docs:
>
> Lukas Wagner (1):
>   notification: add documentation for webhook target endpoints.
>
>  notifications.adoc | 93 ++
>  1 file changed, 93 insertions(+)
>
>
> proxmox-backup:
>
> Lukas Wagner (3):
>   api: notification: add API routes for webhook targets
>   ui: utils: enable webhook edit window
>   docs: notification: add webhook endpoint documentation
>
>  docs/notifications.rst   | 100 +

Re: [pve-devel] [pbs-devel] [PATCH proxmox v2 01/12] notify: implement webhook targets

[Max Carrara]

On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> This target type allows users to perform HTTP requests to arbitrary
> third party (notification) services, for instance
> ntfy.sh/Discord/Slack.
>
> The configuration for these endpoints allows one to freely configure
> the URL, HTTP Method, headers and body. The URL, header values and
> body support handlebars templating to inject notification text,
> metadata and secrets. Secrets are stored in the protected
> configuration file (e.g. /etc/pve/priv/notification.cfg) as key value
> pairs, allowing users to protect sensitive tokens/passwords.
> Secrets are accessible in handlebar templating via the secrets.*
> namespace, e.g. if there is a secret named 'token', a body
> could contain '{{ secrets.token }}' to inject the token into the
> payload.
>
> A couple of handlebars helpers are also provided:
>   - url-encoding (useful for templating in URLs)
>   - escape (escape any control characters in strings)
>   - json (print a property as json)
>
> In the configuration, the body, header values and secret values
> are stored in base64 encoding so that we can store any string we want.
>
> Signed-off-by: Lukas Wagner 
> ---
>  proxmox-notify/Cargo.toml   |   9 +-
>  proxmox-notify/src/config.rs|  23 ++
>  proxmox-notify/src/endpoints/mod.rs |   2 +
>  proxmox-notify/src/endpoints/webhook.rs | 509 
>  proxmox-notify/src/lib.rs   |  17 +
>  5 files changed, 557 insertions(+), 3 deletions(-)
>  create mode 100644 proxmox-notify/src/endpoints/webhook.rs
>
> diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml
> index 7801814d..484aff19 100644
> --- a/proxmox-notify/Cargo.toml
> +++ b/proxmox-notify/Cargo.toml
> @@ -9,13 +9,15 @@ exclude.workspace = true
>  
>  [dependencies]
>  anyhow.workspace = true
> -base64.workspace = true
> +base64 = { workspace = true, optional = true }
>  const_format.workspace = true
>  handlebars = { workspace = true }
> +http = { workspace = true, optional = true }
>  lettre = { workspace = true, optional = true }
>  log.workspace = true
>  mail-parser = { workspace = true, optional = true }
>  openssl.workspace = true
> +percent-encoding = { workspace = true, optional = true }
>  regex.workspace = true
>  serde = { workspace = true, features = ["derive"] }
>  serde_json.workspace = true
> @@ -31,10 +33,11 @@ proxmox-time.workspace = true
>  proxmox-uuid = { workspace = true, features = ["serde"] }
>  
>  [features]
> -default = ["sendmail", "gotify", "smtp"]
> +default = ["sendmail", "gotify", "smtp", "webhook"]
>  mail-forwarder = ["dep:mail-parser", "dep:proxmox-sys"]
> -sendmail = ["dep:proxmox-sys"]
> +sendmail = ["dep:proxmox-sys", "dep:base64"]
>  gotify = ["dep:proxmox-http"]
>  pve-context = ["dep:proxmox-sys"]
>  pbs-context = ["dep:proxmox-sys"]
>  smtp = ["dep:lettre"]
> +webhook = ["dep:base64", "dep:http", "dep:percent-encoding", 
> "dep:proxmox-http"]
> diff --git a/proxmox-notify/src/config.rs b/proxmox-notify/src/config.rs
> index 789c4a7d..4d0b53f7 100644
> --- a/proxmox-notify/src/config.rs
> +++ b/proxmox-notify/src/config.rs
> @@ -57,6 +57,17 @@ fn config_init() -> SectionConfig {
>  GOTIFY_SCHEMA,
>  ));
>  }
> +#[cfg(feature = "webhook")]
> +{
> +use crate::endpoints::webhook::{WebhookConfig, WEBHOOK_TYPENAME};
> +
> +const WEBHOOK_SCHEMA: &ObjectSchema = 
> WebhookConfig::API_SCHEMA.unwrap_object_schema();
> +config.register_plugin(SectionConfigPlugin::new(
> +WEBHOOK_TYPENAME.to_string(),
> +Some(String::from("name")),
> +WEBHOOK_SCHEMA,
> +));
> +}
>  
>  const MATCHER_SCHEMA: &ObjectSchema = 
> MatcherConfig::API_SCHEMA.unwrap_object_schema();
>  config.register_plugin(SectionConfigPlugin::new(
> @@ -110,6 +121,18 @@ fn private_config_init() -> SectionConfig {
>  ));
>  }
>  
> +#[cfg(feature = "webhook")]
> +{
> +use crate::endpoints::webhook::{WebhookPrivateConfig, 
> WEBHOOK_TYPENAME};
> +
> +const WEBHOOK_SCHEMA: &ObjectSchema =
> +WebhookPrivateConfig::API_SCHEMA.unwrap_object_schema();
> +config.register_plugin(SectionConfigPlugin::new(
> +WEBHOOK_TYPENAME.to_string(),
> +Some(String::from("name")),
> +WEBHOOK_SCHEMA,
> +));
> +}
>  config
>  }
>  
> diff --git a/proxmox-notify/src/endpoints/mod.rs 
> b/proxmox-notify/src/endpoints/mod.rs
> index 97f79fcc..f20bee21 100644
> --- a/proxmox-notify/src/endpoints/mod.rs
> +++ b/proxmox-notify/src/endpoints/mod.rs
> @@ -4,5 +4,7 @@ pub mod gotify;
>  pub mod sendmail;
>  #[cfg(feature = "smtp")]
>  pub mod smtp;
> +#[cfg(feature = "webhook")]
> +pub mod webhook;
>  
>  mod common;
> diff --git a/proxmox-notify/src/endpoints/webhook.rs 
> b/proxmox-notify/src/endpoints/webhook.rs
> new file mode 100644
> index ..7e976f6b
> --- /dev/null
> +++ b/proxmox-notif

[pve-devel] [RFC pve-storage 36/36] plugin: zfspool: refactor method `zfs_request` into helper subroutine

[Max Carrara]

In order to separate the invocation of ZFS CLI utlis from the plugin,
the `zfs_request` method is refactored into a helper subroutine and
placed in the common ZFS module.

The signature is changed, removing the `$class` parameter. The body
remains the same, so no behaviour is actually altered.

The new helper sub is also documented and given a prototype.

The original method is changed to wrap the new helper and also emit a
deprecation warning when used.

The sites where the original method is called are updated to use the
helper subroutine instead.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common/ZFS.pm|  84 +++
 src/PVE/Storage/ZFSPoolPlugin.pm | 112 ++-
 2 files changed, 150 insertions(+), 46 deletions(-)

diff --git a/src/PVE/Storage/Common/ZFS.pm b/src/PVE/Storage/Common/ZFS.pm
index 21b28d9..327a592 100644
--- a/src/PVE/Storage/Common/ZFS.pm
+++ b/src/PVE/Storage/Common/ZFS.pm
@@ -3,10 +3,16 @@ package PVE::Storage::Common::ZFS;
 use strict;
 use warnings;
 
+use PVE::RPCEnvironment;
+use PVE::Tools qw(
+run_command
+);
+
 use parent qw(Exporter);
 
 our @EXPORT_OK = qw(
 zfs_parse_zvol_list
+zfs_request
 );
 
 =pod
@@ -21,6 +27,84 @@ PVE::Storage::Common::ZFS - Shared ZFS utilities and command 
line wrappers
 
 =pod
 
+=head3 zfs_request
+
+$output = zfs_request($scfg, $timeout, $method, @params)
+
+Wrapper that runs a ZFS command and returns its output.
+
+This subroutine has some special handling depending on which arguments are
+passed. See the description of the individual parameters below.
+
+=over
+
+=item C<$scfg>
+
+A section config hash. Unused and kept for backward compatibility.
+
+=item C<$timeout>
+
+Optional. The number of seconds to elapse before the wrapped command times out.
+
+If not provided, the invocation will time out after a default of C<10> seconds.
+
+If C<"zpool_import"> is passed as C<$method>, the timeout instead has a minimum
+of C<15> seconds and will automatically be increased if it is below.
+
+B: Should this subroutine be invoked in the context of an I or
+I worker, the above is disregarded and the timeout has a minimum of
+B, automatically increasing it if it is below. Should no timeout
+be provided in this case, the default is B instead.
+
+=item C<$method>
+
+The subcommand of the C CLI to run. This can be something like C<"get">
+(C), C<"list"> (C), etc.
+
+There are two exceptions, however: If C<$method> is C<"zpool_list"> or
+C<"zpool_import">, C or C will respectively be called
+instead.
+
+=item C<@params>
+
+Optional. Further parameters to pass along to the C or C CLI.
+
+=back
+
+=cut
+
+sub zfs_request : prototype($$$;@) {
+my ($scfg, $timeout, $method, @params) = @_;
+
+my $cmd = [];
+
+if ($method eq 'zpool_list') {
+   push @$cmd, 'zpool', 'list';
+} elsif ($method eq 'zpool_import') {
+   push @$cmd, 'zpool', 'import';
+   $timeout = 15 if !$timeout || $timeout < 15;
+} else {
+   push @$cmd, 'zfs', $method;
+}
+push @$cmd, @params;
+
+my $msg = '';
+my $output = sub { $msg .= "$_[0]\n" };
+
+if (PVE::RPCEnvironment->is_worker()) {
+   $timeout = 60*60 if !$timeout;
+   $timeout = 60*5 if $timeout < 60*5;
+} else {
+   $timeout = 10 if !$timeout;
+}
+
+run_command($cmd, errmsg => "zfs error", outfunc => $output, timeout => 
$timeout);
+
+return $msg;
+}
+
+=pod
+
 =head3 zfs_parse_zvol_list
 
 $zvol_list = zfs_parse_zvol_list($text, $pool)
diff --git a/src/PVE/Storage/ZFSPoolPlugin.pm b/src/PVE/Storage/ZFSPoolPlugin.pm
index fdfedca..c666166 100644
--- a/src/PVE/Storage/ZFSPoolPlugin.pm
+++ b/src/PVE/Storage/ZFSPoolPlugin.pm
@@ -132,31 +132,13 @@ sub path {
 sub zfs_request {
 my ($class, $scfg, $timeout, $method, @params) = @_;
 
-my $cmd = [];
-
-if ($method eq 'zpool_list') {
-   push @$cmd, 'zpool', 'list';
-} elsif ($method eq 'zpool_import') {
-   push @$cmd, 'zpool', 'import';
-   $timeout = 15 if !$timeout || $timeout < 15;
-} else {
-   push @$cmd, 'zfs', $method;
-}
-push @$cmd, @params;
-
-my $msg = '';
-my $output = sub { $msg .= "$_[0]\n" };
-
-if (PVE::RPCEnvironment->is_worker()) {
-   $timeout = 60*60 if !$timeout;
-   $timeout = 60*5 if $timeout < 60*5;
-} else {
-   $timeout = 10 if !$timeout;
-}
-
-run_command($cmd, errmsg => "zfs error", outfunc => $output, timeout => 
$timeout);
+warn get_deprecation_warning(
+   "PVE::Storage::Common::ZFS::zfs_request"
+);
 
-return $msg;
+return PVE::Storage::Common::

[pve-devel] [RFC pve-storage 34/36] common: zfs: introduce module for common ZFS helpers

[Max Carrara]

Like the LVM and ISCSI modules, this module is meant to provide
commonly used ZFS helpers, so that ZFS-related functionality may be
shared more easily among plugins. Moreover, this module also ought to
help to reduce the inter-dependency of our own ZFS plugins, while
simultaneously making ZFS utilities availble to third-party plugin
authors.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common.pm   |  4 
 src/PVE/Storage/Common/Makefile |  1 +
 src/PVE/Storage/Common/ZFS.pm   | 21 +
 3 files changed, 26 insertions(+)
 create mode 100644 src/PVE/Storage/Common/ZFS.pm

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index 8a52498..f0d5ddf 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -52,6 +52,10 @@ Utilities concerned with LVM, such as manipulating logical 
volumes.
 
 Utilities concerned with working with paths.
 
+=item C
+
+Shared ZFS utilities and command line wrappers.
+
 =back
 
 =head1 FUNCTIONS
diff --git a/src/PVE/Storage/Common/Makefile b/src/PVE/Storage/Common/Makefile
index e15a47c..d77d67e 100644
--- a/src/PVE/Storage/Common/Makefile
+++ b/src/PVE/Storage/Common/Makefile
@@ -2,6 +2,7 @@ SOURCES = \
  ISCSI.pm \
  LVM.pm \
  Path.pm \
+ ZFS.pm \
 
 
 .PHONY: install
diff --git a/src/PVE/Storage/Common/ZFS.pm b/src/PVE/Storage/Common/ZFS.pm
new file mode 100644
index 000..8b0969c
--- /dev/null
+++ b/src/PVE/Storage/Common/ZFS.pm
@@ -0,0 +1,21 @@
+package PVE::Storage::Common::ZFS;
+
+use strict;
+use warnings;
+
+use parent qw(Exporter);
+
+our @EXPORT_OK = qw();
+
+=pod
+
+=head1 NAME
+
+PVE::Storage::Common::ZFS - Shared ZFS utilities and command line wrappers
+
+=head1 FUNCTIONS
+
+=cut
+
+
+1;
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 32/36] plugin: nfs: make helper subroutines private

[Max Carrara]

The `nfs_is_mounted` and `nfs_mount` subroutines are not part of any
(official) public interface / API, nor are they used anywhere in our
code, so make them private to prevent haphazard use.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/NFSPlugin.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/PVE/Storage/NFSPlugin.pm b/src/PVE/Storage/NFSPlugin.pm
index 0d02b49..6dbf8b4 100644
--- a/src/PVE/Storage/NFSPlugin.pm
+++ b/src/PVE/Storage/NFSPlugin.pm
@@ -23,7 +23,7 @@ use base qw(PVE::Storage::Plugin);
 
 # NFS helper functions
 
-sub nfs_is_mounted {
+my sub nfs_is_mounted {
 my ($server, $export, $mountpoint, $mountdata) = @_;
 
 $server = "[$server]" if Net::IP::ip_is_ipv6($server);
@@ -38,7 +38,7 @@ sub nfs_is_mounted {
 return undef;
 }
 
-sub nfs_mount {
+my sub nfs_mount {
 my ($server, $export, $mountpoint, $options) = @_;
 
 $server = "[$server]" if Net::IP::ip_is_ipv6($server);
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 31/36] plugin: lvmthin: update definition of subroutine `activate_lv`

[Max Carrara]

.. so that it is not just an anonymous sub assigned to a reference,
but a "proper" private subroutine instead. Also update its call sites
correspondingly.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/LvmThinPlugin.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/PVE/Storage/LvmThinPlugin.pm b/src/PVE/Storage/LvmThinPlugin.pm
index 1fcafdd..c89b382 100644
--- a/src/PVE/Storage/LvmThinPlugin.pm
+++ b/src/PVE/Storage/LvmThinPlugin.pm
@@ -204,7 +204,7 @@ sub status {
 );
 }
 
-my $activate_lv = sub {
+my sub activate_lv {
 my ($vg, $lv, $cache) = @_;
 
 my $lvs = $cache->{lvs} ||= lvm_list_volumes();
@@ -225,7 +225,7 @@ sub activate_storage {
 
 $class->SUPER::activate_storage($storeid, $scfg, $cache);
 
-$activate_lv->($scfg->{vgname}, $scfg->{thinpool}, $cache);
+activate_lv($scfg->{vgname}, $scfg->{thinpool}, $cache);
 }
 
 sub activate_volume {
@@ -234,7 +234,7 @@ sub activate_volume {
 my $vg = $scfg->{vgname};
 my $lv = $snapname ? "snap_${volname}_$snapname" : $volname;
 
-$activate_lv->($vg, $lv, $cache);
+activate_lv($vg, $lv, $cache);
 }
 
 sub deactivate_volume {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 35/36] plugin: zfspool: move helper `zfs_parse_zvol_list` to common module

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common/ZFS.pm| 93 +++-
 src/PVE/Storage/ZFSPoolPlugin.pm | 45 +++-
 2 files changed, 99 insertions(+), 39 deletions(-)

diff --git a/src/PVE/Storage/Common/ZFS.pm b/src/PVE/Storage/Common/ZFS.pm
index 8b0969c..21b28d9 100644
--- a/src/PVE/Storage/Common/ZFS.pm
+++ b/src/PVE/Storage/Common/ZFS.pm
@@ -5,7 +5,9 @@ use warnings;
 
 use parent qw(Exporter);
 
-our @EXPORT_OK = qw();
+our @EXPORT_OK = qw(
+zfs_parse_zvol_list
+);
 
 =pod
 
@@ -17,5 +19,94 @@ PVE::Storage::Common::ZFS - Shared ZFS utilities and command 
line wrappers
 
 =cut
 
+=pod
+
+=head3 zfs_parse_zvol_list
+
+$zvol_list = zfs_parse_zvol_list($text, $pool)
+
+Parses ZVOLs from a C command output for a given C<$pool> and returns
+them as a list of hashes.
+
+C<$text> must contain the output of the following command, where 
CPOOLE>
+is expected to be the same as the provided C<$pool>:
+
+zfs list -o name,volsize,origin,type,refquota -t volume,filesystem -d1 -Hp 

+
+C<$pool> should refer to the actual pool / dataset that contains ZVOLs. Usually
+this is CPOOLNAMEE/vm-store>.
+
+The returned list has the following structure:
+
+(
+   {
+   name => "pool/vm-store/vm-9000-disk-0",
+   size => 68719476736,  # size in bytes
+   origin => "...",  # optional
+   format => "raw",
+   vmid => 9000,
+   },
+   {
+   name => "pool/vm-store/vm-9000-disk-1",
+   size => 68719476736,  # size in bytes
+   origin => "...",  # optional
+   format => "raw",
+   vmid => 9000,
+   },
+   {
+   name => "pool/vm-store/vm-9000-disk-2",
+   size => 137438953472,  # size in bytes
+   origin => "...",   # optional
+   format => "raw",
+   vmid => 9000,
+   },
+   ...
+)
+
+=cut
+
+sub zfs_parse_zvol_list : prototype($$) {
+my ($text, $pool) = @_;
+
+my $list = ();
+
+return $list if !$text;
+
+my @lines = split /\n/, $text;
+foreach my $line (@lines) {
+   my ($dataset, $size, $origin, $type, $refquota) = split(/\s+/, $line);
+   next if !($type eq 'volume' || $type eq 'filesystem');
+
+   my $zvol = {};
+   my @parts = split /\//, $dataset;
+   next if scalar(@parts) < 2; # we need pool/name
+   my $name = pop @parts;
+   my $parsed_pool = join('/', @parts);
+   next if $parsed_pool ne $pool;
+
+   next unless $name =~ m!^(vm|base|subvol|basevol)-(\d+)-(\S+)$!;
+   $zvol->{owner} = $2;
+
+   $zvol->{name} = $name;
+   if ($type eq 'filesystem') {
+   if ($refquota eq 'none') {
+   $zvol->{size} = 0;
+   } else {
+   $zvol->{size} = $refquota + 0;
+   }
+   $zvol->{format} = 'subvol';
+   } else {
+   $zvol->{size} = $size + 0;
+   $zvol->{format} = 'raw';
+   }
+   if ($origin !~ /^-$/) {
+   $zvol->{origin} = $origin;
+   }
+   push @$list, $zvol;
+}
+
+return $list;
+}
+
 
 1;
diff --git a/src/PVE/Storage/ZFSPoolPlugin.pm b/src/PVE/Storage/ZFSPoolPlugin.pm
index 3669fe1..fdfedca 100644
--- a/src/PVE/Storage/ZFSPoolPlugin.pm
+++ b/src/PVE/Storage/ZFSPoolPlugin.pm
@@ -9,6 +9,8 @@ use POSIX;
 
 use PVE::ProcFSTools;
 use PVE::RPCEnvironment;
+use PVE::Storage::Common qw(get_deprecation_warning);
+use PVE::Storage::Common::ZFS;
 use PVE::Storage::Plugin;
 use PVE::Tools qw(run_command);
 
@@ -60,44 +62,11 @@ sub options {
 sub zfs_parse_zvol_list {
 my ($text, $pool) = @_;
 
-my $list = ();
-
-return $list if !$text;
-
-my @lines = split /\n/, $text;
-foreach my $line (@lines) {
-   my ($dataset, $size, $origin, $type, $refquota) = split(/\s+/, $line);
-   next if !($type eq 'volume' || $type eq 'filesystem');
-
-   my $zvol = {};
-   my @parts = split /\//, $dataset;
-   next if scalar(@parts) < 2; # we need pool/name
-   my $name = pop @parts;
-   my $parsed_pool = join('/', @parts);
-   next if $parsed_pool ne $pool;
-
-   next unless $name =~ m!^(vm|base|subvol|basevol)-(\d+)-(\S+)$!;
-   $zvol->{owner} = $2;
-
-   $zvol->{name} = $name;
-   if ($type eq 'filesystem') {
-   if ($refquota eq 'none') {
-   $zvol->{size} = 0;
-   } else {
-   $zvol->{size} = $refquota + 0;
-   }
-   $zvol->{format} = 'subvol';
-   } else {
-   $zvol->{size} = $size + 0;
-   $zvol->{format} = 'raw';
-   }
-   if ($origin !~ /^-$/) {
-   $z

[pve-devel] [RFC pve-storage 29/36] plugin: iscsi: make helper subroutine `iscsi_session` private

[Max Carrara]

Because this subroutine is not really a valuable helper for the common
ISCSI module, keep it inside the plugin and make it private instead.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/ISCSIPlugin.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm
index 8a56cfe..2e55f23 100644
--- a/src/PVE/Storage/ISCSIPlugin.pm
+++ b/src/PVE/Storage/ISCSIPlugin.pm
@@ -167,7 +167,7 @@ sub list_images {
 return $res;
 }
 
-sub iscsi_session {
+my sub iscsi_session {
 my ($cache, $target) = @_;
 $cache->{iscsi_sessions} = iscsi_session_list() if 
!$cache->{iscsi_sessions};
 return $cache->{iscsi_sessions}->{$target};
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 33/36] plugin: rbd: update private sub signatures and make helpers private

[Max Carrara]

Instead of assigning anonymous subs to a reference, make them "proper"
private subs instead. Thus, signatures like

  my $foo = sub { ... };

are changed to

  my sub foo { ... }

and their call sites are updated correspondingly.

The remaining helpers are also made private, because they're not used
anywhere else in our code.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/RBDPlugin.pm | 84 ++--
 1 file changed, 42 insertions(+), 42 deletions(-)

diff --git a/src/PVE/Storage/RBDPlugin.pm b/src/PVE/Storage/RBDPlugin.pm
index f45ad3f..919b9f9 100644
--- a/src/PVE/Storage/RBDPlugin.pm
+++ b/src/PVE/Storage/RBDPlugin.pm
@@ -20,13 +20,13 @@ use PVE::Tools qw(run_command trim file_read_firstline);
 
 use base qw(PVE::Storage::Plugin);
 
-my $get_parent_image_name = sub {
+my sub get_parent_image_name {
 my ($parent) = @_;
 return undef if !$parent;
 return $parent->{image} . "@" . $parent->{snapshot};
 };
 
-my $librados_connect = sub {
+my sub librados_connect {
 my ($scfg, $storeid, $options) = @_;
 
 $options->{timeout} = 60
@@ -55,7 +55,7 @@ my sub get_rbd_dev_path {
# NOTE: the config doesn't support this currently (but it could!), hack 
for qemu-server tests
$cluster_id = $scfg->{fsid};
 } elsif ($scfg->{monhost}) {
-   my $rados = $librados_connect->($scfg, $storeid);
+   my $rados = librados_connect($scfg, $storeid);
$cluster_id = $rados->mon_command({ prefix => 'fsid', format => 'json' 
})->{fsid};
 } else {
$cluster_id = cfs_read_file('ceph.conf')->{global}->{fsid};
@@ -82,7 +82,7 @@ my sub get_rbd_dev_path {
 return $pve_path;
 }
 
-my $build_cmd = sub {
+my sub build_cmd {
 my ($binary, $scfg, $storeid, $op, @options) = @_;
 
 my $cmd_option = PVE::CephConfig::ceph_connect_option($scfg, $storeid);
@@ -110,20 +110,20 @@ my $build_cmd = sub {
 return $cmd;
 };
 
-my $rbd_cmd = sub {
+my sub rbd_cmd {
 my ($scfg, $storeid, $op, @options) = @_;
 
-return $build_cmd->('/usr/bin/rbd', $scfg, $storeid, $op, @options);
+return build_cmd('/usr/bin/rbd', $scfg, $storeid, $op, @options);
 };
 
-my $rados_cmd = sub {
+my sub rados_cmd {
 my ($scfg, $storeid, $op, @options) = @_;
 
-return $build_cmd->('/usr/bin/rados', $scfg, $storeid, $op, @options);
+return build_cmd('/usr/bin/rados', $scfg, $storeid, $op, @options);
 };
 
 # needed for volumes created using ceph jewel (or higher)
-my $krbd_feature_update = sub {
+my sub krbd_feature_update {
 my ($scfg, $storeid, $name) = @_;
 
 my (@disable, @enable);
@@ -150,7 +150,7 @@ my $krbd_feature_update = sub {
 
 if ($to_disable) {
print "disable RBD image features this kernel RBD drivers is not 
compatible with: $to_disable\n";
-   my $cmd = $rbd_cmd->($scfg, $storeid, 'feature', 'disable', $name, 
$to_disable);
+   my $cmd = rbd_cmd($scfg, $storeid, 'feature', 'disable', $name, 
$to_disable);
run_rbd_command(
$cmd,
errmsg => "could not disable krbd-incompatible image features 
'$to_disable' for rbd image: $name",
@@ -159,7 +159,7 @@ my $krbd_feature_update = sub {
 if ($to_enable) {
print "enable RBD image features this kernel RBD drivers supports: 
$to_enable\n";
eval {
-   my $cmd = $rbd_cmd->($scfg, $storeid, 'feature', 'enable', $name, 
$to_enable);
+   my $cmd = rbd_cmd($scfg, $storeid, 'feature', 'enable', $name, 
$to_enable);
run_rbd_command(
$cmd,
errmsg => "could not enable krbd-compatible image features 
'$to_enable' for rbd image: $name",
@@ -169,7 +169,7 @@ my $krbd_feature_update = sub {
 }
 };
 
-sub run_rbd_command {
+my sub run_rbd_command {
 my ($cmd, %args) = @_;
 
 my $lasterr;
@@ -198,7 +198,7 @@ sub run_rbd_command {
 return undef;
 }
 
-sub rbd_ls {
+my sub rbd_ls {
 my ($scfg, $storeid) = @_;
 
 my $pool =  $scfg->{pool} ? $scfg->{pool} : 'rbd';
@@ -207,7 +207,7 @@ sub rbd_ls {
 my $raw = '';
 my $parser = sub { $raw .= shift };
 
-my $cmd = $rbd_cmd->($scfg, $storeid, 'ls', '-l', '--format', 'json');
+my $cmd = rbd_cmd($scfg, $storeid, 'ls', '-l', '--format', 'json');
 eval {
run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc 
=> $parser);
 };
@@ -237,7 +237,7 @@ sub rbd_ls {
$list->{$pool}->{$image} = {
name => $image,
size => $el->{size},
-   parent => $get_parent_image_name->($el->{parent}),
+   par

[pve-devel] [RFC pve-storage 27/36] plugin: iscsi-direct: make helper subroutine `iscsi_ls` private

[Max Carrara]

in order to prevent it from being used in other places or third-party
plugins.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/ISCSIDirectPlugin.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/PVE/Storage/ISCSIDirectPlugin.pm 
b/src/PVE/Storage/ISCSIDirectPlugin.pm
index eb329d4..c804ab4 100644
--- a/src/PVE/Storage/ISCSIDirectPlugin.pm
+++ b/src/PVE/Storage/ISCSIDirectPlugin.pm
@@ -11,7 +11,7 @@ use PVE::JSONSchema qw(get_standard_option);
 
 use base qw(PVE::Storage::Plugin);
 
-sub iscsi_ls {
+my sub iscsi_ls {
 my ($scfg, $storeid) = @_;
 
 my $portal = $scfg->{portal};
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 30/36] plugin: lvm: update definition of subroutine `check_tags`

[Max Carrara]

So it's not just an anonymous sub assigned to a reference, but a
"proper" private subroutine instead. Also update its only call site
correspondingly.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/LVMPlugin.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index a9bc178..df041aa 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -307,7 +307,7 @@ sub free_image {
 return undef;
 }
 
-my $check_tags = sub {
+my sub check_tags {
 my ($tags) = @_;
 
 return defined($tags) && $tags =~ /(^|,)pve-vm-\d+(,|$)/;
@@ -331,7 +331,7 @@ sub list_images {
 
my $info = $dat->{$volname};
 
-   next if $scfg->{tagged_only} && !&$check_tags($info->{tags});
+   next if $scfg->{tagged_only} && !check_tags($info->{tags});
 
# Allow mirrored and RAID LVs
next if $info->{lv_type} !~ m/^[-mMrR]$/;
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 23/36] plugin: esxi: make helper subroutines private

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/ESXiPlugin.pm | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/PVE/Storage/ESXiPlugin.pm b/src/PVE/Storage/ESXiPlugin.pm
index a35693d..d4b6afc 100644
--- a/src/PVE/Storage/ESXiPlugin.pm
+++ b/src/PVE/Storage/ESXiPlugin.pm
@@ -59,12 +59,12 @@ sub options {
};
 }
 
-sub esxi_cred_file_name {
+my sub esxi_cred_file_name {
 my ($storeid) = @_;
 return "/etc/pve/priv/storage/${storeid}.pw";
 }
 
-sub esxi_delete_credentials {
+my sub esxi_delete_credentials {
 my ($storeid) = @_;
 
 if (my $cred_file = get_cred_file($storeid)) {
@@ -72,7 +72,7 @@ sub esxi_delete_credentials {
 }
 }
 
-sub esxi_set_credentials {
+my sub esxi_set_credentials {
 my ($password, $storeid) = @_;
 
 my $cred_file = esxi_cred_file_name($storeid);
@@ -83,7 +83,7 @@ sub esxi_set_credentials {
 return $cred_file;
 }
 
-sub get_cred_file {
+my sub get_cred_file {
 my ($storeid) = @_;
 
 my $cred_file = esxi_cred_file_name($storeid);
@@ -267,7 +267,7 @@ sub esxi_unmount : prototype($$$) {
 }
 
 # Split a path into (datacenter, datastore, path)
-sub split_path : prototype($) {
+my sub split_path : prototype($) {
 my ($path) = @_;
 if ($path =~ m!^([^/]+)/([^/]+)/(.+)$!) {
return ($1, $2, $3);
@@ -752,7 +752,7 @@ sub vmx_path { $_[0]->{'pve.vmx.path'} }
 sub manifest { $_[0]->{'pve.manifest'} }
 
 # (Also used for the fileName config key...)
-sub is_disk_entry : prototype($) {
+my sub is_disk_entry : prototype($) {
 my ($id) = @_;
 if ($id =~ /^(scsi|ide|sata|nvme)(\d+:\d+)(:?\.fileName)?$/) {
return ($1, $2);
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 28/36] plugin: iscsi: factor helper functions into common module

[Max Carrara]

Because the `iscsi_discovery` subroutine is used by `PVE::Storage`
directly, move it and all other helpers independent from the plugin
into the new `PVE::Storage::Common::ISCSI` module. As the name
suggests, this new module collects ISCSI-related functionalities and
utils.

Due to the `iscsi_discovery` sub being the only subroutine that was
actually used publicly, keep its original definition and let it wrap
its "new" version from the common module. Also add a deprecation
warning for any potential users.

The code of all moved subroutines stays the same, though the subs
themselves are now also documented and use prototypes.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage.pm  |   4 +-
 src/PVE/Storage/Common.pm   |   4 +
 src/PVE/Storage/Common/ISCSI.pm | 475 
 src/PVE/Storage/Common/Makefile |   1 +
 src/PVE/Storage/ISCSIPlugin.pm  | 255 +
 5 files changed, 498 insertions(+), 241 deletions(-)
 create mode 100644 src/PVE/Storage/Common/ISCSI.pm

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 57b2038..3865b44 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -24,6 +24,8 @@ use PVE::RPCEnvironment;
 use PVE::SSHInfo;
 use PVE::RESTEnvironment qw(log_warn);
 
+use PVE::Storage::Common::ISCSI;
+
 use PVE::Storage::Plugin;
 use PVE::Storage::DirPlugin;
 use PVE::Storage::LVMPlugin;
@@ -1457,7 +1459,7 @@ sub scan_iscsi {
die "unable to parse/resolve portal address '${portal_in}'\n";
 }
 
-return PVE::Storage::ISCSIPlugin::iscsi_discovery([ $portal ]);
+return PVE::Storage::Common::ISCSI::iscsi_discovery([ $portal ]);
 }
 
 sub storage_default_format {
diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index 3cc3c37..8a52498 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -40,6 +40,10 @@ be grouped in a submodule can also be found here.
 
 =over
 
+=item C
+
+Subroutines that provide ISCSI-related functionalities.
+
 =item C
 
 Utilities concerned with LVM, such as manipulating logical volumes.
diff --git a/src/PVE/Storage/Common/ISCSI.pm b/src/PVE/Storage/Common/ISCSI.pm
new file mode 100644
index 000..2eb7f65
--- /dev/null
+++ b/src/PVE/Storage/Common/ISCSI.pm
@@ -0,0 +1,475 @@
+package PVE::Storage::Common::ISCSI;
+
+use strict;
+use warnings;
+
+use File::stat;
+use IO::Dir;
+use IO::File;
+
+use PVE::Network;
+use PVE::Tools qw(
+$IPV4RE
+$IPV6RE
+dir_glob_foreach
+dir_glob_regex
+file_read_firstline
+run_command
+);
+
+use parent qw(Exporter);
+
+our @EXPORT_OK = qw(
+assert_iscsi_support
+iscsi_device_list
+iscsi_discovery
+iscsi_login
+iscsi_logout
+iscsi_portals
+iscsi_session_list
+iscsi_session_rescan
+iscsi_test_portal
+);
+
+=pod
+
+=head1 NAME
+
+PVE::Storage::Common::ISCSI - Provides helper subroutines that wrap commonly 
used ISCSI commands
+
+=head1 FUNCTIONS
+
+=cut
+
+my $ISCSIADM = '/usr/bin/iscsiadm';
+my $found_iscsi_adm_exe;
+
+=pod
+
+=head3 assert_iscsi_support
+
+$is_supported = assert_iscsi_support($noerr)
+
+Asserts whether ISCSI operations are supported by the host, raising an 
exception
+if they are not.
+
+Optionally, C<$noerr> may be set to C<1> in order to return C instead
+of raising an exception.
+
+ISCSI operations are considered to be supported if C exists
+and is executable by the current user.
+
+=cut
+
+sub assert_iscsi_support : prototype(;$) {
+my ($noerr) = @_;
+return $found_iscsi_adm_exe if $found_iscsi_adm_exe; # assume it won't be 
removed if ever found
+
+$found_iscsi_adm_exe = -x $ISCSIADM;
+
+if (!$found_iscsi_adm_exe) {
+   die "error: no iscsi support - please install open-iscsi\n" if !$noerr;
+   warn "warning: no iscsi support - please install open-iscsi\n";
+}
+return $found_iscsi_adm_exe;
+}
+
+# Example: 192.168.122.252:3260,1 
iqn.2003-01.org.linux-iscsi.proxmox-nfs.x8664:sn.00567885ba8f
+my $ISCSI_TARGET_RE = qr/^((?:$IPV4RE|\[$IPV6RE\]):\d+)\,\S+\s+(\S+)\s*$/;
+
+=pod
+
+=head3 iscsi_session_list
+
+$active_sessions = iscsi_session_list()
+
+Scans for active ISCSI sessions and returns a hash that maps each ISCSI target
+to a list of hashes describing the session.
+
+Asserts whether ISCSI is supported on the host beforehand.
+
+The returned hash has the following structure:
+
+{
+   'iqn.1998-01.example.hostname.iscsi:name1' => [
+   {
+   session_id => ...,
+   portal => ...,
+   },
+   ...
+   ],
+   'iqn.1998-01.example.hostname.iscsi:name1001' => [
+   {
+   session_id => ...,
+   portal => ...,
+   },
+   {
+   session_id => ...,
+   portal => ...,
+   },
+   ...
+   ],
+   ...
+}
+
+=cut
+
+sub iscsi_session_list : pro

[pve-devel] [RFC pve-storage 22/36] plugin: btrfs: make helper methods private

[Max Carrara]

The methods `btrfs_cmd` and `btrfs_get_subvol_id` are made private in
order to prevent them from accidentally becoming part of the plugin's
public interface in the future.

The call sites are adapted accordingly, due to the methods not being
accessible by reference via `$class` anymore. Therefore, calls like

  $class->btrfs_cmd(['some', 'command']);

are changed to

  btrfs_cmd($class, ['some', 'command']);

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/BTRFSPlugin.pm | 48 +-
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/PVE/Storage/BTRFSPlugin.pm b/src/PVE/Storage/BTRFSPlugin.pm
index daff8a4..24694a6 100644
--- a/src/PVE/Storage/BTRFSPlugin.pm
+++ b/src/PVE/Storage/BTRFSPlugin.pm
@@ -234,7 +234,7 @@ sub filesystem_path {
 return wantarray ? ($path, $vmid, $vtype) : $path;
 }
 
-sub btrfs_cmd {
+my sub btrfs_cmd {
 my ($class, $cmd, $outfunc) = @_;
 
 my $msg = '';
@@ -252,9 +252,9 @@ sub btrfs_cmd {
 return $msg;
 }
 
-sub btrfs_get_subvol_id {
+my sub btrfs_get_subvol_id {
 my ($class, $path) = @_;
-my $info = $class->btrfs_cmd(['subvolume', 'show', '--', $path]);
+my $info = btrfs_cmd($class, ['subvolume', 'show', '--', $path]);
 if ($info !~ /^\s*(?:Object|Subvolume) ID:\s*(\d+)$/m) {
die "failed to get btrfs subvolume ID from: $info\n";
 }
@@ -299,7 +299,7 @@ sub create_base {
 
 rename($subvol, $newsubvol)
|| die "rename '$subvol' to '$newsubvol' failed - $!\n";
-eval { $class->btrfs_cmd(['property', 'set', $newsubvol, 'ro', 'true']) };
+eval { btrfs_cmd($class, ['property', 'set', $newsubvol, 'ro', 'true']) };
 warn $@ if $@;
 
 return $newvolname;
@@ -336,7 +336,7 @@ sub clone_image {
$newsubvol = raw_file_to_subvol($newsubvol);
 }
 
-$class->btrfs_cmd(['subvolume', 'snapshot', '--', $subvol, $newsubvol]);
+btrfs_cmd($class, ['subvolume', 'snapshot', '--', $subvol, $newsubvol]);
 
 return $newvolname;
 }
@@ -380,7 +380,7 @@ sub alloc_image {
die "btrfs quotas are currently not supported, use an unsized subvolume 
or a raw file\n";
 }
 
-$class->btrfs_cmd(['subvolume', 'create', '--', $subvol]);
+btrfs_cmd($class, ['subvolume', 'create', '--', $subvol]);
 
 eval {
if ($fmt eq 'subvol') {
@@ -391,9 +391,9 @@ sub alloc_image {
# eval {
# # This call should happen at storage creation instead and 
therefore governed by a
# # configuration option!
-   # # $class->btrfs_cmd(['quota', 'enable', $subvol]);
-   # my $id = $class->btrfs_get_subvol_id($subvol);
-   # $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", 
$subvol]);
+   # # btrfs_cmd($class, ['quota', 'enable', $subvol]);
+   # my $id = btrfs_get_subvol_id($class, $subvol);
+   # btrfs_cmd($class, ['qgroup', 'limit', "${size}k", "0/$id", 
$subvol]);
# };
} elsif ($fmt eq 'raw') {
sysopen my $fh, $path, O_WRONLY | O_CREAT | O_EXCL
@@ -408,7 +408,7 @@ sub alloc_image {
 };
 
 if (my $err = $@) {
-   eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $subvol]); };
+   eval { btrfs_cmd($class, ['subvolume', 'delete', '--', $subvol]); };
warn $@ if $@;
die $err;
 }
@@ -466,7 +466,7 @@ sub free_image {
push @snapshot_vols, "$dir/$volume";
 });
 
-$class->btrfs_cmd(['subvolume', 'delete', '--', @snapshot_vols, $subvol]);
+btrfs_cmd($class, ['subvolume', 'delete', '--', @snapshot_vols, $subvol]);
 # try to cleanup directory to not clutter storage with empty $vmid dirs if
 # all images from a guest got deleted
 rmdir($dir);
@@ -477,10 +477,10 @@ sub free_image {
 # Currently not used because quotas clash with send/recv.
 # my sub btrfs_subvol_quota {
 # my ($class, $path) = @_;
-# my $id = '0/' . $class->btrfs_get_subvol_id($path);
+# my $id = '0/' . btrfs_get_subvol_id($class, $path);
 # my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/;
 # my ($used, $size);
-# $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub {
+# btrfs_cmd($class, ['qgroup', 'show'

[pve-devel] [RFC pve-storage 26/36] plugin: gluster: make helper subroutines private

[Max Carrara]

.. and also use a regular sub definition for `get_active_server` in
order to avoid the sub prefix deref syntax when calling it.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/GlusterfsPlugin.pm | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/PVE/Storage/GlusterfsPlugin.pm 
b/src/PVE/Storage/GlusterfsPlugin.pm
index 2b7f9e1..634a090 100644
--- a/src/PVE/Storage/GlusterfsPlugin.pm
+++ b/src/PVE/Storage/GlusterfsPlugin.pm
@@ -16,7 +16,7 @@ use base qw(PVE::Storage::Plugin);
 
 my $server_test_results = {};
 
-my $get_active_server = sub {
+my sub get_active_server {
 my ($scfg, $return_default_if_offline) = @_;
 
 my $defaultserver = $scfg->{server} ? $scfg->{server} : 'localhost';
@@ -66,7 +66,7 @@ my $get_active_server = sub {
 return undef;
 };
 
-sub glusterfs_is_mounted {
+my sub glusterfs_is_mounted {
 my ($volume, $mountpoint, $mountdata) = @_;
 
 $mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata;
@@ -79,7 +79,7 @@ sub glusterfs_is_mounted {
 return undef;
 }
 
-sub glusterfs_mount {
+my sub glusterfs_mount {
 my ($server, $volume, $mountpoint) = @_;
 
 my $source = "$server:$volume";
@@ -179,7 +179,7 @@ sub path {
 my $path = undef;
 if ($vtype eq 'images') {
 
-   my $server = &$get_active_server($scfg, 1);
+   my $server = get_active_server($scfg, 1);
my $glustervolume = $scfg->{volume};
my $transport = $scfg->{transport};
my $protocol = "gluster";
@@ -227,7 +227,7 @@ sub clone_image {
 
 die "disk image '$path' already exists\n" if -e $path;
 
-my $server = &$get_active_server($scfg, 1);
+my $server = get_active_server($scfg, 1);
 my $glustervolume = $scfg->{volume};
 my $volumepath = "gluster://$server/$glustervolume/images/$vmid/$name";
 
@@ -258,7 +258,7 @@ sub alloc_image {
 
 die "disk image '$path' already exists\n" if -e $path;
 
-my $server = &$get_active_server($scfg, 1);
+my $server = get_active_server($scfg, 1);
 my $glustervolume = $scfg->{volume};
 my $volumepath = "gluster://$server/$glustervolume/images/$vmid/$name";
 
@@ -309,7 +309,7 @@ sub activate_storage {
die "unable to activate storage '$storeid' - " .
"directory '$path' does not exist\n" if ! -d $path;
 
-   my $server = &$get_active_server($scfg, 1);
+   my $server = get_active_server($scfg, 1);
 
glusterfs_mount($server, $volume, $path);
 }
@@ -347,7 +347,7 @@ sub deactivate_volume {
 sub check_connection {
 my ($class, $storeid, $scfg, $cache) = @_;
 
-my $server = &$get_active_server($scfg);
+my $server = get_active_server($scfg);
 
 return defined($server) ? 1 : 0;
 }
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 25/36] plugin: esxi: make helper methods private

[Max Carrara]

The methods `get_manifest`, `esxi_mount` and `esxi_unmount` are made
private in order to prevent them from accidentally becoming part of
the plugin's public interface in the future.

Similar to the BTRFS changes, adapt the call sites accordingly. This
means that calls like

  my $manifest = $class->get_manifest($storeid, $scfg, 0);

are changed to

  my $manifest = get_manifest($class, $storeid, $scfg, 0);

because the methods cannot be accessed via the `$class` reference
anymore.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/ESXiPlugin.pm | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/PVE/Storage/ESXiPlugin.pm b/src/PVE/Storage/ESXiPlugin.pm
index 2bab7f8..355a33c 100644
--- a/src/PVE/Storage/ESXiPlugin.pm
+++ b/src/PVE/Storage/ESXiPlugin.pm
@@ -120,7 +120,7 @@ my sub is_old : prototype($) {
 return !defined($mtime) || ($mtime + 30) < CORE::time();
 }
 
-sub get_manifest : prototype($$$;$) {
+my sub get_manifest : prototype($$$;$) {
 my ($class, $storeid, $scfg, $force_query) = @_;
 
 my $rundir = run_path($storeid);
@@ -177,12 +177,12 @@ my sub is_mounted : prototype($) {
 return PVE::Systemd::is_unit_active($scope_name_base . '.scope');
 }
 
-sub esxi_mount : prototype($$$;$) {
+my sub esxi_mount : prototype($$$;$) {
 my ($class, $storeid, $scfg, $force_requery) = @_;
 
 return if !$force_requery && is_mounted($storeid);
 
-$class->get_manifest($storeid, $scfg, $force_requery);
+get_manifest($class, $storeid, $scfg, $force_requery);
 
 my $rundir = run_path($storeid);
 my $manifest_file = "$rundir/manifest.json";
@@ -253,7 +253,7 @@ sub esxi_mount : prototype($$$;$) {
 }
 }
 
-sub esxi_unmount : prototype($$$) {
+my sub esxi_unmount : prototype($$$) {
 my ($class, $storeid, $scfg) = @_;
 
 my $scope_name_base = scope_name_base($storeid);
@@ -287,7 +287,7 @@ sub get_import_metadata : prototype($) {
die "storage '$storeid' is not activated\n";
 }
 
-my $manifest = $class->get_manifest($storeid, $scfg, 0);
+my $manifest = get_manifest($class, $storeid, $scfg, 0);
 my $contents = file_get_contents($vmx_path);
 my $vmx = PVE::Storage::ESXiPlugin::VMX->parse(
$storeid,
@@ -351,13 +351,13 @@ sub on_delete_hook {
 sub activate_storage {
 my ($class, $storeid, $scfg, $cache) = @_;
 
-$class->esxi_mount($storeid, $scfg, 0);
+esxi_mount($class, $storeid, $scfg, 0);
 }
 
 sub deactivate_storage {
 my ($class, $storeid, $scfg, $cache) = @_;
 
-$class->esxi_unmount($storeid, $scfg);
+esxi_unmount($class, $storeid, $scfg);
 
 my $rundir = run_path($storeid);
 remove_tree($rundir); # best-effort, ignore errors for now
@@ -418,7 +418,7 @@ sub list_volumes {
 
 return if !grep { $_ eq 'import' } @$content_types;
 
-my $data = $class->get_manifest($storeid, $scfg, 0);
+my $data = get_manifest($class, $storeid, $scfg, 0);
 
 my $res = [];
 for my $dc_name (keys $data->%*) {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 24/36] plugin: esxi: remove unused helper subroutine `query_vmdk_size`

[Max Carrara]

This subroutine does not appear to be used in any of our Perl code, so
remove it.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/ESXiPlugin.pm | 18 --
 1 file changed, 18 deletions(-)

diff --git a/src/PVE/Storage/ESXiPlugin.pm b/src/PVE/Storage/ESXiPlugin.pm
index d4b6afc..2bab7f8 100644
--- a/src/PVE/Storage/ESXiPlugin.pm
+++ b/src/PVE/Storage/ESXiPlugin.pm
@@ -299,24 +299,6 @@ sub get_import_metadata : prototype($) {
 return $vmx->get_create_args();
 }
 
-# Returns a size in bytes, this is a helper for already-mounted files.
-sub query_vmdk_size : prototype($;$) {
-my ($filename, $timeout) = @_;
-
-my $json = eval {
-   my $json = '';
-   run_command(['/usr/bin/qemu-img', 'info', '--output=json', $filename],
-   timeout => $timeout,
-   outfunc => sub { $json .= $_[0]; },
-   errfunc => sub { warn "$_[0]\n"; }
-   );
-   from_json($json)
-};
-warn $@ if $@;
-
-return int($json->{'virtual-size'});
-}
-
 #
 # Storage API implementation
 #
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 19/36] plugin: cifs: remove dependency on directory plugin

[Max Carrara]

As with the BTRFS plugin, the dependency on the directory plugin is
removed by using the helpers from the `Common` module instead of
calling the dir plugin's methods.

Deprecation warnings for the `get_volume_notes` and
`update_volume_notes` methods are also added.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/CIFSPlugin.pm | 41 ++-
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/PVE/Storage/CIFSPlugin.pm b/src/PVE/Storage/CIFSPlugin.pm
index 18dc017..ae51be8 100644
--- a/src/PVE/Storage/CIFSPlugin.pm
+++ b/src/PVE/Storage/CIFSPlugin.pm
@@ -6,6 +6,13 @@ use Net::IP;
 use PVE::Tools qw(run_command);
 use PVE::ProcFSTools;
 use File::Path;
+use PVE::Storage::Common qw(
+get_deprecation_warning
+storage_dir_get_volume_notes
+storage_dir_update_volume_notes
+storage_dir_get_volume_attribute
+storage_dir_update_volume_attribute
+);
 use PVE::Storage::Plugin;
 use PVE::JSONSchema qw(get_standard_option);
 
@@ -295,23 +302,45 @@ sub check_connection {
 # FIXME remove on the next APIAGE reset.
 # Deprecated, use get_volume_attribute instead.
 sub get_volume_notes {
-my $class = shift;
-PVE::Storage::DirPlugin::get_volume_notes($class, @_);
+warn get_deprecation_warning(
+   "PVE::Storage::Common::storage_dir_get_volume_notes"
+);
+
+my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+return storage_dir_get_volume_notes(
+   $class, $scfg, $storeid, $volname, $timeout
+);
 }
 
 # FIXME remove on the next APIAGE reset.
 # Deprecated, use update_volume_attribute instead.
 sub update_volume_notes {
-my $class = shift;
-PVE::Storage::DirPlugin::update_volume_notes($class, @_);
+warn get_deprecation_warning(
+   "PVE::Storage::Common::storage_dir_update_volume_notes"
+);
+
+my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_;
+
+return storage_dir_update_volume_notes(
+   $class, $scfg, $storeid, $volname, $notes, $timeout
+);
 }
 
 sub get_volume_attribute {
-return PVE::Storage::DirPlugin::get_volume_attribute(@_);
+my ($class, $scfg, $storeid, $volname, $attribute) = @_;
+
+return storage_dir_get_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute
+);
 }
 
 sub update_volume_attribute {
-return PVE::Storage::DirPlugin::update_volume_attribute(@_);
+my ($class, $scfg, $storeid, $volname, $attribute, $value) = @_;
+
+return storage_dir_update_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute, $value
+);
 }
 
 1;
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 18/36] plugin: btrfs: remove dependency on directory plugin

[Max Carrara]

This commit removes the BTRFS plugin's dependency on the directory
plugin. This is done by replacing calls to the dir plugin's methods
with the corresponding helper subroutines from the `Common` module
instead.

Furthermore, the methods `check_config` and `status` both get their
own implementations that behave essentially the same as the ones from
the dir plugin.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/BTRFSPlugin.pm | 49 +++---
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/src/PVE/Storage/BTRFSPlugin.pm b/src/PVE/Storage/BTRFSPlugin.pm
index 10fd441..daff8a4 100644
--- a/src/PVE/Storage/BTRFSPlugin.pm
+++ b/src/PVE/Storage/BTRFSPlugin.pm
@@ -13,9 +13,15 @@ use POSIX qw(EEXIST);
 
 use PVE::Tools qw(run_command dir_glob_foreach);
 
-use PVE::Storage::Common qw(storage_parse_is_mountpoint);
-use PVE::Storage::Common::Path qw(path_is_mounted);
-use PVE::Storage::DirPlugin;
+use PVE::Storage::Common qw(
+storage_parse_is_mountpoint
+storage_dir_get_volume_attribute
+storage_dir_update_volume_attribute
+);
+use PVE::Storage::Common::Path qw(
+path_is_mounted
+path_is_storage_dir
+);
 
 use constant {
 BTRFS_FIRST_FREE_OBJECTID => 256,
@@ -92,10 +98,16 @@ sub options {
 #   -> `images/VMID/vm-VMID-disk-ID/disk.raw`
 #   where the `vm-VMID-disk-ID/` subdirectory is a btrfs subvolume
 
-# Reuse `DirPlugin`'s `check_config`. This simply checks for invalid paths.
 sub check_config {
 my ($self, $sectionId, $config, $create, $skipSchemaCheck) = @_;
-return PVE::Storage::DirPlugin::check_config($self, $sectionId, $config, 
$create, $skipSchemaCheck);
+my $opts = PVE::SectionConfig::check_config($self, $sectionId, $config, 
$create, $skipSchemaCheck);
+return $opts if !$create;
+
+if (!path_is_storage_dir($opts->{path})) {
+   die "illegal path for directory of BTRFS storage: $opts->{path}\n";
+}
+
+return $opts;
 }
 
 my sub getfsmagic($) {
@@ -137,23 +149,30 @@ sub activate_storage {
 
 sub status {
 my ($class, $storeid, $scfg, $cache) = @_;
-return PVE::Storage::DirPlugin::status($class, $storeid, $scfg, $cache);
+
+if (defined(my $mp = storage_parse_is_mountpoint($scfg))) {
+   $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
+   if !$cache->{mountdata};
+
+   return undef if !path_is_mounted($mp, $cache->{mountdata});
+}
+
+return $class->SUPER::status($storeid, $scfg, $cache);
 }
 
 sub get_volume_attribute {
 my ($class, $scfg, $storeid, $volname, $attribute) = @_;
-return PVE::Storage::DirPlugin::get_volume_attribute($class, $scfg, 
$storeid, $volname, $attribute);
+
+return storage_dir_get_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute
+);
 }
 
 sub update_volume_attribute {
 my ($class, $scfg, $storeid, $volname, $attribute, $value) = @_;
-return PVE::Storage::DirPlugin::update_volume_attribute(
-   $class,
-   $scfg,
-   $storeid,
-   $volname,
-   $attribute,
-   $value,
+
+return storage_dir_update_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute, $value
 );
 }
 
@@ -295,7 +314,7 @@ sub clone_image {
 # If we're not working with a 'raw' file, which is the only thing that's 
"different" for btrfs,
 # or a subvolume, we forward to the DirPlugin
 if ($format ne 'raw' && $format ne 'subvol') {
-   return PVE::Storage::DirPlugin::clone_image(@_);
+   return $class->SUPER::clone_image(@_);
 }
 
 my $imagedir = $class->get_subdir($scfg, 'images');
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 17/36] plugin: dir: factor path validity check into helper methods

[Max Carrara]

Whether a directory-based storage's path is valid or not should not be
solely decided within a method of the directoy plugin, but should
instead be available to other plugins, possibly third-party plugins,
as well.

Therefore, factor that check into three different helper functions in
`Common::Path`, so that they may be re-used by other plugins in the
future. Document the helper functions as well.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common/Path.pm | 73 ++
 src/PVE/Storage/DirPlugin.pm   |  4 +-
 2 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/src/PVE/Storage/Common/Path.pm b/src/PVE/Storage/Common/Path.pm
index 7535dda..b9072bf 100644
--- a/src/PVE/Storage/Common/Path.pm
+++ b/src/PVE/Storage/Common/Path.pm
@@ -11,6 +11,9 @@ use parent qw(Exporter);
 
 our @EXPORT_OK = qw(
 path_is_mounted
+path_is_absolute
+path_contains_valid_chars
+path_is_storage_dir
 );
 
 =pod
@@ -48,4 +51,74 @@ sub path_is_mounted {
 return undef;
 }
 
+=pod
+
+=head3 path_is_absolute
+
+$result = path_is_absolute($path)
+
+Checks whether a C<$path> is absolute.
+
+Will return C if C<$path> is C, or a boolean otherwise.
+
+=cut
+
+sub path_is_absolute : prototype($) {
+my ($path) = @_;
+
+return undef if !defined($path);
+
+return ($path =~ m|^/|) + 0; # convert to number
+}
+
+=pod
+
+=head3 path_contains_valid_chars
+
+$result = path_contains_valid_chars($path)
+
+Checks whether a C<$path> contains only valid characters.
+
+"Valid" in this context means "the characters that we allow". While 
Unix/Linux/POSIX
+paths Lhttps://lwn.net/Articles/71472/>,
+I almost any sequence of bytes can lead to many unforeseen issues.
+See Lhttps://dwheeler.com/essays/fixing-unix-linux-filenames.html> for 
more
+information.
+
+Valid characters are the letters C as well as their uppercase variants
+C, the numbers C<0-9> and the symbols C<->, C, C<_>, C<.> and C<@>.
+
+Will return C if C<$path> is C, or a boolean otherwise.
+
+=cut
+
+sub path_contains_valid_chars : prototype($) {
+my ($path) = @_;
+
+return undef if !defined($path);
+
+return ($path =~ m|[-/a-zA-Z0-9_.@]+|) + 0; # convert to number
+}
+
+
+=pod
+
+=head3 path_is_storage_dir
+
+$result = path_is_storage_dir($path)
+
+Shorthand for C> C<&&> C>.
+
+Will return C if C<$path> is C, or a boolean otherwise.
+
+=cut
+
+sub path_is_storage_dir : prototype($) {
+my ($path) = @_;
+
+return undef if !defined($path);
+
+return path_is_absolute($path) && path_contains_valid_chars($path);
+}
+
 1;
diff --git a/src/PVE/Storage/DirPlugin.pm b/src/PVE/Storage/DirPlugin.pm
index f6e1d73..4be39f9 100644
--- a/src/PVE/Storage/DirPlugin.pm
+++ b/src/PVE/Storage/DirPlugin.pm
@@ -187,9 +187,11 @@ sub check_config {
 my ($self, $sectionId, $config, $create, $skipSchemaCheck) = @_;
 my $opts = PVE::SectionConfig::check_config($self, $sectionId, $config, 
$create, $skipSchemaCheck);
 return $opts if !$create;
-if ($opts->{path} !~ m|^/[-/a-zA-Z0-9_.@]+$|) {
+
+if (!PVE::Storage::Common::Path::path_is_storage_dir($opts->{path})) {
die "illegal path for directory storage: $opts->{path}\n";
 }
+
 return $opts;
 }
 
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 20/36] plugin: cephfs: remove dependency on directory plugin

[Max Carrara]

As with the BTRFS plugin, the dependency on the directory plugin is
removed by using the helpers from the `Common` module instead of
calling the dir plugin's methods.

Deprecation warnings for the `get_volume_notes` and
`update_volume_notes` methods are also added.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/CephFSPlugin.pm | 41 -
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/PVE/Storage/CephFSPlugin.pm b/src/PVE/Storage/CephFSPlugin.pm
index 98d1ba6..3be7259 100644
--- a/src/PVE/Storage/CephFSPlugin.pm
+++ b/src/PVE/Storage/CephFSPlugin.pm
@@ -10,6 +10,13 @@ use File::Path;
 use PVE::CephConfig;
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::ProcFSTools;
+use PVE::Storage::Common qw(
+get_deprecation_warning
+storage_dir_get_volume_notes
+storage_dir_update_volume_notes
+storage_dir_get_volume_attribute
+storage_dir_update_volume_attribute
+);
 use PVE::Storage::Plugin;
 use PVE::Systemd;
 use PVE::Tools qw(run_command file_set_contents);
@@ -242,23 +249,45 @@ sub deactivate_storage {
 # FIXME remove on the next APIAGE reset.
 # Deprecated, use get_volume_attribute instead.
 sub get_volume_notes {
-my $class = shift;
-PVE::Storage::DirPlugin::get_volume_notes($class, @_);
+warn get_deprecation_warning(
+   "PVE::Storage::Common::storage_dir_get_volume_notes"
+);
+
+my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+return storage_dir_get_volume_notes(
+   $class, $scfg, $storeid, $volname, $timeout
+);
 }
 
 # FIXME remove on the next APIAGE reset.
 # Deprecated, use update_volume_attribute instead.
 sub update_volume_notes {
-my $class = shift;
-PVE::Storage::DirPlugin::update_volume_notes($class, @_);
+warn get_deprecation_warning(
+   "PVE::Storage::Common::storage_dir_update_volume_notes"
+);
+
+my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_;
+
+return storage_dir_update_volume_notes(
+   $class, $scfg, $storeid, $volname, $notes, $timeout
+);
 }
 
 sub get_volume_attribute {
-return PVE::Storage::DirPlugin::get_volume_attribute(@_);
+my ($class, $scfg, $storeid, $volname, $attribute) = @_;
+
+return storage_dir_get_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute
+);
 }
 
 sub update_volume_attribute {
-return PVE::Storage::DirPlugin::update_volume_attribute(@_);
+my ($class, $scfg, $storeid, $volname, $attribute, $value) = @_;
+
+return storage_dir_update_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute, $value
+);
 }
 
 1;
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 21/36] plugin: nfs: remove dependency on directory plugin

[Max Carrara]

As with the BTRFS plugin, the dependency on the directory plugin is
removed by using the helpers from the `Common` module instead of
calling the dir plugin's methods.

Deprecation warnings for the `get_volume_notes` and
`update_volume_notes` methods are also added.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/NFSPlugin.pm | 41 ++--
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/PVE/Storage/NFSPlugin.pm b/src/PVE/Storage/NFSPlugin.pm
index f2e4c0d..0d02b49 100644
--- a/src/PVE/Storage/NFSPlugin.pm
+++ b/src/PVE/Storage/NFSPlugin.pm
@@ -9,6 +9,13 @@ use File::Path;
 use PVE::Network;
 use PVE::Tools qw(run_command);
 use PVE::ProcFSTools;
+use PVE::Storage::Common qw(
+get_deprecation_warning
+storage_dir_get_volume_notes
+storage_dir_update_volume_notes
+storage_dir_get_volume_attribute
+storage_dir_update_volume_attribute
+);
 use PVE::Storage::Plugin;
 use PVE::JSONSchema qw(get_standard_option);
 
@@ -204,23 +211,45 @@ sub check_connection {
 # FIXME remove on the next APIAGE reset.
 # Deprecated, use get_volume_attribute instead.
 sub get_volume_notes {
-my $class = shift;
-PVE::Storage::DirPlugin::get_volume_notes($class, @_);
+warn get_deprecation_warning(
+   "PVE::Storage::Common::storage_dir_get_volume_notes"
+);
+
+my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+return storage_dir_get_volume_notes(
+   $class, $scfg, $storeid, $volname, $timeout
+);
 }
 
 # FIXME remove on the next APIAGE reset.
 # Deprecated, use update_volume_attribute instead.
 sub update_volume_notes {
-my $class = shift;
-PVE::Storage::DirPlugin::update_volume_notes($class, @_);
+warn get_deprecation_warning(
+   "PVE::Storage::Common::storage_dir_update_volume_notes"
+);
+
+my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_;
+
+return storage_dir_update_volume_notes(
+   $class, $scfg, $storeid, $volname, $notes, $timeout
+);
 }
 
 sub get_volume_attribute {
-return PVE::Storage::DirPlugin::get_volume_attribute(@_);
+my ($class, $scfg, $storeid, $volname, $attribute) = @_;
+
+return storage_dir_get_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute
+);
 }
 
 sub update_volume_attribute {
-return PVE::Storage::DirPlugin::update_volume_attribute(@_);
+my ($class, $scfg, $storeid, $volname, $attribute, $value) = @_;
+
+return storage_dir_update_volume_attribute(
+   $class, $scfg, $storeid, $volname, $attribute, $value
+);
 }
 
 1;
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 13/36] common: lvm: update code style

[Max Carrara]

in order to be more in line with our style guide [0]. This also
renames some parameters for clarity.

[0]: https://pve.proxmox.com/wiki/Perl_Style_Guide

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common/LVM.pm | 80 +--
 1 file changed, 57 insertions(+), 23 deletions(-)

diff --git a/src/PVE/Storage/Common/LVM.pm b/src/PVE/Storage/Common/LVM.pm
index d4fa95f..2e010b6 100644
--- a/src/PVE/Storage/Common/LVM.pm
+++ b/src/PVE/Storage/Common/LVM.pm
@@ -167,7 +167,12 @@ sub lvm_create_volume_group : prototype($$$) {
 $cmd = ['/sbin/vgcreate', $vgname, $device];
 # push @$cmd, '-c', 'y' if $shared; # we do not use this yet
 
-run_command($cmd, errmsg => "vgcreate $vgname $device error", errfunc => 
$ignore_no_medium_warnings, outfunc => $ignore_no_medium_warnings);
+run_command(
+   $cmd,
+   errmsg => "vgcreate $vgname $device error",
+   errfunc => $ignore_no_medium_warnings,
+   outfunc => $ignore_no_medium_warnings
+);
 }
 
 =pod
@@ -248,10 +253,10 @@ sub lvm_vgs : prototype(;$) {
 my $cols = [qw(vg_name vg_size vg_free lv_count)];
 
 if ($includepvs) {
-   push @$cols, qw(pv_name pv_size pv_free);
+   push $cols->@*, qw(pv_name pv_size pv_free);
 }
 
-push @$cmd, join(',', @$cols);
+push $cmd->@*, join(',', @$cols);
 
 my $vgs = {};
 eval {
@@ -268,7 +273,7 @@ sub lvm_vgs : prototype(;$) {
};
 
if (defined($pvname) && defined($pvsize) && defined($pvfree)) {
-   push @{$vgs->{$name}->{pvs}}, {
+   push $vgs->{$name}->{pvs}->@*, {
name => $pvname,
size => int($pvsize),
free => int($pvfree),
@@ -333,7 +338,20 @@ The returned hash has the following structure:
 sub lvm_list_volumes : prototype(;$) {
 my ($vgname) = @_;
 
-my $option_list = 
'vg_name,lv_name,lv_size,lv_attr,pool_lv,data_percent,metadata_percent,snap_percent,uuid,tags,metadata_size,time';
+my $option_list = join(',', qw(
+   vg_name
+   lv_name
+   lv_size
+   lv_attr
+   pool_lv
+   data_percent
+   metadata_percent
+   snap_percent
+   uuid
+   tags
+   metadata_size
+   time
+));
 
 my $cmd = [
'/sbin/lvs', '--separator', ':', '--noheadings', '--units', 'b',
@@ -342,7 +360,7 @@ sub lvm_list_volumes : prototype(;$) {
'--options', $option_list,
 ];
 
-push @$cmd, $vgname if $vgname;
+push $cmd->@*, $vgname if $vgname;
 
 my $lvs = {};
 run_command($cmd, outfunc => sub {
@@ -350,7 +368,21 @@ sub lvm_list_volumes : prototype(;$) {
 
$line = trim($line);
 
-   my ($vg_name, $lv_name, $lv_size, $lv_attr, $pool_lv, $data_percent, 
$meta_percent, $snap_percent, $uuid, $tags, $meta_size, $ctime) = split(':', 
$line);
+   my (
+   $vg_name,
+   $lv_name,
+   $lv_size,
+   $lv_attr,
+   $pool_lv,
+   $data_percent,
+   $meta_percent,
+   $snap_percent,
+   $uuid,
+   $tags,
+   $meta_size,
+   $ctime
+   ) = split(':', $line);
+
return if !$vg_name;
return if !$lv_name;
 
@@ -370,8 +402,8 @@ sub lvm_list_volumes : prototype(;$) {
$meta_percent ||= 0;
$snap_percent ||= 0;
$d->{metadata_size} = int($meta_size);
-   $d->{metadata_used} = int(($meta_percent * $meta_size)/100);
-   $d->{used} = int(($data_percent * $lv_size)/100);
+   $d->{metadata_used} = int(($meta_percent * $meta_size) / 100);
+   $d->{used} = int(($data_percent * $lv_size) / 100);
}
$lvs->{$vg_name}->{$lv_name} = $d;
 },
@@ -411,18 +443,20 @@ See also: L|/lvm_list_volumes>
 =cut
 
 sub lvm_list_thinpools : prototype(;$) {
-my ($vg) = @_;
+my ($vgname) = @_;
 
-my $lvs = lvm_list_volumes($vg);
+my $lvs = lvm_list_volumes($vgname);
 my $thinpools = [];
 
-foreach my $vg (keys %$lvs) {
-   foreach my $lvname (keys %{$lvs->{$vg}}) {
-   next if $lvs->{$vg}->{$lvname}->{lv_type} ne 't';
-   my $lv = $lvs->{$vg}->{$lvname};
+for my $vgname (keys $lvs->%*) {
+   for my $lvname (keys $lvs->{$vgname}->%*) {
+   next if $lvs->{$vgname}->{$lvname}->{lv_type} ne 't';
+
+   my $lv = $lvs->{$vgname}->{$lvname};
$lv->{lv} = $lvname;
-   $lv->{vg} = $vg;
-   push @$thinpools, $lv;
+   $lv->{vg} = $vgname;
+
+   push $thinpools->@*, $lv;
}
 }
 
@@ -443,19 +477,19 @@ C<"50g"> (50 gibibytes),

[pve-devel] [RFC pve-storage 12/36] plugin: lvmthin: move helper that lists thinpools to common LVM module

[Max Carrara]

and deprecate the original helper, emitting a warning if it's used.
Also document the moved subroutine.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common/LVM.pm| 49 
 src/PVE/Storage/LvmThinPlugin.pm | 30 +++
 2 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/src/PVE/Storage/Common/LVM.pm b/src/PVE/Storage/Common/LVM.pm
index e0e3263..d4fa95f 100644
--- a/src/PVE/Storage/Common/LVM.pm
+++ b/src/PVE/Storage/Common/LVM.pm
@@ -16,6 +16,7 @@ our @EXPORT_OK = qw(
 lvm_destroy_volume_group
 lvm_vgs
 lvm_list_volumes
+lvm_list_thinpools
 lvm_lvcreate
 lvm_lvrename
 );
@@ -380,6 +381,54 @@ sub lvm_list_volumes : prototype(;$) {
 return $lvs;
 }
 
+=pod
+
+=head3 lvm_list_thinpools
+
+$thinpools = lvm_list_thinpools()
+$thinpools = lvm_list_thinpools($vgname)
+
+Returns a list of hashes containing all I (I with
+C B>). May optionally be limited to a single I by
+providing its name C<$vgname>.
+
+The returned list has the following structure:
+
+[
+   {
+   lv => 'lv-name-00',
+   vg => 'vg-name-00',
+   },
+   {
+   lv => 'lv-name-01',
+   vg => 'vg-name-00',
+   },
+   ...
+]
+
+See also: L|/lvm_list_volumes>
+
+=cut
+
+sub lvm_list_thinpools : prototype(;$) {
+my ($vg) = @_;
+
+my $lvs = lvm_list_volumes($vg);
+my $thinpools = [];
+
+foreach my $vg (keys %$lvs) {
+   foreach my $lvname (keys %{$lvs->{$vg}}) {
+   next if $lvs->{$vg}->{$lvname}->{lv_type} ne 't';
+   my $lv = $lvs->{$vg}->{$lvname};
+   $lv->{lv} = $lvname;
+   $lv->{vg} = $vg;
+   push @$thinpools, $lv;
+   }
+}
+
+return $thinpools;
+}
+
 =head3 lvm_lvcreate
 
 lvm_lvcreate($vgname, $name, $size, $tags)
diff --git a/src/PVE/Storage/LvmThinPlugin.pm b/src/PVE/Storage/LvmThinPlugin.pm
index 480cc78..1fcafdd 100644
--- a/src/PVE/Storage/LvmThinPlugin.pm
+++ b/src/PVE/Storage/LvmThinPlugin.pm
@@ -6,6 +6,7 @@ use warnings;
 use IO::File;
 
 use PVE::Tools qw(run_command trim);
+use PVE::Storage::Common qw(get_deprecation_warning);
 use PVE::Storage::Common::LVM qw(lvm_vgs lvm_list_volumes);
 use PVE::Storage::Plugin;
 use PVE::Storage::LVMPlugin;
@@ -25,6 +26,16 @@ use PVE::JSONSchema qw(get_standard_option);
 
 use base qw(PVE::Storage::LVMPlugin);
 
+sub list_thinpools {
+warn get_deprecation_warning(
+   "PVE::Storage::Common::LVM::lvm_list_thinpools"
+);
+
+my ($vgname) = @_;
+
+return PVE::Storage::Common::LVM::lvm_list_thinpools($vgname);
+}
+
 sub type {
 return 'lvmthin';
 }
@@ -174,25 +185,6 @@ sub list_images {
 return $res;
 }
 
-sub list_thinpools {
-my ($vg) = @_;
-
-my $lvs = lvm_list_volumes($vg);
-my $thinpools = [];
-
-foreach my $vg (keys %$lvs) {
-   foreach my $lvname (keys %{$lvs->{$vg}}) {
-   next if $lvs->{$vg}->{$lvname}->{lv_type} ne 't';
-   my $lv = $lvs->{$vg}->{$lvname};
-   $lv->{lv} = $lvname;
-   $lv->{vg} = $vg;
-   push @$thinpools, $lv;
-   }
-}
-
-return $thinpools;
-}
-
 sub status {
 my ($class, $storeid, $scfg, $cache) = @_;
 
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 16/36] plugin: dir: factor storage methods into separate common subs

[Max Carrara]

This creates new helper subroutines in the `Common` module in order
to remove other plugins' dependency on `DirPlugin` in the future.
The methods are changed to call these helpers instead.

Simultaneously, emit a `warn`ing if deprecated methods / functions are
being used instead of just relying on a comment in the source code.

The new helper functions are also fully documented and use prototyes.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common.pm| 201 +++
 src/PVE/Storage/DirPlugin.pm | 104 +-
 2 files changed, 227 insertions(+), 78 deletions(-)

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index 0ca0db1..3cc3c37 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -3,13 +3,21 @@ package PVE::Storage::Common;
 use strict;
 use warnings;
 
+use Encode qw(decode encode);
+use POSIX;
+
 use PVE::JSONSchema;
+use PVE::Tools;
 
 use parent qw(Exporter);
 
 our @EXPORT_OK = qw(
 get_deprecation_warning
 storage_parse_is_mountpoint
+storage_dir_get_volume_notes
+storage_dir_update_volume_notes
+storage_dir_get_volume_attribute
+storage_dir_update_volume_attribute
 );
 
 =pod
@@ -91,4 +99,197 @@ sub storage_parse_is_mountpoint : prototype($) {
 return $is_mp; # contains a path
 }
 
+# FIXME move into 'storage_dir_get_volume_attribute' when removing
+# 'storage_dir_get_volume_notes'
+my $get_volume_notes_impl = sub {
+my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+my ($vtype) = $class->parse_volname($volname);
+return if $vtype ne 'backup';
+
+my $path = $class->filesystem_path($scfg, $volname);
+$path .= $class->SUPER::NOTES_EXT;
+
+if (-f $path) {
+   my $data = PVE::Tools::file_get_contents($path);
+   return eval { decode('UTF-8', $data, 1) } // $data;
+}
+
+return '';
+};
+
+=pod
+
+=head3 storage_dir_get_volume_notes
+
+$notes = storage_dir_get_volume_notes($class, $scfg, $storeid, $volname, 
$timeout)
+
+B This helper is deprecated in favour of
+C> and will be removed in the future.
+
+This is a general implementation of 
C>
+that may be used by storage plugins with a directory-based storage. It is 
therefore
+useful for something like L, which
+(network stuff aside) also behaves like a directory.
+
+Returns the notes from a volume named C<$volname>. The volume is expected to
+belong to the storage with ID C<$storeid> that has the configuration C<$scfg>.
+The storage must in turn stem from the storage plugin C<$class>.
+
+The C<$timeout> parameter has no effect in this case.
+
+=cut
+
+sub storage_dir_get_volume_notes : prototype($) {
+warn get_deprecation_warning(
+   __PACKAGE__ . "::storage_dir_get_volume_attribute"
+);
+
+my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+return $get_volume_notes_impl->($class, $scfg, $storeid, $volname, 
$timeout);
+}
+
+# FIXME move into 'storage_dir_update_volume_attribute' when removing
+# 'storage_dir_update_volume_notes'
+my $update_volume_notes_impl = sub {
+my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_;
+
+my ($vtype) = $class->parse_volname($volname);
+die "only backups can have notes\n" if $vtype ne 'backup';
+
+my $path = $class->filesystem_path($scfg, $volname);
+$path .= $class->SUPER::NOTES_EXT;
+
+if (defined($notes) && $notes ne '') {
+   my $encoded = encode('UTF-8', $notes);
+   PVE::Tools::file_set_contents($path, $encoded);
+} else {
+   unlink $path or $! == ENOENT or die "could not delete notes - $!\n";
+}
+return;
+};
+
+
+=pod
+
+=head3 storage_dir_update_volume_notes
+
+$notes = storage_dir_update_volume_notes($class, $scfg, $storeid, 
$volname, $notes, $timeout)
+
+B This helper is deprecated in favour of
+C> and will be removed in the future.
+
+This is a general implementation of 
C>
+that may be used by storage plugins with a directory-based storage. It is 
therefore
+useful for something like L, which
+(network stuff aside) also behaves like a directory.
+
+Sets the notes of a volume named C<$volname> to C<$notes>. The volume is
+expected to belong to the storage with ID C<$storeid> that has the 
configuration
+C<$scfg>. The storage must in turn stem from the storage plugin C<$class>.
+
+The C<$timeout> parameter has no effect in this case.
+
+=cut
+
+sub storage_dir_update_volume_notes : prototype($$) {
+warn get_deprecation_warning(
+   __PACKAGE__ . "storage_dir_update_volume_attribute"
+);
+
+my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_;
+
+return $update_volume_notes_impl->($class, $scfg, $storeid, $volname, 
$notes, $timeout);
+}
+
+=pod
+
+=head3 storage_dir_get_

[pve-devel] [RFC pve-storage 09/36] plugin: lvm: move LVM helper subroutines into separate common module

[Max Carrara]

A plugin should only do "plugin stuff" and not provide helper subs
that other modules also use, so move those helpers into a separate
module and document them.

This new `Common::LVM` module is a submodule of `Common` (as its name
implies) as that logically groups LVM-related subroutines together.

The changes of this commit are backwards-compatible; the old subs act
as mere wrappers and will emit a warning when used.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common.pm   |   4 +
 src/PVE/Storage/Common/LVM.pm   | 432 
 src/PVE/Storage/Common/Makefile |   1 +
 src/PVE/Storage/LVMPlugin.pm| 254 +--
 4 files changed, 497 insertions(+), 194 deletions(-)
 create mode 100644 src/PVE/Storage/Common/LVM.pm

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index a2ae979..0ca0db1 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -32,6 +32,10 @@ be grouped in a submodule can also be found here.
 
 =over
 
+=item C
+
+Utilities concerned with LVM, such as manipulating logical volumes.
+
 =item C
 
 Utilities concerned with working with paths.
diff --git a/src/PVE/Storage/Common/LVM.pm b/src/PVE/Storage/Common/LVM.pm
new file mode 100644
index 000..e0e3263
--- /dev/null
+++ b/src/PVE/Storage/Common/LVM.pm
@@ -0,0 +1,432 @@
+package PVE::Storage::Common::LVM;
+
+use strict;
+use warnings;
+
+use IO::File;
+
+use PVE::Tools qw(run_command trim);
+
+use parent qw(Exporter);
+
+our @EXPORT_OK = qw(
+lvm_pv_info
+lvm_clear_first_sector
+lvm_create_volume_group
+lvm_destroy_volume_group
+lvm_vgs
+lvm_list_volumes
+lvm_lvcreate
+lvm_lvrename
+);
+
+=pod
+
+=head1 NAME
+
+Common::LVM - Provides helper subroutines that wrap commonly used LVM commands
+
+=head1 FUNCTIONS
+
+=cut
+
+my $ignore_no_medium_warnings = sub {
+my $line = shift;
+# ignore those, most of the time they're from (virtual) IPMI/iKVM devices
+# and just spam the log..
+if ($line !~ /open failed: No medium found/) {
+   print STDERR "$line\n";
+}
+};
+
+=pod
+
+=head3 lvm_pv_info
+
+$pvinfo = lvm_pv_info($device)
+
+Returns a hash containing information for a I specified
+by C<$device>, which must be a valid device path under C.
+
+The returned hash has the following structure:
+
+{
+   pvname => "some-pv-name",
+   size => 15728640,  # size in kibibytes!
+   vgname => "some-vg-name",
+   uuid => "1ba3cb42-5407-4cd5-9754-7060dc36ce6d",
+}
+
+This function will die if no C<$device> is specified of if multiple PV entries
+exist for C<$device>.
+
+Should the C<$device> not have an C label, this function will return
+C instead.
+
+=cut
+
+sub lvm_pv_info : prototype($) {
+my ($device) = @_;
+
+die "no device specified" if !$device;
+
+my $has_label = 0;
+
+my $cmd = ['/usr/bin/file', '-L', '-s', $device];
+run_command($cmd, outfunc => sub {
+   my $line = shift;
+   $has_label = 1 if $line =~ m/LVM2/;
+});
+
+return undef if !$has_label;
+
+$cmd = ['/sbin/pvs', '--separator', ':', '--noheadings', '--units', 'k',
+   '--unbuffered', '--nosuffix', '--options',
+   'pv_name,pv_size,vg_name,pv_uuid', $device];
+
+my $pvinfo;
+run_command($cmd, outfunc => sub {
+   my $line = shift;
+
+   $line = trim($line);
+
+   my ($pvname, $size, $vgname, $uuid) = split(':', $line);
+
+   die "found multiple pvs entries for device '$device'\n"
+   if $pvinfo;
+
+   $pvinfo = {
+   pvname => $pvname,
+   size => int($size),
+   vgname => $vgname,
+   uuid => $uuid,
+   };
+});
+
+return $pvinfo;
+}
+
+=pod
+
+=head3 lvm_clear_first_sector
+
+lvm_clear_first_sector($device)
+
+Clears the first sector (first 512 bits) of the given block device C<$device>.
+
+B Use with caution. This function does not actually check whether
+a valid device is passed or not.
+
+=cut
+
+sub lvm_clear_first_sector : prototype($) {
+my ($dev) = shift;
+
+if (my $fh = IO::File->new($dev, "w")) {
+   my $buf = 0 x 512;
+   syswrite $fh, $buf;
+   $fh->close();
+}
+}
+
+=pod
+
+=head3 lvm_create_volume_group
+
+lvm_create_volume_group($device, $vgname, $shared)
+
+Creates a I for the block device C<$device> with the name
+C<$vgname>. The C<$shared> parameter is currently unused.
+
+Dies if C<$device> is already part of a volume group.
+
+If C<$device> is already part of a volume group with the exact same name as in
+C<$vgname>, nothing will be done and the function returns early.
+
+=cut
+
+sub lvm_create_volume_group :

[pve-devel] [RFC pve-storage 15/36] plugin: btrfs: replace deprecated helpers from directory plugin

[Max Carrara]

with the recently moved ones from `Common` and `Common::Path`.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/BTRFSPlugin.pm | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/PVE/Storage/BTRFSPlugin.pm b/src/PVE/Storage/BTRFSPlugin.pm
index a503404..10fd441 100644
--- a/src/PVE/Storage/BTRFSPlugin.pm
+++ b/src/PVE/Storage/BTRFSPlugin.pm
@@ -13,6 +13,8 @@ use POSIX qw(EEXIST);
 
 use PVE::Tools qw(run_command dir_glob_foreach);
 
+use PVE::Storage::Common qw(storage_parse_is_mountpoint);
+use PVE::Storage::Common::Path qw(path_is_mounted);
 use PVE::Storage::DirPlugin;
 
 use constant {
@@ -122,8 +124,8 @@ sub activate_storage {
 my $path = $scfg->{path};
 $class->config_aware_base_mkdir($scfg, $path);
 
-my $mp = PVE::Storage::DirPlugin::parse_is_mountpoint($scfg);
-if (defined($mp) && !PVE::Storage::DirPlugin::path_is_mounted($mp, 
$cache->{mountdata})) {
+my $mp = storage_parse_is_mountpoint($scfg);
+if (defined($mp) && !path_is_mounted($mp, $cache->{mountdata})) {
die "unable to activate storage '$storeid' - directory is expected to 
be a mount point but"
." is not mounted: '$mp'\n";
 }
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 14/36] api: replace usages of deprecated LVM thin pool helper sub

[Max Carrara]

with the "new" sub in Common::LVM.

Signed-off-by: Max Carrara 
---
 src/PVE/API2/Disks/LVMThin.pm | 6 +++---
 src/PVE/API2/Storage/Scan.pm  | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/PVE/API2/Disks/LVMThin.pm b/src/PVE/API2/Disks/LVMThin.pm
index bd46478..142a062 100644
--- a/src/PVE/API2/Disks/LVMThin.pm
+++ b/src/PVE/API2/Disks/LVMThin.pm
@@ -3,7 +3,7 @@ package PVE::API2::Disks::LVMThin;
 use strict;
 use warnings;
 
-use PVE::Storage::Common::LVM qw(lvm_vgs lvm_create_volume_group lvm_pv_info);
+use PVE::Storage::Common::LVM qw(lvm_vgs lvm_create_volume_group lvm_pv_info 
lvm_list_thinpools);
 use PVE::Diskmanage;
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::API2::Storage::Config;
@@ -65,7 +65,7 @@ __PACKAGE__->register_method ({
 },
 code => sub {
my ($param) = @_;
-   return PVE::Storage::LvmThinPlugin::list_thinpools(undef);
+   return lvm_list_thinpools(undef);
 }});
 
 __PACKAGE__->register_method ({
@@ -221,7 +221,7 @@ __PACKAGE__->register_method ({
 
my $worker = sub {
PVE::Diskmanage::locked_disk_action(sub {
-   my $thinpools = PVE::Storage::LvmThinPlugin::list_thinpools();
+   my $thinpools = lvm_list_thinpools();
 
die "no such thin pool ${vg}/${lv}\n"
if !grep { $_->{lv} eq $lv && $_->{vg} eq $vg } 
$thinpools->@*;
diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm
index bad280d..5bb9883 100644
--- a/src/PVE/API2/Storage/Scan.pm
+++ b/src/PVE/API2/Storage/Scan.pm
@@ -8,7 +8,7 @@ use warnings;
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::RESTHandler;
 use PVE::SafeSyslog;
-use PVE::Storage::Common::LVM qw(lvm_vgs);
+use PVE::Storage::Common::LVM qw(lvm_vgs lvm_list_thinpools);
 use PVE::Storage;
 use PVE::SysFSTools;
 
@@ -409,7 +409,7 @@ __PACKAGE__->register_method({
 code => sub {
my ($param) = @_;
 
-   return PVE::Storage::LvmThinPlugin::list_thinpools($param->{vg});
+   return lvm_list_thinpools($param->{vg});
 }});
 
 __PACKAGE__->register_method({
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 07/36] common: introduce common module

[Max Carrara]

This module's purpose is to provide shared functions, constants, etc.
for storage plugins and storage-related operations.

It also contains the `get_deprecation_warning` subroutine that makes
it easier to warn developers and/or plugin authors that a subroutine
will be removed in the future.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common.pm   | 59 +
 src/PVE/Storage/Common/Makefile |  6 
 src/PVE/Storage/Makefile|  1 +
 3 files changed, 66 insertions(+)
 create mode 100644 src/PVE/Storage/Common.pm
 create mode 100644 src/PVE/Storage/Common/Makefile

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
new file mode 100644
index 000..f828c8c
--- /dev/null
+++ b/src/PVE/Storage/Common.pm
@@ -0,0 +1,59 @@
+package PVE::Storage::Common;
+
+use strict;
+use warnings;
+
+use parent qw(Exporter);
+
+our @EXPORT_OK = qw(
+get_deprecation_warning
+);
+
+=pod
+
+=head1 NAME
+
+PVE::Storage::Common - Shared functions and utilities for storage plugins and 
storage operations
+
+=head1 DESCRIPTION
+
+This module contains common subroutines that are mainly to be used by storage
+plugins. This module's submodules contain subroutines that are tailored towards
+a more specific or related purpose.
+
+Functions concerned with storage-related C things, helpers
+for the C API can be found in this module. Functions that can't
+be grouped in a submodule can also be found here.
+
+=head1 SUBMODULES
+
+=over
+
+=back
+
+=head1 FUNCTIONS
+
+=cut
+
+=pod
+
+=head3 get_deprecation_warning
+
+$warning = get_deprecation_warning($new_sub_name)
+
+Returns a string that warns that the subroutine that called 
C
+will be removed in the future and notes that C<$new_sub_name> should be used
+instead.
+
+=cut
+
+sub get_deprecation_warning : prototype($) {
+my ($new_sub_name) = @_;
+
+my $calling_sub = (caller(1))[3];
+
+return "The subroutine '$calling_sub' is deprecated and will be removed in 
"
+   . "the future. Please use '$new_sub_name' instead.";
+}
+
+1;
diff --git a/src/PVE/Storage/Common/Makefile b/src/PVE/Storage/Common/Makefile
new file mode 100644
index 000..0c4bba5
--- /dev/null
+++ b/src/PVE/Storage/Common/Makefile
@@ -0,0 +1,6 @@
+SOURCES = \
+
+
+.PHONY: install
+install:
+   for i in ${SOURCES}; do install -D -m 0644 $$i 
${DESTDIR}${PERLDIR}/PVE/Storage/Common/$$i; done
diff --git a/src/PVE/Storage/Makefile b/src/PVE/Storage/Makefile
index d5cc942..2627062 100644
--- a/src/PVE/Storage/Makefile
+++ b/src/PVE/Storage/Makefile
@@ -18,5 +18,6 @@ SOURCES= \
 
 .PHONY: install
 install:
+   make -C Common install
for i in ${SOURCES}; do install -D -m 0644 $$i 
${DESTDIR}${PERLDIR}/PVE/Storage/$$i; done
make -C LunCmd install
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 06/36] api: remove unused import of LVM storage plugin

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/API2/Storage/Config.pm | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/PVE/API2/Storage/Config.pm b/src/PVE/API2/Storage/Config.pm
index 56124fe..648b0f9 100755
--- a/src/PVE/API2/Storage/Config.pm
+++ b/src/PVE/API2/Storage/Config.pm
@@ -8,7 +8,6 @@ use PVE::Tools qw(extract_param extract_sensitive_params);
 use PVE::Cluster qw(cfs_read_file cfs_write_file);
 use PVE::Storage;
 use PVE::Storage::Plugin;
-use PVE::Storage::LVMPlugin;
 use HTTP::Status qw(:constants);
 use Storable qw(dclone);
 use PVE::JSONSchema qw(get_standard_option);
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 05/36] plugin: cifs: make plugin-specific helpers private

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/CIFSPlugin.pm | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/PVE/Storage/CIFSPlugin.pm b/src/PVE/Storage/CIFSPlugin.pm
index 2184471..18dc017 100644
--- a/src/PVE/Storage/CIFSPlugin.pm
+++ b/src/PVE/Storage/CIFSPlugin.pm
@@ -13,7 +13,7 @@ use base qw(PVE::Storage::Plugin);
 
 # CIFS helper functions
 
-sub cifs_is_mounted : prototype($$) {
+my sub cifs_is_mounted : prototype($$) {
 my ($scfg, $mountdata) = @_;
 
 my ($mountpoint, $server, $share) = $scfg->@{'path', 'server', 'share'};
@@ -31,12 +31,12 @@ sub cifs_is_mounted : prototype($$) {
 return undef;
 }
 
-sub cifs_cred_file_name {
+my sub cifs_cred_file_name {
 my ($storeid) = @_;
 return "/etc/pve/priv/storage/${storeid}.pw";
 }
 
-sub cifs_delete_credentials {
+my sub cifs_delete_credentials {
 my ($storeid) = @_;
 
 if (my $cred_file = get_cred_file($storeid)) {
@@ -44,7 +44,7 @@ sub cifs_delete_credentials {
 }
 }
 
-sub cifs_set_credentials {
+my sub cifs_set_credentials {
 my ($password, $storeid) = @_;
 
 my $cred_file = cifs_cred_file_name($storeid);
@@ -55,7 +55,7 @@ sub cifs_set_credentials {
 return $cred_file;
 }
 
-sub get_cred_file {
+my sub get_cred_file {
 my ($storeid) = @_;
 
 my $cred_file = cifs_cred_file_name($storeid);
@@ -66,7 +66,7 @@ sub get_cred_file {
 return undef;
 }
 
-sub cifs_mount : prototype($) {
+my sub cifs_mount : prototype($) {
 my ($scfg, $storeid, $smbver, $user, $domain) = @_;
 
 my ($mountpoint, $server, $share, $options) = $scfg->@{'path', 'server', 
'share', 'options'};
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 11/36] plugin: lvmthin: replace usages of deprecated LVM helpers with new ones

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/LvmThinPlugin.pm | 17 +
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/PVE/Storage/LvmThinPlugin.pm b/src/PVE/Storage/LvmThinPlugin.pm
index 4b23623..480cc78 100644
--- a/src/PVE/Storage/LvmThinPlugin.pm
+++ b/src/PVE/Storage/LvmThinPlugin.pm
@@ -6,6 +6,7 @@ use warnings;
 use IO::File;
 
 use PVE::Tools qw(run_command trim);
+use PVE::Storage::Common::LVM qw(lvm_vgs lvm_list_volumes);
 use PVE::Storage::Plugin;
 use PVE::Storage::LVMPlugin;
 use PVE::JSONSchema qw(get_standard_option);
@@ -89,7 +90,7 @@ sub alloc_image {
 die "illegal name '$name' - should be 'vm-$vmid-*'\n"
if  $name && $name !~ m/^vm-$vmid-/;
 
-my $vgs = PVE::Storage::LVMPlugin::lvm_vgs();
+my $vgs = lvm_vgs();
 
 my $vg = $scfg->{vgname};
 
@@ -111,7 +112,7 @@ sub free_image {
 
 my $vg = $scfg->{vgname};
 
-my $lvs = PVE::Storage::LVMPlugin::lvm_list_volumes($vg);
+my $lvs = lvm_list_volumes($vg);
 
 if (my $dat = $lvs->{$scfg->{vgname}}) {
 
@@ -137,7 +138,7 @@ sub list_images {
 
 my $vgname = $scfg->{vgname};
 
-$cache->{lvs} = PVE::Storage::LVMPlugin::lvm_list_volumes() if 
!$cache->{lvs};
+$cache->{lvs} = lvm_list_volumes() if !$cache->{lvs};
 
 my $res = [];
 
@@ -176,7 +177,7 @@ sub list_images {
 sub list_thinpools {
 my ($vg) = @_;
 
-my $lvs = PVE::Storage::LVMPlugin::lvm_list_volumes($vg);
+my $lvs = lvm_list_volumes($vg);
 my $thinpools = [];
 
 foreach my $vg (keys %$lvs) {
@@ -195,7 +196,7 @@ sub list_thinpools {
 sub status {
 my ($class, $storeid, $scfg, $cache) = @_;
 
-my $lvs = $cache->{lvs} ||= PVE::Storage::LVMPlugin::lvm_list_volumes();
+my $lvs = $cache->{lvs} ||= lvm_list_volumes();
 
 return if !$lvs->{$scfg->{vgname}};
 
@@ -214,7 +215,7 @@ sub status {
 my $activate_lv = sub {
 my ($vg, $lv, $cache) = @_;
 
-my $lvs = $cache->{lvs} ||= PVE::Storage::LVMPlugin::lvm_list_volumes();
+my $lvs = $cache->{lvs} ||= lvm_list_volumes();
 
 die "no such logical volume $vg/$lv\n" if !$lvs->{$vg} || 
!$lvs->{$vg}->{$lv};
 
@@ -295,7 +296,7 @@ sub create_base {
 die "create_base not possible with base image\n" if $isBase;
 
 my $vg = $scfg->{vgname};
-my $lvs = PVE::Storage::LVMPlugin::lvm_list_volumes($vg);
+my $lvs = lvm_list_volumes($vg);
 
 if (my $dat = $lvs->{$vg}) {
# to avoid confusion, reject if we find volume snapshots
@@ -404,7 +405,7 @@ sub volume_import {
 } else {
my $tempname;
my $vg = $scfg->{vgname};
-   my $lvs = PVE::Storage::LVMPlugin::lvm_list_volumes($vg);
+   my $lvs = lvm_list_volumes($vg);
if ($lvs->{$vg}->{$volname}) {
die "volume $vg/$volname already exists\n" if !$allow_rename;
warn "volume $vg/$volname already exists - importing with a 
different name\n";
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 10/36] api: replace usages of deprecated LVM helper subs with new ones

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/API2/Disks/LVM.pm | 12 ++--
 src/PVE/API2/Disks/LVMThin.pm | 10 +-
 src/PVE/API2/Storage/Scan.pm  |  4 ++--
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/PVE/API2/Disks/LVM.pm b/src/PVE/API2/Disks/LVM.pm
index 3c5bdfa..d062758 100644
--- a/src/PVE/API2/Disks/LVM.pm
+++ b/src/PVE/API2/Disks/LVM.pm
@@ -3,7 +3,7 @@ package PVE::API2::Disks::LVM;
 use strict;
 use warnings;
 
-use PVE::Storage::LVMPlugin;
+use PVE::Storage::Common::LVM qw(lvm_vgs lvm_create_volume_group 
lvm_destroy_volume_group);
 use PVE::Diskmanage;
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::API2::Storage::Config;
@@ -91,7 +91,7 @@ __PACKAGE__->register_method ({
 
my $result = [];
 
-   my $vgs = PVE::Storage::LVMPlugin::lvm_vgs(1);
+   my $vgs = lvm_vgs(1);
 
foreach my $vg_name (sort keys %$vgs) {
my $vg = $vgs->{$vg_name};
@@ -174,14 +174,14 @@ __PACKAGE__->register_method ({
PVE::Diskmanage::locked_disk_action(sub {
PVE::Diskmanage::assert_disk_unused($dev);
die "volume group with name '${name}' already exists on node 
'${node}'\n"
-   if PVE::Storage::LVMPlugin::lvm_vgs()->{$name};
+   if lvm_vgs()->{$name};
 
if (PVE::Diskmanage::is_partition($dev)) {
eval { PVE::Diskmanage::change_parttype($dev, '8E00'); };
warn $@ if $@;
}
 
-   PVE::Storage::LVMPlugin::lvm_create_volume_group($dev, $name);
+   lvm_create_volume_group($dev, $name);
 
PVE::Diskmanage::udevadm_trigger($dev);
 
@@ -240,10 +240,10 @@ __PACKAGE__->register_method ({
 
my $worker = sub {
PVE::Diskmanage::locked_disk_action(sub {
-   my $vgs = PVE::Storage::LVMPlugin::lvm_vgs(1);
+   my $vgs = lvm_vgs(1);
die "no such volume group '$name'\n" if !$vgs->{$name};
 
-   PVE::Storage::LVMPlugin::lvm_destroy_volume_group($name);
+   lvm_destroy_volume_group($name);
 
my $config_err;
if ($param->{'cleanup-config'}) {
diff --git a/src/PVE/API2/Disks/LVMThin.pm b/src/PVE/API2/Disks/LVMThin.pm
index f1c3957..bd46478 100644
--- a/src/PVE/API2/Disks/LVMThin.pm
+++ b/src/PVE/API2/Disks/LVMThin.pm
@@ -3,7 +3,7 @@ package PVE::API2::Disks::LVMThin;
 use strict;
 use warnings;
 
-use PVE::Storage::LvmThinPlugin;
+use PVE::Storage::Common::LVM qw(lvm_vgs lvm_create_volume_group lvm_pv_info);
 use PVE::Diskmanage;
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::API2::Storage::Config;
@@ -133,15 +133,15 @@ __PACKAGE__->register_method ({
PVE::Diskmanage::assert_disk_unused($dev);
 
die "volume group with name '${name}' already exists on node 
'${node}'\n"
-   if PVE::Storage::LVMPlugin::lvm_vgs()->{$name};
+   if lvm_vgs()->{$name};
 
if (PVE::Diskmanage::is_partition($dev)) {
eval { PVE::Diskmanage::change_parttype($dev, '8E00'); };
warn $@ if $@;
}
 
-   PVE::Storage::LVMPlugin::lvm_create_volume_group($dev, $name);
-   my $pv = PVE::Storage::LVMPlugin::lvm_pv_info($dev);
+   lvm_create_volume_group($dev, $name);
+   my $pv = lvm_pv_info($dev);
# keep some free space just in case
my $datasize = $pv->{size} - 128*1024;
# default to 1% for metadata
@@ -241,7 +241,7 @@ __PACKAGE__->register_method ({
}
 
if ($param->{'cleanup-disks'}) {
-   my $vgs = PVE::Storage::LVMPlugin::lvm_vgs(1);
+   my $vgs = lvm_vgs(1);
 
die "no such volume group '$vg'\n" if !$vgs->{$vg};
die "volume group '$vg' still in use\n" if 
$vgs->{$vg}->{lvcount} > 0;
diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm
index d7a8743..bad280d 100644
--- a/src/PVE/API2/Storage/Scan.pm
+++ b/src/PVE/API2/Storage/Scan.pm
@@ -8,7 +8,7 @@ use warnings;
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::RESTHandler;
 use PVE::SafeSyslog;
-use PVE::Storage::LVMPlugin;
+use PVE::Storage::Common::LVM qw(lvm_vgs);
 use PVE::Storage;
 use PVE::SysFSTools;
 
@@ -369,7 +369,7 @@ __PACKAGE__->register_method({
 code => sub {
my ($param) = @_;
 
-   my $res = PVE::Storage::LVMPlugin::lvm_vgs();
+   my $res = lvm_vgs();
return PVE::RESTHandler::hash_to_array($res, 'vg');
 }});
 
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 04/36] api: remove unused import of CIFS storage plugin

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/API2/Storage/Config.pm | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/PVE/API2/Storage/Config.pm b/src/PVE/API2/Storage/Config.pm
index e04b6ab..56124fe 100755
--- a/src/PVE/API2/Storage/Config.pm
+++ b/src/PVE/API2/Storage/Config.pm
@@ -9,7 +9,6 @@ use PVE::Cluster qw(cfs_read_file cfs_write_file);
 use PVE::Storage;
 use PVE::Storage::Plugin;
 use PVE::Storage::LVMPlugin;
-use PVE::Storage::CIFSPlugin;
 use HTTP::Status qw(:constants);
 use Storable qw(dclone);
 use PVE::JSONSchema qw(get_standard_option);
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 08/36] plugin: dir: move helper subs of directory plugin to common modules

[Max Carrara]

Because the directory plugin's subroutines are frequently used by
other plugins, it's best to factor these helpers into the common
module. This is a first step in making other plugins not depend on the
directory plugin anymore.

Simultaneously this commit also introduces the `Common::Path` module,
which ought to provide shared subroutines for operations on paths.

The changes of this commit are backwards-compatible; the old subs act
as mere wrappers and will emit a warning when used.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Common.pm   | 31 
 src/PVE/Storage/Common/Makefile |  1 +
 src/PVE/Storage/Common/Path.pm  | 51 +
 src/PVE/Storage/DirPlugin.pm| 38 
 4 files changed, 101 insertions(+), 20 deletions(-)
 create mode 100644 src/PVE/Storage/Common/Path.pm

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index f828c8c..a2ae979 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -3,10 +3,13 @@ package PVE::Storage::Common;
 use strict;
 use warnings;
 
+use PVE::JSONSchema;
+
 use parent qw(Exporter);
 
 our @EXPORT_OK = qw(
 get_deprecation_warning
+storage_parse_is_mountpoint
 );
 
 =pod
@@ -29,10 +32,19 @@ be grouped in a submodule can also be found here.
 
 =over
 
+=item C
+
+Utilities concerned with working with paths.
+
 =back
 
 =head1 FUNCTIONS
 
+B Functions prefixed with C are related to the C
+API and usually expect an C<$scfg> ("storage config") hash. This hash is
+expected to contain the configuration for I storage, which is (usually)
+acquired via C.
+
 =cut
 
 =pod
@@ -56,4 +68,23 @@ sub get_deprecation_warning : prototype($) {
. "the future. Please use '$new_sub_name' instead.";
 }
 
+=head3 storage_parse_is_mountpoint
+
+$path = storage_parse_is_mountpoint($scfg)
+
+Helper that tries to return a path if the given I hash C<$scfg>
+contains an C property. Returns C if it can't.
+
+=cut
+
+sub storage_parse_is_mountpoint : prototype($) {
+my ($scfg) = @_;
+my $is_mp = $scfg->{is_mountpoint};
+return undef if !defined $is_mp;
+if (defined(my $bool = PVE::JSONSchema::parse_boolean($is_mp))) {
+   return $bool ? $scfg->{path} : undef;
+}
+return $is_mp; # contains a path
+}
+
 1;
diff --git a/src/PVE/Storage/Common/Makefile b/src/PVE/Storage/Common/Makefile
index 0c4bba5..9455b81 100644
--- a/src/PVE/Storage/Common/Makefile
+++ b/src/PVE/Storage/Common/Makefile
@@ -1,4 +1,5 @@
 SOURCES = \
+ Path.pm \
 
 
 .PHONY: install
diff --git a/src/PVE/Storage/Common/Path.pm b/src/PVE/Storage/Common/Path.pm
new file mode 100644
index 000..7535dda
--- /dev/null
+++ b/src/PVE/Storage/Common/Path.pm
@@ -0,0 +1,51 @@
+package PVE::Storage::Common::Path;
+
+use strict;
+use warnings;
+
+use Cwd;
+
+use PVE::ProcFSTools;
+
+use parent qw(Exporter);
+
+our @EXPORT_OK = qw(
+path_is_mounted
+);
+
+=pod
+
+=head1 NAME
+
+PVE::Storage::Common::Path - Shared functions and utilities for manipulating 
paths
+
+=head1 FUNCTIONS
+
+=cut
+
+=pod
+
+=head3 path_is_mounted
+
+$is_mounted = path_is_mounted($mountpoint)
+$is_mounted = path_is_mounted($mountpoint, $mountdata)
+
+Checks if the given path in C<$mountpoint> is actually mounted. Optionally 
takes
+a C<$mountdata> hash returned by C in
+order to avoid repeatedly reading and parsing C.
+
+=cut
+
+# NOTE: should ProcFSTools::is_mounted accept an optional cache like this?
+sub path_is_mounted {
+my ($mountpoint, $mountdata) = @_;
+
+$mountpoint = Cwd::realpath($mountpoint); # symlinks
+return 0 if !defined($mountpoint); # path does not exist
+
+$mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata;
+return 1 if grep { $_->[1] eq $mountpoint } @$mountdata;
+return undef;
+}
+
+1;
diff --git a/src/PVE/Storage/DirPlugin.pm b/src/PVE/Storage/DirPlugin.pm
index 2efa8d5..ac0d365 100644
--- a/src/PVE/Storage/DirPlugin.pm
+++ b/src/PVE/Storage/DirPlugin.pm
@@ -3,12 +3,16 @@ package PVE::Storage::DirPlugin;
 use strict;
 use warnings;
 
-use Cwd;
 use Encode qw(decode encode);
 use File::Path;
 use IO::File;
 use POSIX;
 
+use PVE::Storage::Common qw(
+get_deprecation_warning
+storage_parse_is_mountpoint
+);
+use PVE::Storage::Common::Path;
 use PVE::Storage::Plugin;
 use PVE::JSONSchema qw(get_standard_option);
 
@@ -86,26 +90,20 @@ sub options {
 # Storage implementation
 #
 
-# NOTE: should ProcFSTools::is_mounted accept an optional cache like this?
 sub path_is_mounted {
-my ($mountpoint, $mountdata) = @_;
+warn get_deprecation_warning(
+   "PVE::Storage::Common::Path::path_is_mounted"
+);
 
-$mountpoint = Cwd::realpath($mountpoint); # symlinks
-return 0 if !defined($mountpoint); # path does not exist
-
-$mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata;
-return 1 if grep { $_->

[pve-devel] [RFC pve-storage 00/36] Refactor / Cleanup of Storage Plugins

[Max Carrara]

eparate sub-packages. In total, the
following new modules are introduced so far:

  * PVE::Storage::Common
  * PVE::Storage::Common::ISCSI
  * PVE::Storage::Common::LVM
  * PVE::Storage::Common::Path
  * PVE::Storage::Common::ZFS

Each module comes with documentation written in Perl's POD format, which
can be rendered via the `perldoc` CLI or be viewed via an LSP.

Open Questions
--

This is a non-exhaustive list of questions that arose during
development:

  1. Instead of making some helpers private and some explicitly public
 in a new module, should all helpers be made public instead? There
 could be third-party plugins out there using some of the helpers
 that were made private that we do not know of.

  2. Because the helpers of the PBSPlugin (which are also in
 `pve-common`) are expected to be put into a separate package in the
 future, should the helpers that are factored out into
 PVE::Storage::Common and its sub-packages also be put into a
 separate package or moved to an existing one?

  3. What would be the best way to keep track of deprecated helpers so
 that we know when to remove them?

  4. Because the refactored helpers (those in PVE::Storage::Common etc.)
 are public, should they fall under the same API guarantees as
 PVE::Storage? Should there be a different mechanism instead?

  5. What would be the best way to split these changes up into multiple
 smaller series (so that e.g. reviewing becomes easier)?

 My current idea would be to put the following changes into separate
 series:
   * Making other plugins independent of the DirPlugin
   * Factoring out LVM-related helpers
   * Factoring out ISCSI-related helpers
   * Factoring out ZFS-related helpers

  6. Are there any alternatives to the current approach(es) that might
 be better than what's currently in this RFC?

Closing Thoughts


Please don't hesitate to let me know what you think - and thank you very
much for reading this far :) Even if it's just a minor thing, any
feedback is greatly appreciated.

Summary of Changes
--

Max Carrara (36):
  plugin: base: remove old fixme comments
  plugin: btrfs: make plugin-specific helpers private
  plugin: cephfs: make plugin-specific helpers private
  api: remove unused import of CIFS storage plugin
  plugin: cifs: make plugin-specific helpers private
  api: remove unused import of LVM storage plugin
  common: introduce common module
  plugin: dir: move helper subs of directory plugin to common modules
  plugin: lvm: move LVM helper subroutines into separate common module
  api: replace usages of deprecated LVM helper subs with new ones
  plugin: lvmthin: replace usages of deprecated LVM helpers with new
ones
  plugin: lvmthin: move helper that lists thinpools to common LVM module
  common: lvm: update code style
  api: replace usages of deprecated LVM thin pool helper sub
  plugin: btrfs: replace deprecated helpers from directory plugin
  plugin: dir: factor storage methods into separate common subs
  plugin: dir: factor path validity check into helper methods
  plugin: btrfs: remove dependency on directory plugin
  plugin: cifs: remove dependency on directory plugin
  plugin: cephfs: remove dependency on directory plugin
  plugin: nfs: remove dependency on directory plugin
  plugin: btrfs: make helper methods private
  plugin: esxi: make helper subroutines private
  plugin: esxi: remove unused helper subroutine `query_vmdk_size`
  plugin: esxi: make helper methods private
  plugin: gluster: make helper subroutines private
  plugin: iscsi-direct: make helper subroutine `iscsi_ls` private
  plugin: iscsi: factor helper functions into common module
  plugin: iscsi: make helper subroutine `iscsi_session` private
  plugin: lvm: update definition of subroutine `check_tags`
  plugin: lvmthin: update definition of subroutine `activate_lv`
  plugin: nfs: make helper subroutines private
  plugin: rbd: update private sub signatures and make helpers private
  common: zfs: introduce module for common ZFS helpers
  plugin: zfspool: move helper `zfs_parse_zvol_list` to common module
  plugin: zfspool: refactor method `zfs_request` into helper subroutine

 src/PVE/API2/Disks/LVM.pm|  12 +-
 src/PVE/API2/Disks/LVMThin.pm|  14 +-
 src/PVE/API2/Storage/Config.pm   |   2 -
 src/PVE/API2/Storage/Scan.pm |   6 +-
 src/PVE/Storage.pm   |   4 +-
 src/PVE/Storage/BTRFSPlugin.pm   | 105 +++---
 src/PVE/Storage/CIFSPlugin.pm|  53 ++-
 src/PVE/Storage/CephFSPlugin.pm  |  47 ++-
 src/PVE/Storage/Common.pm| 303 
 src/PVE/Storage/Common/ISCSI.pm  | 475 
 src/PVE/Storage/Common/LVM.pm| 515 +++
 src/PVE/Storage/Common/Makefile  |  10 +
 src/PVE/Storage/Common/Path.pm   | 124 +++
 src/PVE/Storage/Common/ZFS.pm| 19

[pve-devel] [RFC pve-storage 03/36] plugin: cephfs: make plugin-specific helpers private

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/CephFSPlugin.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/PVE/Storage/CephFSPlugin.pm b/src/PVE/Storage/CephFSPlugin.pm
index 8aad518..98d1ba6 100644
--- a/src/PVE/Storage/CephFSPlugin.pm
+++ b/src/PVE/Storage/CephFSPlugin.pm
@@ -16,7 +16,7 @@ use PVE::Tools qw(run_command file_set_contents);
 
 use base qw(PVE::Storage::Plugin);
 
-sub cephfs_is_mounted {
+my sub cephfs_is_mounted {
 my ($scfg, $storeid, $mountdata) = @_;
 
 my $cmd_option = PVE::CephConfig::ceph_connect_option($scfg, $storeid);
@@ -39,7 +39,7 @@ sub cephfs_is_mounted {
 }
 
 # FIXME: remove once it's possible to specify _netdev for fuse.ceph mounts
-sub systemd_netmount {
+my sub systemd_netmount {
 my ($where, $type, $what, $opts) = @_;
 
 # don't do default deps, systemd v241 generator produces ordering deps on both
@@ -76,7 +76,7 @@ EOF
 
 }
 
-sub cephfs_mount {
+my sub cephfs_mount {
 my ($scfg, $storeid) = @_;
 
 my $mountpoint = $scfg->{path};
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 02/36] plugin: btrfs: make plugin-specific helpers private

[Max Carrara]

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/BTRFSPlugin.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/PVE/Storage/BTRFSPlugin.pm b/src/PVE/Storage/BTRFSPlugin.pm
index 42815cb..a503404 100644
--- a/src/PVE/Storage/BTRFSPlugin.pm
+++ b/src/PVE/Storage/BTRFSPlugin.pm
@@ -156,7 +156,7 @@ sub update_volume_attribute {
 }
 
 # croak would not include the caller from within this module
-sub __error {
+my sub __error {
 my ($msg) = @_;
 my (undef, $f, $n) = caller(1);
 die "$msg at $f: $n\n";
@@ -164,7 +164,7 @@ sub __error {
 
 # Given a name (eg. `vm-VMID-disk-ID.raw`), take the part up to the format 
suffix as the name of
 # the subdirectory (subvolume).
-sub raw_name_to_dir($) {
+my sub raw_name_to_dir($) {
 my ($raw) = @_;
 
 # For the subvolume directory Strip the `.` suffix:
@@ -175,7 +175,7 @@ sub raw_name_to_dir($) {
 __error "internal error: bad disk name: $raw";
 }
 
-sub raw_file_to_subvol($) {
+my sub raw_file_to_subvol($) {
 my ($file) = @_;
 
 if ($file =~ m|^(.*)/disk\.raw$|) {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [RFC pve-storage 01/36] plugin: base: remove old fixme comments

[Max Carrara]

These have been around since 2012 - suffice to say they're not needed
anymore.

Signed-off-by: Max Carrara 
---
 src/PVE/Storage/Plugin.pm | 19 ---
 1 file changed, 19 deletions(-)

diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 6444390..3219f1f 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -299,19 +299,6 @@ sub parse_lvm_name {
 return $name;
 }
 
-# fixme: do we need this
-#PVE::JSONSchema::register_format('pve-storage-portal', \&verify_portal);
-#sub verify_portal {
-#my ($portal, $noerr) = @_;
-#
-## IP with optional port
-#if ($portal !~ m/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?$/) {
-#  return undef if $noerr;
-#  die "value does not look like a valid portal address\n";
-#}
-#return $portal;
-#}
-
 PVE::JSONSchema::register_format('pve-storage-portal-dns', 
\&verify_portal_dns);
 sub verify_portal_dns {
 my ($portal, $noerr) = @_;
@@ -457,12 +444,6 @@ sub decode_value {
}
}
 
-   # fixme:
-   # no node restrictions for local storage
-   #if ($storeid && $storeid eq 'local' && scalar(keys(%$res))) {
-   #die "storage '$storeid' does not allow node restrictions\n";
-   #}
-
return $res;
 } elsif ($key eq 'content-dirs') {
my $valid_content = $def->{content}->[0];
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH many v8 00/13] notifications: notification metadata matching improvements

[Max Carrara]

On Fri Jul 5, 2024 at 3:46 PM CEST, Lukas Wagner wrote:
> This patch series attempts to improve the user experience when creating
> notification matchers.

The below can pretty much just be considered "proofreading" as I haven't
built and tested your changes, but since you already got a lot of
feedback on the last couple versions, I think that's fine. ;) Just
wanted to comment anyway.

The patches are rather easy to follow, and even though I'm no expert
when it comes to Ext JS, the UI changes look fine to me too. The new UI
logic feels (and is) much cleaner than before. There's nothing I can
otherwise comment on; everything's pretty straight-forward.

The *only* things I have noticed are rather minor - there are two tiny
typos in the commit messages of patch 01 and 02, but these can probably
be fixed when applying the series:
  01: Last sentence of message - "It it can be considered internal." 
  02: First sentence of message - "This allows us to access us the [...]"

That's it otherwise from me - LGTM.

Reviewed-by: Max Carrara 

>
> Some of the noteworthy changes:
>   - Allow setting a custom backup job ID, similar how we handle it for
>   sync/prune jobs in PBS (to allow recognizable names used in matchers)
>   - New metadata fields:
> - job-id: Job ID for backup-jobs or replication-jobs
>   - Add an API that enumerates known notification metadata fields/values
>   - Suggest known fields/values in match rule window
>   - Some code clean up for match rule edit window
>   - Extended the 'exact' match-field mode - it now allows setting multiple
> allowed values, separated via ',':
>   e.g. `match-field exact:type=replication,fencing
> Originally, I created a separate 'list' match type for this, but
> since the semantics for a list with one value and 'exact' mode
> are identical, I decided to just extend 'exact'.
> This is a safe change since there are are no values where a ','
> makes any sense (config IDs, hostnames)
>
> NOTE: Might need a versionened break, since the widget-toolkit-patches
> depend on new APIs provided by pve-manager. If the API is not present,
> creating matchers with 'match-field' does not work (cannot load lists
> of known values/fields)
>
> Inter-Dependencies:
>   - the widget-toolkit dep in pve-manager needs to be bumped
> to at least 4.1.4
> (we need "utils: add mechanism to add and override translatable 
> notification event
> descriptions in the product specific UIs", otherwise the UI breaks
> with the pve-manager patches applied) --> already included a patch for
> this
>   - widget-toolkit relies on a new API endpoint provided by pve-manager:
> --> we require a versioned break in widget-toolkit on pve-manager
>   - pve-manager needs bumped pve-guest-common (thx @Fabian)
>
> Changelog:
>   - v8: incorporate feedback from @Fabian, thx a lot!
> - Made 'job-id' API param usable by root@pam only - this should prevent
>   abuse by spoofing job-id, potentially bothering other users with bogus
>   notifications.
> - Don't set 'job-id' when starting a backup job via 'Run now' in the UI
> - Add a note to the docs explaining when job-id is set and when not.
> - Drop already applied patches
>   - v7: incorporated some more feedback from @Fiona, thx!
> - Fixed error when switching from 'exact' to 'regex' if the text field
>   was empty
> - rebased to latest master
> - 'backport' doc improvements from PBS
> - bumped widget-toolkit dep
>   - v6: incorporate feedback from @Fiona, thx!
> - rename 'id' -> 'job-id' in VZDump API handler
> - consolidate 'replication-job'/'backup-job' to 'job-id'
> - Move 'job-id' setting to advanced tab in backup job edit.
> - Don't use 'internal' flag to mark translatable fields, since
>   the only field where that's necessary is 'type' for now - so
>   just add a hardcoded check
>   - v5:
> - Rebased onto latest master, resolving some small conflict
>   - v4:
> - widget-toolkit: break out changes for the utils module so that they
>   can be applied ahead of time to ease dep bumping
> - don't show Job IDs in the backup/replication job columns
>   - v3:
> - Drop already applied patches for `proxmox`
> - Rebase onto latest master - minor conflict resolution was needed
>   - v2:
> - include 'type' metadata field for forwarded mails
>   --> otherwise it's not possible to match them
&g

Re: [pve-devel] [PATCH v1 proxmox 0/3] Fix #5105: Overhaul TLS Handshake Checking Logic

[Max Carrara]

On Fri Jul 5, 2024 at 6:20 PM CEST, Max Carrara wrote:
> Fix #5105: Overhaul TLS Handshake Checking Logic
> 

Oh, woops - this should've gone to pbs-devel. Will send it there;
disregard this series, please. Was a bit too quick on the trigger on
Friday it seems ;)

>
> This series fixes bug #5105 [1] by overhauling the TLS handshake
> checking logic, which is performed when using a connection acceptor
> variant with optional TLS.
>
> In the case of PBS (the only place where this is used, to my knowledge),
> any requests made over plain HTTP are redirected to the same host, but
> clients are instructed to use HTTPS instead.
>
> The TLS handshake checking logic determines whether the client uses HTTP
> or HTTPS by peeking into the stream buffer -- if the first 5 received
> bytes look like a TLS handshake fragment, the connection is passed on to
> OpenSSL before being accepted. Otherwise the connection is assumed to be
> unencrypted, i.e. plain HTTP.
>
> However, this logic contains two errors:
>
>   1. The timeout duration is too short - one second is too little
>   2. When a timeout occurs, the connection is assumed to be unencrypted
>  (and thus plain HTTP)
>
> The patches 01 and 02 are mainly done in preparation for patch 03 (which
> contains the actual fix), improving the overall quality of the code and
> including the peer's address in error logs.
>
> Please see the individual patches for more information.
>
> Special thanks go to Stefan Hanreich whose advice helped identifying
> many individual puzzle pieces comprising this issue.
>
> References
> ------
>
> [1]: https://bugzilla.proxmox.com/show_bug.cgi?id=5105
>
> Summary of Changes
> --
>
> Max Carrara (3):
>   rest-server: connection: clean up accept data flow
>   rest-server: connection: log peer address on error
>   fix #5105: rest-server: connection: overhaul TLS handshake check logic
>
>  proxmox-rest-server/src/connection.rs | 165 +-
>  1 file changed, 85 insertions(+), 80 deletions(-)



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v1 proxmox 2/3] rest-server: connection: log peer address on error

[Max Carrara]

.. in order to make debugging easier and logs more helpful.

Signed-off-by: Max Carrara 
---
 proxmox-rest-server/src/connection.rs | 42 ---
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/proxmox-rest-server/src/connection.rs 
b/proxmox-rest-server/src/connection.rs
index 243348c0..470021d7 100644
--- a/proxmox-rest-server/src/connection.rs
+++ b/proxmox-rest-server/src/connection.rs
@@ -2,6 +2,7 @@
 //!
 //! Hyper building block.
 
+use std::net::SocketAddr;
 use std::os::unix::io::AsRawFd;
 use std::path::PathBuf;
 use std::pin::Pin;
@@ -257,6 +258,7 @@ impl From<(ClientSender, InsecureClientSender)> for Sender {
 
 struct AcceptState {
 pub socket: InsecureClientStream,
+pub peer: SocketAddr,
 pub acceptor: Arc>,
 pub accept_counter: Arc<()>,
 }
@@ -276,9 +278,9 @@ impl AcceptBuilder {
 let mut shutdown_future = crate::shutdown_future().fuse();
 
 loop {
-let socket = futures::select! {
+let (socket, peer) = futures::select! {
 res = self.try_setup_socket(&listener).fuse() => match res {
-Ok(socket) => socket,
+Ok(socket_peer) => socket_peer,
 Err(err) => {
 log::error!("couldn't set up TCP socket: {err}");
 continue;
@@ -291,12 +293,13 @@ impl AcceptBuilder {
 let accept_counter = Arc::clone(&accept_counter);
 
 if Arc::strong_count(&accept_counter) > self.max_pending_accepts {
-log::error!("connection rejected - too many open connections");
+log::error!("[{peer}] connection rejected - too many open 
connections");
 continue;
 }
 
 let state = AcceptState {
 socket,
+peer,
 acceptor,
 accept_counter,
 };
@@ -328,7 +331,7 @@ impl AcceptBuilder {
 async fn try_setup_socket(
 &self,
 listener: &TcpListener,
-) -> Result {
+) -> Result<(InsecureClientStream, SocketAddr), Error> {
 let (socket, peer) = match listener.accept().await {
 Ok(connection) => connection,
 Err(error) => {
@@ -338,10 +341,10 @@ impl AcceptBuilder {
 
 socket
 .set_nodelay(true)
-.context("error while setting TCP_NODELAY on socket")?;
+.with_context(|| format!("[{peer}] error while setting TCP_NODELAY 
on socket"))?;
 
 proxmox_sys::linux::socket::set_tcp_keepalive(socket.as_raw_fd(), 
self.tcp_keepalive_time)
-.context("error while setting SO_KEEPALIVE on socket")?;
+.with_context(|| format!("[{peer}] error while setting 
SO_KEEPALIVE on socket"))?;
 
 #[cfg(feature = "rate-limited-stream")]
 let socket = match self.lookup_rate_limiter.clone() {
@@ -349,13 +352,12 @@ impl AcceptBuilder {
 None => RateLimitedStream::with_limiter(socket, None, None),
 };
 
-#[cfg(not(feature = "rate-limited-stream"))]
-let _peer = peer;
-
-Ok(socket)
+Ok((socket, peer))
 }
 
 async fn do_accept_tls(state: AcceptState, flags: AcceptFlags, 
secure_sender: ClientSender) {
+let peer = state.peer;
+
 let ssl = {
 // limit acceptor_guard scope
 // Acceptor can be reloaded using the command socket 
"reload-certificate" command
@@ -364,7 +366,9 @@ impl AcceptBuilder {
 match openssl::ssl::Ssl::new(acceptor_guard.context()) {
 Ok(ssl) => ssl,
 Err(err) => {
-log::error!("failed to create Ssl object from Acceptor 
context - {err}");
+log::error!(
+"[{peer}] failed to create Ssl object from Acceptor 
context - {err}"
+);
 return;
 }
 }
@@ -373,7 +377,9 @@ impl AcceptBuilder {
 let secure_stream = match tokio_openssl::SslStream::new(ssl, 
state.socket) {
 Ok(stream) => stream,
 Err(err) => {
-log::error!("failed to create SslStream using ssl and 
connection socket - {err}");
+log::error!(
+"[{peer}] failed to create SslStream using ssl and 
connection socket - {err}"
+);
 return;
 }
 };
@@ -388,17 +394,17 @@ impl AcceptBuilder {
 match result {
 Ok(Ok(())) => {
 if secure_sender.send(Ok(secure_stream)).await.is_err() && 
flags.is_debug {
-log::error!("detected closed connection channel");
+log::error!(

[pve-devel] [PATCH v1 proxmox 3/3] fix #5105: rest-server: connection: overhaul TLS handshake check logic

[Max Carrara]

On rare occasions, the TLS "client hello" message [1] is delayed after
a connection with the server was established, which causes HTTPS
requests to fail before TLS was even negotiated. In these cases, the
server would incorrectly respond with "HTTP/1.1 400 Bad Request"
instead of closing the connection (or similar).

The reasons for the "client hello" being delayed seem to vary; one
user noticed that the issue went away completely after they turned off
UFW [2]. Another user noticed (during private correspondence) that the
issue only appeared when connecting to their PBS instance via WAN, but
not from within their VPN. In the WAN case a firewall was also
present. The same user kindly provided tcpdumps and strace logs on
request.

The issue was finally reproduced with the following Python script:

  import socket
  import time

  HOST: str = ...
  PORT: int = ...

  with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
  sock.connect((HOST, PORT))
  time.sleep(1.5) # simulate firewall / proxy / etc. delay
  sock.sendall(b"\x16\x03\x01\x02\x00")
  data = sock.recv(256)
  print(data)

The additional delay before sending the first 5 bytes of the "client
hello" message causes the handshake checking logic to incorrectly fall
back to plain HTTP.

All of this is fixed by the following:

  1. Increase the timeout duration to 10 seconds (from 1)
  2. Instead of falling back to plain HTTP, refuse to accept the
 connection if the TLS handshake wasn't initiated before the
 timeout limit is reached
  3. Only accept plain HTTP if the first 5 bytes do not correspond to
 a TLS handshake fragment [3]
  4. Do not take the last number of bytes that were in the buffer into
 account; instead, only perform the actual handshake check if
 5 bytes are in the peek buffer

Regarding 1.: This should be generous enough for any client to be able
to initiate a TLS handshake, despite its surrounding circumstances.

Regarding 4.: While this is not 100% related to the issue, altering
this condition simplifies the overall logic and removes a potential
source of unintended behaviour.

[1]: https://www.rfc-editor.org/rfc/rfc8446.html#section-4.1.2
[2]: 
https://forum.proxmox.com/threads/disable-default-http-redirects-on-8007.142312/post-675352
[3]: https://www.rfc-editor.org/rfc/rfc8446.html#section-4.1.2

Signed-off-by: Max Carrara 
Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=5105
---
 proxmox-rest-server/src/connection.rs | 69 ---
 1 file changed, 31 insertions(+), 38 deletions(-)

diff --git a/proxmox-rest-server/src/connection.rs 
b/proxmox-rest-server/src/connection.rs
index 470021d7..a2d54b18 100644
--- a/proxmox-rest-server/src/connection.rs
+++ b/proxmox-rest-server/src/connection.rs
@@ -418,70 +418,63 @@ impl AcceptBuilder {
 secure_sender: ClientSender,
 insecure_sender: InsecureClientSender,
 ) {
-let peer = state.peer;
+const CLIENT_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
 
-let client_initiates_handshake = {
-#[cfg(feature = "rate-limited-stream")]
-let socket_ref = state.socket.inner();
+let peer = state.peer;
 
-#[cfg(not(feature = "rate-limited-stream"))]
-let socket_ref = &state.socket;
+#[cfg(feature = "rate-limited-stream")]
+let socket_ref = state.socket.inner();
 
-match Self::wait_for_client_tls_handshake(socket_ref).await {
-Ok(initiates_handshake) => initiates_handshake,
-Err(err) => {
-log::error!("[{peer}] error checking for TLS handshake: 
{err}");
-return;
-}
-}
-};
+#[cfg(not(feature = "rate-limited-stream"))]
+let socket_ref = &state.socket;
 
-if !client_initiates_handshake {
-let insecure_stream = Box::pin(state.socket);
+let handshake_res =
+Self::wait_for_client_tls_handshake(socket_ref, 
CLIENT_HANDSHAKE_TIMEOUT).await;
 
-if insecure_sender.send(Ok(insecure_stream)).await.is_err() && 
flags.is_debug {
-log::error!("[{peer}] detected closed connection channel")
+match handshake_res {
+Ok(true) => {
+Self::do_accept_tls(state, flags, secure_sender).await;
 }
+Ok(false) => {
+let insecure_stream = Box::pin(state.socket);
 
-return;
+if let Err(send_err) = 
insecure_sender.send(Ok(insecure_stream)).await {
+log::error!("[{peer}] failed to accept connection - 
connection channel closed: {send_err}");
+}
+}
+Err(err) => {
+log::error!("[{peer}

[pve-devel] [PATCH v1 proxmox 1/3] rest-server: connection: clean up accept data flow

[Max Carrara]

This adds the structs `AcceptState` and `AcceptFlags` and adapts
relevant method signatures of `AcceptBuilder` accordingly. This makes
it easier to add further parameters in the future.

Signed-off-by: Max Carrara 
---
 proxmox-rest-server/src/connection.rs | 72 ++-
 1 file changed, 38 insertions(+), 34 deletions(-)

diff --git a/proxmox-rest-server/src/connection.rs 
b/proxmox-rest-server/src/connection.rs
index 34b585cb..243348c0 100644
--- a/proxmox-rest-server/src/connection.rs
+++ b/proxmox-rest-server/src/connection.rs
@@ -255,6 +255,16 @@ impl From<(ClientSender, InsecureClientSender)> for Sender 
{
 }
 }
 
+struct AcceptState {
+pub socket: InsecureClientStream,
+pub acceptor: Arc>,
+pub accept_counter: Arc<()>,
+}
+
+struct AcceptFlags {
+pub is_debug: bool,
+}
+
 impl AcceptBuilder {
 async fn accept_connections(
 self,
@@ -285,24 +295,26 @@ impl AcceptBuilder {
 continue;
 }
 
+let state = AcceptState {
+socket,
+acceptor,
+accept_counter,
+};
+
+let flags = AcceptFlags {
+is_debug: self.debug,
+};
+
 match sender {
 Sender::Secure(ref secure_sender) => {
-let accept_future = Self::do_accept_tls(
-socket,
-acceptor,
-accept_counter,
-self.debug,
-secure_sender.clone(),
-);
+let accept_future = Self::do_accept_tls(state, flags, 
secure_sender.clone());
 
 tokio::spawn(accept_future);
 }
 Sender::SecureAndInsecure(ref secure_sender, ref 
insecure_sender) => {
 let accept_future = Self::do_accept_tls_optional(
-socket,
-acceptor,
-accept_counter,
-self.debug,
+state,
+flags,
 secure_sender.clone(),
 insecure_sender.clone(),
 );
@@ -343,17 +355,11 @@ impl AcceptBuilder {
 Ok(socket)
 }
 
-async fn do_accept_tls(
-socket: InsecureClientStream,
-acceptor: Arc>,
-accept_counter: Arc<()>,
-debug: bool,
-secure_sender: ClientSender,
-) {
+async fn do_accept_tls(state: AcceptState, flags: AcceptFlags, 
secure_sender: ClientSender) {
 let ssl = {
 // limit acceptor_guard scope
 // Acceptor can be reloaded using the command socket 
"reload-certificate" command
-let acceptor_guard = acceptor.lock().unwrap();
+let acceptor_guard = state.acceptor.lock().unwrap();
 
 match openssl::ssl::Ssl::new(acceptor_guard.context()) {
 Ok(ssl) => ssl,
@@ -364,7 +370,7 @@ impl AcceptBuilder {
 }
 };
 
-let secure_stream = match tokio_openssl::SslStream::new(ssl, socket) {
+let secure_stream = match tokio_openssl::SslStream::new(ssl, 
state.socket) {
 Ok(stream) => stream,
 Err(err) => {
 log::error!("failed to create SslStream using ssl and 
connection socket - {err}");
@@ -381,41 +387,39 @@ impl AcceptBuilder {
 
 match result {
 Ok(Ok(())) => {
-if secure_sender.send(Ok(secure_stream)).await.is_err() && 
debug {
+if secure_sender.send(Ok(secure_stream)).await.is_err() && 
flags.is_debug {
 log::error!("detected closed connection channel");
 }
 }
 Ok(Err(err)) => {
-if debug {
+if flags.is_debug {
 log::error!("https handshake failed - {err}");
 }
 }
 Err(_) => {
-if debug {
+if flags.is_debug {
 log::error!("https handshake timeout");
 }
 }
 }
 
-drop(accept_counter); // decrease reference count
+drop(state.accept_counter); // decrease reference count
 }
 
 async fn do_accept_tls_optional(
-socket: InsecureClientStream,
-acceptor: Arc>,
-accept_counter: Arc<()>,
-debug: bool,
+state: AcceptState,
+flags: AcceptFlags,
 secure_sender: ClientSender,
 insecure_sender: InsecureClientSender,
 ) {
 let client_initiates_handshake = {
 #[cfg(feature = "rate-limited-stream")]
-let socket = socket.inner();
+let socket_ref = state.socket.inner();
 
 #[cfg(not(feature

[pve-devel] [PATCH v1 proxmox 0/3] Fix #5105: Overhaul TLS Handshake Checking Logic

[Max Carrara]

Fix #5105: Overhaul TLS Handshake Checking Logic


This series fixes bug #5105 [1] by overhauling the TLS handshake
checking logic, which is performed when using a connection acceptor
variant with optional TLS.

In the case of PBS (the only place where this is used, to my knowledge),
any requests made over plain HTTP are redirected to the same host, but
clients are instructed to use HTTPS instead.

The TLS handshake checking logic determines whether the client uses HTTP
or HTTPS by peeking into the stream buffer -- if the first 5 received
bytes look like a TLS handshake fragment, the connection is passed on to
OpenSSL before being accepted. Otherwise the connection is assumed to be
unencrypted, i.e. plain HTTP.

However, this logic contains two errors:

  1. The timeout duration is too short - one second is too little
  2. When a timeout occurs, the connection is assumed to be unencrypted
 (and thus plain HTTP)

The patches 01 and 02 are mainly done in preparation for patch 03 (which
contains the actual fix), improving the overall quality of the code and
including the peer's address in error logs.

Please see the individual patches for more information.

Special thanks go to Stefan Hanreich whose advice helped identifying
many individual puzzle pieces comprising this issue.

References
--

[1]: https://bugzilla.proxmox.com/show_bug.cgi?id=5105

Summary of Changes
------

Max Carrara (3):
  rest-server: connection: clean up accept data flow
  rest-server: connection: log peer address on error
  fix #5105: rest-server: connection: overhaul TLS handshake check logic

 proxmox-rest-server/src/connection.rs | 165 +-
 1 file changed, 85 insertions(+), 80 deletions(-)

-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH storage] volume import: assume target API version is at least 9

[Max Carrara]

On Mon Jun 10, 2024 at 11:04 AM CEST, Fiona Ebner wrote:
> The storage API version has been bumped to at least 9 since
> libpve-storage = 7.0-4. If the source node is on Proxmox VE 8, where
> this change will come in, then the target node can be assumed to be
> running either Proxmox VE 8 or, during upgrade, the latest version of
> Proxmox VE 7.4, so it's safe to assume a storage API version of at
> least 9 in all cases.
>
> As reported by Maximiliano, the fact that the 'apiinfo' call was
> guarded with a quiet eval could lead to strange errors for replication
> on a customer system where an SSH connection could not always be
> established, because the target's API version would fall back to 1.
> Because of that, the '-base' argument would be missing for the import
> call on the target which would in turn lead to an error about the
> target ZFS volume already existing (rather than doing an incremental
> sync).
>
> Reported-by: Maximiliano Sandoval 
> Signed-off-by: Fiona Ebner 

Wasn't able to test this at the moment yet (please holler at me if you
need me to), but these changes are pretty straightforward anyway.

I completely agree with all of these changes; LGTM.

Reviewed-by: Max Carrara 

> ---
>  src/PVE/Storage.pm | 18 ++
>  1 file changed, 6 insertions(+), 12 deletions(-)
>
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index f19a115..bee2361 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -709,7 +709,7 @@ sub storage_migrate_snapshot {
>  }
>  
>  my $volume_import_prepare = sub {
> -my ($volid, $format, $path, $apiver, $opts) = @_;
> +my ($volid, $format, $path, $opts) = @_;
>  
>  my $base_snapshot = $opts->{base_snapshot};
>  my $snapshot = $opts->{snapshot};
> @@ -724,11 +724,11 @@ my $volume_import_prepare = sub {
>  if ($migration_snapshot) {
>   push @$recv, '-delete-snapshot', $snapshot;
>  }
> -push @$recv, '-allow-rename', $allow_rename if $apiver >= 5;
> +push @$recv, '-allow-rename', $allow_rename;
>  
>  if (defined($base_snapshot)) {
>   # Check if the snapshot exists on the remote side:
> - push @$recv, '-base', $base_snapshot if $apiver >= 9;
> + push @$recv, '-base', $base_snapshot;
>  }
>  
>  return $recv;
> @@ -814,12 +814,7 @@ sub storage_migrate {
>   $import_fn = "tcp://$net";
>  }
>  
> -my $target_apiver = 1; # if there is no apiinfo call, assume 1
> -my $get_api_version = [@$ssh, 'pvesm', 'apiinfo'];
> -my $match_api_version = sub { $target_apiver = $1 if $_[0] =~ m!^APIVER 
> (\d+)$!; };
> -eval { run_command($get_api_version, logfunc => $match_api_version); };
> -
> -my $recv = [ @$ssh, '--', $volume_import_prepare->($target_volid, 
> $format, $import_fn, $target_apiver, $opts)->@* ];
> +my $recv = [ @$ssh, '--', $volume_import_prepare->($target_volid, 
> $format, $import_fn, $opts)->@* ];
>  
>  my $new_volid;
>  my $pattern = volume_imported_message(undef, 1);
> @@ -911,8 +906,7 @@ sub storage_migrate {
>   run_command($cmds, logfunc => $match_volid_and_log);
>   }
>  
> - die "unable to get ID of the migrated volume\n"
> - if !defined($new_volid) && $target_apiver >= 5;
> + die "unable to get ID of the migrated volume\n" if !defined($new_volid);
>  };
>  my $err = $@;
>  if ($opts->{migration_snapshot}) {
> @@ -1961,7 +1955,7 @@ sub volume_import_start {
>  my $info = IO::File->new();
>  
>  my $unix = $opts->{unix} // "/run/pve/storage-migrate-$vmid.$$.unix";
> -my $import = $volume_import_prepare->($volid, $format, "unix://$unix", 
> APIVER, $opts);
> +my $import = $volume_import_prepare->($volid, $format, "unix://$unix", 
> $opts);
>  
>  unlink $unix;
>  my $cpid = open3($input, $info, $info, @$import)



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH installer 0/4] tui: make disk options view tabbed on small screens

[Max Carrara]

On Thu Jun 13, 2024 at 1:53 PM CEST, Christoph Heiss wrote:
> This adds a tabbed view component, for usage in the advanced disk
> options dialog when selecting ZFS or Btrfs. Works pretty much the same
> as its GUI counterpart, as much as that is possible.
>
> It's currently only activated for small (<=80 columns) displays, to make
> disk selection a lot more usable in these cases. This mostly affects
> serial console installation, but possibly also installations using a
> virtual screen via IPMI/BMC.
>
> Testing can be done using the `stty` to set specific terminal sizes,
> e.g. `stty columns 80 rows 24` for a standard VT100-spec terminal.
>
> This componont/view may also be made the default for the advanced disk
> options dialog, to align the TUI with it GUI in more cases - I'm open
> for discussion on that. Would also simplify the code a lot, so there
> are certainly other benefits to it as well.

Pretty neat! I like this a lot. See my entire feedback below.

Leaving these here already so they don't get lost ;)

Reviewed-by: Max Carrara 
Tested-by: Max Carrara 


Building


* Changes applied cleanly, no issues here
* Submitted code is formatted using `cargo fmt`, very nice
* `cargo clippy` also didn't complain, very nice

Code Review
---

All changes are easy to follow, no surprises here. There's nothing that
I have an issue with, pretty solid work! Also nice to see some comments
and docstrings that provide actual context, explaining what something is
for or why it is done. These kinds of things might seem small, but help
others that aren't as familiar with the installer code (like me, e.g.)
to grasp what's going on without having to "discover" everything
themselves. Very much appreciated!

Testing
---

* tty size: 60x15
  - tabs work as intended, but the actual contents of the tabs get lost
for ZFS and BTRFS configurations, because there's not enough
vertical space
  - it's (unsurprisingly) still possible to navigate through those
options, they're just not visible
  - my suggestion would be to add some kind of scrolling view here,
though I'm not sure if we're aiming to support ttys that are *this*
small

* tty size: 60x20
  - same as above, except that the first (four) configuration options
in the advanced disk config tab are displayed, the other ones are
still lost

* tty size: 80x24
  - works pretty great, as there now is enough vertical space to display
everything
  - haven't tested with multiple disks, but if a user wanted to e.g.
use ZFS with *a lot* of disks, there's still a risk that they might
run out of vertical space in the disk selection tab

* tty size: 81x24
  - feels awkward (as it does without this series :P) because the right
column in the advanced disk config tab gets squeezed (squoze?) too
much
  - as you suggested above, the tab view could therefore IMO be made the
default in general, it just looks really neat overall; I personally
prefer it quite a lot over the old one

Summary
---

Great work, great code, great results - can't really say more than that,
can I? As mentioned before, the tab view could IMO be the new default
for the disk stuff.

One thing that should perhaps still be addressed is the case of there
not being enough vertical space when the tty's height is rather small.
I recognize however that this isn't really in the scope of this series,
so unless someone else wants to chime in, this can be applied as-is.

Reviewed-by: Max Carrara 
Tested-by: Max Carrara 

>
> Christoph Heiss (4):
>   tui: fix some comment typos
>   tui: bootdisk: align btrfs dialog interface with zfs equivalent
>   tui: views: add new TabbedView component
>   tui: bootdisk: use tabbed view for disk options on small screens
>
>  proxmox-tui-installer/src/views/bootdisk.rs   | 260 +++---
>  proxmox-tui-installer/src/views/mod.rs|   3 +
>  .../src/views/tabbed_view.rs  | 196 +
>  3 files changed, 358 insertions(+), 101 deletions(-)
>  create mode 100644 proxmox-tui-installer/src/views/tabbed_view.rs



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH v2 pve-manager 00/10] Ceph Build Commit in UI

[Max Carrara]

On Mon Jul 1, 2024 at 4:10 PM CEST, Max Carrara wrote:
> Ceph Build Commit in UI - Version 2
> ===

Ah, forgot to note (again) that this fixes #5366 [0]. Mea culpa.

[0]: https://bugzilla.proxmox.com/show_bug.cgi?id=5366

>
> Notable Changes since v1
> 
>
>   * Use camelCase instead of snake_case for new functions / variables
> as per our style guide [0] (thanks Lukas!)
>   * Refrain from using `const` for things that aren't actual constants
> as per our style guide [1] (thanks Lukas!)
>   * NEW: Patch 09: Increase the default width of the version field in
> the OSD tree so that longer strings are immediately readable without
> needing to adjust the column widths manually
> --> e.g. "18.2.2 (e9fe820e7 -> 69ce99eba)" takes up a lot of space
> in the column
>   * NEW: Patch 10: Include Ceph build commit in the version string
> which is part of the object of the `ceph/osd/{osdid}/metadata` call
>
> For a detailed list of changes, please see the comments in the
> individual patches.
>
> NOTE: I added Lukas's T-b and R-b tags to all patches except the new
> ones, as mentioned in a reply to v1 [2].
>
> Older Versions
> --
>
> v1: https://lists.proxmox.com/pipermail/pve-devel/2024-April/063772.html
>
> References
> --
>
> [0]: https://pve.proxmox.com/wiki/Javascript_Style_Guide#Casing
> [1]: https://pve.proxmox.com/wiki/Javascript_Style_Guide#Variables
> [2]: https://lists.proxmox.com/pipermail/pve-devel/2024-June/064084.html
>
> Summary of Changes
> --
>
> Max Carrara (10):
>   ceph: tools: refactor installation check as guard clause
>   ceph: tools: parse Ceph version in separate sub and update regex
>   ceph: services: remove old cluster broadcast
>   ceph: services: refactor version existence check as guard clause
>   utils: align regex of parse_ceph_version with Perl equivalent
>   ui: ceph: services: parse and display build commit
>   api: ceph: add build commit of host to Ceph osd index endpoint data
>   ui: ceph: osd: rework rendering of version field & show build commit
>   ui: ceph: osd: increase width of version column
>   api: ceph: change version format in OSD metadata endpoint
>
>  PVE/API2/Ceph/OSD.pm |  9 -
>  PVE/Ceph/Services.pm | 38 ++--
>  PVE/Ceph/Tools.pm| 59 ++--
>  www/manager6/Utils.js| 17 -
>  www/manager6/ceph/OSD.js | 57 +-
>  www/manager6/ceph/ServiceList.js | 32 +
>  www/manager6/ceph/Services.js| 14 +++-
>  7 files changed, 170 insertions(+), 56 deletions(-)



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v2 pve-common 4/4] section config: make subroutine `delete_from_config` private

[Max Carrara]

because it's just an internal helper method and isn't used anywhere
outside of the package.

Signed-off-by: Max Carrara 
---
Changes v1 --> v2:
  * new

 src/PVE/SectionConfig.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/PVE/SectionConfig.pm b/src/PVE/SectionConfig.pm
index ec074fa..77640c9 100644
--- a/src/PVE/SectionConfig.pm
+++ b/src/PVE/SectionConfig.pm
@@ -1628,7 +1628,7 @@ from C<$config>.
 
 =cut
 
-sub delete_from_config {
+my sub delete_from_config {
 my ($config, $option_schema, $new_options, $to_delete) = @_;
 
 for my $k ($to_delete->@*) {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v2 pve-common 1/4] section config: document package and its methods with POD

[Max Carrara]

Apart from the obvious benefits that documentation has, this also
allows LSPs to provide docstrings e.g. via 'textDocument/hover' [0].

Tested with Perl Navigator [1].

[0]: 
https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_hover
[1]: https://github.com/bscan/PerlNavigator

Signed-off-by: Max Carrara 
---
Changes v1 --> v2:
  * basically all of @Fabian's feedback [0] was taken into account,
which is way too much to list here, so please see the feedback
itself
  * use more links where appropriate, e.g. to refer to methods
associated with a term
  * provide more example code (subroutines `private` and `decode_value`)

[0]: https://lists.proxmox.com/pipermail/pve-devel/2024-June/064165.html

 src/PVE/SectionConfig.pm | 953 ---
 1 file changed, 888 insertions(+), 65 deletions(-)

diff --git a/src/PVE/SectionConfig.pm b/src/PVE/SectionConfig.pm
index a18e9d8..f768eb2 100644
--- a/src/PVE/SectionConfig.pm
+++ b/src/PVE/SectionConfig.pm
@@ -10,65 +10,103 @@ use PVE::Exception qw(raise_param_exc);
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::Tools;
 
-# This package provides a way to have multiple (often similar) types of entries
-# in the same config file, each in its own section, thus "Section Config".
-#
-# The intended structure is to have a single 'base' plugin that inherits from
-# this class and provides meaningful defaults in its '$defaultData', e.g. a
-# default list of the core properties in its propertyList (most often only 'id'
-# and 'type')
-#
-# Each 'real' plugin then has it's own package that should inherit from the
-# 'base' plugin and returns it's specific properties in the 'properties' 
method,
-# its type in the 'type' method and all the known options, from both parent and
-# itself, in the 'options' method.
-# The options method can also be used to define if a property is 'optional' or
-# 'fixed' (only settable on config entity-creation), for example:
-#
-# 
-# sub options {
-# return {
-# 'some-optional-property' => { optional => 1 },
-# 'a-fixed-property' => { fixed => 1 },
-# 'a-required-but-not-fixed-property' => {},
-# };
-# }
-# ```
-#
-# 'fixed' options can be set on create, but not changed afterwards.
-#
-# To actually use it, you have to first register all the plugins and then init
-# the 'base' plugin, like so:
-#
-# ```
-# use PVE::Dummy::Plugin1;
-# use PVE::Dummy::Plugin2;
-# use PVE::Dummy::BasePlugin;
-#
-# PVE::Dummy::Plugin1->register();
-# PVE::Dummy::Plugin2->register();
-# PVE::Dummy::BasePlugin->init();
-# ```
-#
-# There are two modes for how properties are exposed, the default 'unified'
-# mode and the 'isolated' mode.
-# In the default unified mode, there is only a global list of properties
-# which the plugins can use, so you cannot define the same property name twice
-# in different plugins. The reason for this is to force the use of identical
-# properties for multiple plugins.
-#
-# The second way is to use the 'isolated' mode, which can be achieved by
-# calling init with `1` as its parameter like this:
-#
-# ```
-# PVE::Dummy::BasePlugin->init(property_isolation => 1);
-# ```
-#
-# With this, each plugin get's their own isolated list of properties which it
-# can use. Note that in this mode, you only have to specify the property in the
-# options method when it is either 'fixed' or comes from the global list of
-# properties. All locally defined ones get automatically added to the schema
-# for that plugin.
+=pod
+
+=head1 NAME
+
+SectionConfig
+
+=head1 DESCRIPTION
+
+This package provides a way to have multiple (often similar) types of entries
+in the same config file, each in its own section, thus I.
+
+For each C-based config file, a C is derived
+automatically. This schema can be used to implement CRUD operations for
+the config data.
+
+The location of a config file is chosen by the author of the code that uses
+C and is not something this module is concerned with.
+
+=head1 USAGE
+
+The intended structure is to have a single I that uses the
+C module as a base module. Furthermore, it should provide
+meaningful defaults in its C<$defaultData>, such as a default list of core
+C I. The I is thus very similar to an
+I.
+
+Each I is then defined in its own package that should inherit
+from the I and defines which I it itself provides and
+uses, as well as which I it uses from the I.
+
+The methods that need to be implemented are annotated in the L 
section
+below.
+
+  ┌─┐
+  │  SectionConfig  │
+  └┬┘
+   │
+   │
+ 

[pve-devel] [PATCH v2 pve-common 3/4] section config: clean up parser logic and semantics

[Max Carrara]

In order to make the parser somewhat more maintainable in the future,
this commit cleans up its logic and makes its control flow easier to
follow.

Furthermore, regexes are assigned to variables and some variables are
renamed in order to make the code more semantically expressive.

Signed-off-by: Max Carrara 
---
Changes v1 --> v2:
  * reword commit message
  * for errors, use reference of array instead of array directly
  * drop 'x' modifier from $re_kv_pair regex, as it doesn't have any
benefit

 src/PVE/SectionConfig.pm | 187 ---
 1 file changed, 97 insertions(+), 90 deletions(-)

diff --git a/src/PVE/SectionConfig.pm b/src/PVE/SectionConfig.pm
index a6cc468..ec074fa 100644
--- a/src/PVE/SectionConfig.pm
+++ b/src/PVE/SectionConfig.pm
@@ -1190,25 +1190,26 @@ The error.
 sub parse_config {
 my ($class, $filename, $raw, $allow_unknown) = @_;
 
-my $pdata = $class->private();
+if (!defined($raw)) {
+   return {
+   ids => {},
+   order => {},
+   digest => Digest::SHA::sha1_hex(''),
+   };
+}
+
+my $re_begins_with_comment = qr/^\s*#/;
+my $re_kv_pair = qr/^\s+(\S+)(\s+(.*\S))?\s*$/;
 
 my $ids = {};
 my $order = {};
-
-$raw = '' if !defined($raw);
-
 my $digest = Digest::SHA::sha1_hex($raw);
 
-my $pri = 1;
+my $current_section_no = 1;
+my $line_no = 0;
 
-my $lineno = 0;
 my @lines = split(/\n/, $raw);
-my $nextline = sub {
-   while (defined(my $line = shift @lines)) {
-   $lineno++;
-   return $line if ($line !~ /^\s*#/);
-   }
-};
+my $errors = [];
 
 my $is_array = sub {
my ($type, $key) = @_;
@@ -1219,105 +1220,111 @@ sub parse_config {
return $schema->{type} eq 'array';
 };
 
-my $errors = [];
-while (@lines) {
-   my $line = $nextline->();
+my $get_next_line = sub {
+   while (scalar(@lines)) {
+   my $line = shift(@lines);
+   $line_no++;
+
+   next if ($line =~ m/$re_begins_with_comment/);
+
+   return $line;
+   }
+
+   return undef;
+};
+
+my $skip_to_next_empty_line = sub {
+   while ($get_next_line->() ne '') {}
+};
+
+while (defined(my $line = $get_next_line->())) {
next if !$line;
 
-   my $errprefix = "file $filename line $lineno";
+   my $errprefix = "file $filename line $line_no";
 
-   my ($type, $sectionId, $errmsg, $config) = 
$class->parse_section_header($line);
-   if ($config) {
-   my $skip = 0;
-   my $unknown = 0;
+   my ($type, $section_id, $errmsg, $config) = 
$class->parse_section_header($line);
 
-   my $plugin;
+   if (!defined($config)) {
+   warn "$errprefix - ignore config line: $line\n";
+   next;
+   }
 
-   if ($errmsg) {
-   $skip = 1;
-   chomp $errmsg;
-   warn "$errprefix (skip section '$sectionId'): $errmsg\n";
-   } elsif (!$type) {
-   $skip = 1;
-   warn "$errprefix (skip section '$sectionId'): missing type - 
internal error\n";
-   } else {
-   if (!($plugin = $pdata->{plugins}->{$type})) {
-   if ($allow_unknown) {
-   $unknown = 1;
-   } else {
-   $skip = 1;
-   warn "$errprefix (skip section '$sectionId'): 
unsupported type '$type'\n";
-   }
-   }
-   }
+   if ($errmsg) {
+   chomp $errmsg;
+   warn "$errprefix (skip section '$section_id'): $errmsg\n";
+   $skip_to_next_empty_line->();
+   next;
+   }
+
+   if (!$type) {
+   warn "$errprefix (skip section '$section_id'): missing type - 
internal error\n";
+   $skip_to_next_empty_line->();
+   next;
+   }
+
+   my $plugin = eval { $class->lookup($type) };
+   my $is_unknown_type = defined($@) && $@ ne '';
+
+   if ($is_unknown_type && !$allow_unknown) {
+   warn "$errprefix (skip section '$section_id'): unsupported type 
'$type'\n";
+   $skip_to_next_empty_line->();
+   next;
+   }
 
-   while ($line = $nextline->()) {
-   next if $skip;
-
-   $errprefix = "file $filename line $lineno";
-
-   if ($line =~ m/^\s+(\S+)(\s+(.*\S))?\s*$/) {
-   my ($k, $v) = ($1, $3);
-
-   eval {
-   if ($unknown) {
-   if (!defined($config->{$k})) {
-   $config->{$k} = $v;
- 

[pve-devel] [PATCH v2 pve-common 2/4] section config: update code style

[Max Carrara]

Replace `foreach` with `for` and use postfix deref instead of block
(circumfix) dereference (`$foo->%*` instead of `%$foo`).

Furthermore, make `format_config_line` a private sub instead of
unnecessarily declaring it as an anonymous subroutine, which avoids
the `&$sub_ref(...)` syntax altogether.

Signed-off-by: Max Carrara 
---
Changes v1 --> v2:
  * none

 src/PVE/SectionConfig.pm | 62 
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/src/PVE/SectionConfig.pm b/src/PVE/SectionConfig.pm
index f768eb2..a6cc468 100644
--- a/src/PVE/SectionConfig.pm
+++ b/src/PVE/SectionConfig.pm
@@ -490,7 +490,7 @@ sub createSchema {
 my $props = $base || {};
 
 if (!$class->has_isolated_properties()) {
-   foreach my $p (keys %$propertyList) {
+   for my $p (keys $propertyList->%*) {
next if $skip_type && $p eq 'type';
 
if (!$propertyList->{$p}->{optional}) {
@@ -503,7 +503,7 @@ sub createSchema {
my $copts = $class->options();
$required = 0 if defined($copts->{$p}) && $copts->{$p}->{optional};
 
-   foreach my $t (keys %$plugins) {
+   for my $t (keys $plugins->%*) {
my $opts = $pdata->{options}->{$t} || {};
$required = 0 if !defined($opts->{$p}) || 
$opts->{$p}->{optional};
}
@@ -518,7 +518,7 @@ sub createSchema {
}
}
 } else {
-   for my $type (sort keys %$plugins) {
+   for my $type (sort keys $plugins->%*) {
my $opts = $pdata->{options}->{$type} || {};
for my $key (sort keys $opts->%*) {
my $schema = $class->get_property_schema($type, $key);
@@ -596,7 +596,7 @@ sub updateSchema {
 my $filter_type = $single_class ? $class->type() : undef;
 
 if (!$class->has_isolated_properties()) {
-   foreach my $p (keys %$propertyList) {
+   for my $p (keys $propertyList->%*) {
next if $p eq 'type';
 
my $copts = $class->options();
@@ -612,7 +612,7 @@ sub updateSchema {
 
$modifyable = 1 if defined($copts->{$p}) && !$copts->{$p}->{fixed};
 
-   foreach my $t (keys %$plugins) {
+   for my $t (keys $plugins->%*) {
my $opts = $pdata->{options}->{$t} || {};
next if !defined($opts->{$p});
$modifyable = 1 if !$opts->{$p}->{fixed};
@@ -622,7 +622,7 @@ sub updateSchema {
$props->{$p} = $propertyList->{$p};
}
 } else {
-   for my $type (sort keys %$plugins) {
+   for my $type (sort keys $plugins->%*) {
my $opts = $pdata->{options}->{$type} || {};
for my $key (sort keys $opts->%*) {
next if $opts->{$key}->{fixed};
@@ -691,7 +691,7 @@ sub init {
 
 my $pdata = $class->private();
 
-foreach my $k (qw(options plugins plugindata propertyList 
isolatedPropertyList)) {
+for my $k (qw(options plugins plugindata propertyList 
isolatedPropertyList)) {
$pdata->{$k} = {} if !$pdata->{$k};
 }
 
@@ -699,9 +699,9 @@ sub init {
 my $propertyList = $pdata->{propertyList};
 my $isolatedPropertyList = $pdata->{isolatedPropertyList};
 
-foreach my $type (keys %$plugins) {
+for my $type (keys $plugins->%*) {
my $props = $plugins->{$type}->properties();
-   foreach my $p (keys %$props) {
+   for my $p (keys $props->%*) {
my $res;
if ($property_isolation) {
$res = $isolatedPropertyList->{$type}->{$p} = {};
@@ -710,16 +710,16 @@ sub init {
$res = $propertyList->{$p} = {};
}
my $data = $props->{$p};
-   for my $a (keys %$data) {
+   for my $a (keys $data->%*) {
$res->{$a} = $data->{$a};
}
$res->{optional} = 1;
}
 }
 
-foreach my $type (keys %$plugins) {
+for my $type (keys $plugins->%*) {
my $opts = $plugins->{$type}->options();
-   foreach my $p (keys %$opts) {
+   for my $p (keys $opts->%*) {
my $prop;
if ($property_isolation) {
$prop = $isolatedPropertyList->{$type}->{$p};
@@ -730,7 +730,7 @@ sub init {
 
# automatically the properties to options (if not specified explicitly)
if ($property_isolation) {
-   foreach my $p (keys $isolatedPropertyList->{$type}->%*) {
+   for my $p (keys $isolatedPropertyList->{$type}->%*) {
next if $opts->{$p};
$opts->{$p} = {};
$opts->{$p}->{optional} = 1 if 
$isolatedPropertyList->{$type}->{$p}->{optional};
@@ -741,7 +741,7 @@ sub init {
 }
 
 $propertyList->{type}->{ty

[pve-devel] [PATCH v2 pve-common 0/4] Section Config: Documentation & Code Cleanup

[Max Carrara]

Section Config: Documentation & Code Cleanup - v2
=

Notable Changes Since v1


  * Incorporate @Fabian's feedback regarding the docs [0]
  * Reword commit message of patch 03
  * NEW: Patch 04: Make sub `delete_from_config` private

Please see the comments in the individual patches for a detailed list of
changes.

Older Versions
--

v1: https://lists.proxmox.com/pipermail/pve-devel/2024-June/064062.html

References
--

[0]: https://lists.proxmox.com/pipermail/pve-devel/2024-June/064165.html

Summary of Changes
----------

Max Carrara (4):
  section config: document package and its methods with POD
  section config: update code style
  section config: clean up parser logic
  section config: make subroutine `delete_from_config` private

 src/PVE/SectionConfig.pm | 1200 --
 1 file changed, 1015 insertions(+), 185 deletions(-)

-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v2 pve-manager 09/10] ui: ceph: osd: increase width of version column

[Max Carrara]

.. so that the Ceph build commit as well as differing build commits
are shown properly.

Signed-off-by: Max Carrara 
---
Changes v1 --> v2:
  * NEW

 www/manager6/ceph/OSD.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/www/manager6/ceph/OSD.js b/www/manager6/ceph/OSD.js
index 37fe6a9c..474e4137 100644
--- a/www/manager6/ceph/OSD.js
+++ b/www/manager6/ceph/OSD.js
@@ -811,6 +811,7 @@ Ext.define('PVE.node.CephOsdTree', {
dataIndex: 'version',
align: 'right',
renderer: 'render_version',
+   width: 240,
},
{
text: 'weight',
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH v2 pve-manager 08/10] ui: ceph: osd: rework rendering of version field & show build commit

[Max Carrara]

The logic of the `render_version` function is split up in order to
handle how the version is displayed depending on the type of the row.

If the parsed version is `undefined` or the row marks the beginning of
the tree, an empty string is now returned. This behaviour is
equivalent to before, it just makes the overall logic easier.

If the row belongs to a node, the build commit is now additionally
displayed in parentheses next to the installed Ceph version:

  18.2.2 (abcd1234)

If the row belongs to an osd, the build commit is also additionally
displayed in parentheses next to the installed Ceph version.
Furthermore, should the build commit of the running osd differ from
the one that's installed on the host, the new hash will also be shown
in parentheses:

  18.2.2 (abcd1234 -> 5678fedc)

The icon displayed for running an osd with an outdated build is the
same as for running an outdated version. The conditional display of
cluster health-related icons remains the same otherwise.

Signed-off-by: Max Carrara 
Tested-by: Lukas Wagner 
Reviewed-by: Lukas Wagner 
---
Changes v1 --> v2:
  * use camelCase instead of snake_case (thanks Lukas!)
  * use more descriptive variable names (thanks Lukas!)
  * use `let` instead of `const` for variables where applicable (thanks Lukas!)

 www/manager6/ceph/OSD.js | 56 +---
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/www/manager6/ceph/OSD.js b/www/manager6/ceph/OSD.js
index d2caafa4..37fe6a9c 100644
--- a/www/manager6/ceph/OSD.js
+++ b/www/manager6/ceph/OSD.js
@@ -642,23 +642,61 @@ Ext.define('PVE.node.CephOsdTree', {
},
 
render_version: function(value, metadata, rec) {
+   if (value === undefined || rec.data.type === 'root') {
+   return '';
+   }
+
let vm = this.getViewModel();
-   let versions = vm.get('versions');
let icon = "";
-   let version = value || "";
let maxversion = vm.get('maxversion');
-   if (value && PVE.Utils.compare_ceph_versions(value, maxversion) !== 
0) {
-   let host_version = rec.parentNode?.data?.version || 
versions[rec.data.host] || "";
-   if (rec.data.type === 'host' || 
PVE.Utils.compare_ceph_versions(host_version, maxversion) !== 0) {
+   let mixedversions = vm.get('mixedversions');
+
+   if (rec.data.type === 'host') {
+   let buildCommit = rec.data.buildcommit ?? '';
+
+   if (PVE.Utils.compare_ceph_versions(maxversion, value) > 0) {
icon = PVE.Utils.get_ceph_icon_html('HEALTH_UPGRADE');
-   } else {
-   icon = PVE.Utils.get_ceph_icon_html('HEALTH_OLD');
+   } else if (mixedversions) {
+   icon = PVE.Utils.get_ceph_icon_html('HEALTH_OK');
}
-   } else if (value && vm.get('mixedversions')) {
+
+   if (buildCommit === '') {
+   return `${icon}${value}`;
+   }
+
+   return `${icon}${value} (${buildCommit.substring(0, 9)})`;
+   }
+
+   let versionNode = rec.parentNode?.data?.version ?? '';
+
+   let buildCommit = PVE.Utils.parseCephBuildCommit(rec.data) ?? '';
+   let buildCommitNode = rec.parentNode?.data?.buildcommit ?? '';
+
+   let bcChanged =
+   buildCommit !== '' &&
+   buildCommitNode !== '' &&
+   buildCommit !== buildCommitNode;
+
+   if (PVE.Utils.compare_ceph_versions(maxversion, value) > 0) {
+   icon = PVE.Utils.get_ceph_icon_html('HEALTH_UPGRADE');
+   } else if (versionNode && 
PVE.Utils.compare_ceph_versions(versionNode, value) > 0) {
+   icon = PVE.Utils.get_ceph_icon_html('HEALTH_OLD');
+   } else if (mixedversions && !bcChanged) {
icon = PVE.Utils.get_ceph_icon_html('HEALTH_OK');
}
 
-   return icon + version;
+   let buildCommitPart = buildCommit.substring(0, 9);
+   if (bcChanged) {
+   const arrow = '';
+   icon ||= PVE.Utils.get_ceph_icon_html('HEALTH_OLD');
+   buildCommitPart = `${buildCommit.substring(0, 
9)}${arrow}${buildCommitNode.substring(0, 9)}`;
+   }
+
+   if (buildCommitPart === '') {
+   return `${icon}${value}`;
+   }
+
+   return `${icon}${value} (${buildCommitPart})`;
},
 
render_osd_val: function(value, metaData, rec) {
-- 
2.39.2



___
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel