--- Begin Message ---
Dear all,
Would you be so kind and help with problem below :
Dear all
Trying to add 2 news disk to system. Immediately ZFS pool disappear,
cause added new disks /sda and /sdb and "old" ones getting /sdc and /sdb
Then new disks removed - pool restored, but all catalogs with data
invisible now. According to zfs list all information still exist, but
not available.
So, question - any advise how to full restore ZFS pool and get access to data?
Thanks a lot
Dmitry
root@pve:~# pverort
-bash: pverort: command not found
root@pve:~# pveroport
-bash: pveroport: command not found
root@pve:~# pvereport
Process hostname...OK
Process pveversion --verbose...OK
Process cat /etc/hosts...OK
Process top -b -n 1 | head -n 15...OK
Process pvesubscription get...OK
Process lscpu...OK
Process pvesh get /cluster/resources --type node --output-format=yaml...OK
Process cat /etc/pve/storage.cfg...OK
Process pvesm status...OK
Process cat /etc/fstab...OK
Process findmnt --ascii...OK
Process df --human...OK
Process qm list...OK
OK
Process pct list...OK
OK
Process ip -details -statistics address...OK
Process ip -details -4 route show...OK
Process ip -details -6 route show...OK
Process cat /etc/network/interfaces...OK
OK
Process cat /etc/pve/local/host.fw...OK
Process iptables-save...OK
Process pvecm nodes...OK
Process pvecm status...OK
Process cat /etc/pve/corosync.conf 2>/dev/null...OK
Process dmidecode -t bios...OK
Process lspci -nnk...OK
Process lsblk --ascii...OK
Process ls -l /dev/disk/by-*/...OK
Process iscsiadm -m node...OK
Process iscsiadm -m session...OK
Process pvs...OK
Process lvs...OK
Process vgs...OK
Process zpool status...OK
Process zpool list -v...OK
Process zfs list...OK
==== general system info ====
# hostname
pve
# pveversion --verbose
proxmox-ve: 6.2-1 (running kernel: 5.4.60-1-pve)
pve-manager: 6.2-11 (running version: 6.2-11/22fb4983)
pve-kernel-5.4: 6.2-6
pve-kernel-helper: 6.2-6
pve-kernel-5.3: 6.1-6
pve-kernel-5.4.60-1-pve: 5.4.60-2
pve-kernel-5.4.55-1-pve: 5.4.55-1
pve-kernel-5.4.44-2-pve: 5.4.44-2
pve-kernel-5.4.44-1-pve: 5.4.44-1
pve-kernel-5.4.41-1-pve: 5.4.41-1
pve-kernel-5.3.18-3-pve: 5.3.18-3
pve-kernel-5.3.10-1-pve: 5.3.10-1
ceph-fuse: 12.2.11+dfsg1-2.1+b1
corosync: 3.0.4-pve1
criu: 3.11-3
glusterfs-client: 5.5-3
ifupdown: residual config
ifupdown2: 3.0.0-1+pve2
ksm-control-daemon: 1.3-1
libjs-extjs: 6.0.1-10
libknet1: 1.16-pve1
libproxmox-acme-perl: 1.0.5
libpve-access-control: 6.1-2
libpve-apiclient-perl: 3.0-3
libpve-common-perl: 6.2-2
libpve-guest-common-perl: 3.1-3
libpve-http-server-perl: 3.0-6
libpve-storage-perl: 6.2-6
libqb0: 1.0.5-1
libspice-server1: 0.14.2-4~pve6+1
lvm2: 2.03.02-pve4
lxc-pve: 4.0.3-1
lxcfs: 4.0.3-pve3
novnc-pve: 1.1.0-1
proxmox-mini-journalreader: 1.1-1
proxmox-widget-toolkit: 2.2-12
pve-cluster: 6.1-8
pve-container: 3.2-1
pve-docs: 6.2-5
pve-edk2-firmware: 2.20200531-1
pve-firewall: 4.1-2
pve-firmware: 3.1-3
pve-ha-manager: 3.1-1
pve-i18n: 2.2-1
pve-qemu-kvm: 5.1.0-2
pve-xtermjs: 4.7.0-2
qemu-server: 6.2-14
smartmontools: 7.1-pve2
spiceterm: 3.1-1
vncterm: 1.6-2
zfsutils-linux: 0.8.4-pve1
# cat /etc/hosts
127.0.0.1 localhost.localdomain localhost
192.168.89.41 pve.nufnufs.sknt.ru pve
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
# top -b -n 1 | head -n 15
top - 10:28:31 up 20:38, 1 user, load average: 0.09, 0.06, 0.02
Tasks: 310 total, 1 running, 309 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.0 us, 1.6 sy, 0.0 ni, 96.9 id, 0.0 wa, 0.0 hi, 1.6 si, 0.0 st
MiB Mem : 19895.3 total, 17503.9 free, 1339.5 used, 1051.9 buff/cache
MiB Swap: 8192.0 total, 8192.0 free, 0.0 used. 18095.8 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1283 root rt 0 242668 173920 50424 S 6.2 0.9 12:49.54 corosync
2283 100998 20 0 1827788 22632 17148 S 6.2 0.1 2:19.45 Plex DLNA+
1 root 20 0 104480 10808 7948 S 0.0 0.1 0:03.99 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:00.04 kthreadd
3 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_gp
4 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_par_gp
6 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kworker/0+
9 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 mm_percpu+
# pvesubscription get
message: There is no subscription key
serverid: 9B420580CF7193988A95EB18636252B0
status: NotFound
url: https://www.proxmox.com/proxmox-ve/pricing
# lscpu
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
Address sizes: 39 bits physical, 48 bits virtual
CPU(s): 4
On-line CPU(s) list: 0-3
Thread(s) per core: 2
Core(s) per socket: 2
Socket(s): 1
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 94
Model name: Intel(R) Core(TM) i3-6300 CPU @ 3.80GHz
Stepping: 3
CPU MHz: 3800.278
CPU max MHz: 3800.0000
CPU min MHz: 800.0000
BogoMIPS: 7599.80
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 4096K
NUMA node0 CPU(s): 0-3
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca
cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx
pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl
xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx
est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt
tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch
cpuid_fault invpcid_single pti ibrs ibpb stibp tpr_shadow vnmi flexpriority ept
vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx
smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm arat pln pts hwp
hwp_notify hwp_act_window hwp_epp
# pvesh get /cluster/resources --type node --output-format=yaml
---
-
cpu: 0.00957006187987872
disk: 5713743872
id: node/pve
level: ''
maxcpu: 4
maxdisk: 31044079616
maxmem: 20861734912
mem: 1529757696
node: pve
status: online
type: node
uptime: 74292
==== info about storage ====
# cat /etc/pve/storage.cfg
dir: local
path /var/lib/vz
content backup,vztmpl,iso
lvmthin: local-lvm
thinpool data
vgname pve
content rootdir,images
dir: Everything
path /nnpoolz/home
content images,iso
shared 1
dir: Download
path /nnpoolz/home/download
content images
shared 1
dir: Incomplete_torrent
path /nnpoolz/home/download/incomplete
content images
shared 1
dir: Dima_phone
path /nnpoolz/home/D
content images
shared 0
# pvesm status
Name Type Status Total Used
Available %
Dima_phone dir active 30316484 5579840
23173612 18.41%
Download dir active 30316484 5579840
23173612 18.41%
Everything dir active 30316484 5579840
23173612 18.41%
Incomplete_torrent dir active 30316484 5579840
23173612 18.41%
local dir active 30316484 5579840
23173612 18.41%
local-lvm lvmthin active 67620864 19204325
48416538 28.40%
# cat /etc/fstab
# <file system> <mount point> <type> <options> <dump> <pass>
/dev/pve/root / ext4 errors=remount-ro 0 1
/dev/pve/swap none swap sw 0 0
proc /proc proc defaults 0 0
# findmnt --ascii
TARGET SOURCE FSTYPE OPTIONS
/ /dev/mapper/pve-root ext4
rw,relatime,errors=remount-ro
|-/sys sysfs sysfs
rw,nosuid,nodev,noexec,relatime
| |-/sys/kernel/security securityfs securityfs
rw,nosuid,nodev,noexec,relatime
| |-/sys/fs/cgroup tmpfs tmpfs
ro,nosuid,nodev,noexec,mode=755
| | |-/sys/fs/cgroup/unified cgroup2 cgroup2
rw,nosuid,nodev,noexec,relatime
| | |-/sys/fs/cgroup/systemd cgroup cgroup
rw,nosuid,nodev,noexec,relatime,xattr,name=systemd
| | |-/sys/fs/cgroup/perf_event cgroup cgroup
rw,nosuid,nodev,noexec,relatime,perf_event
| | |-/sys/fs/cgroup/net_cls,net_prio cgroup cgroup
rw,nosuid,nodev,noexec,relatime,net_cls,net_prio
| | |-/sys/fs/cgroup/memory cgroup cgroup
rw,nosuid,nodev,noexec,relatime,memory
| | |-/sys/fs/cgroup/rdma cgroup cgroup
rw,nosuid,nodev,noexec,relatime,rdma
| | |-/sys/fs/cgroup/hugetlb cgroup cgroup
rw,nosuid,nodev,noexec,relatime,hugetlb
| | |-/sys/fs/cgroup/cpu,cpuacct cgroup cgroup
rw,nosuid,nodev,noexec,relatime,cpu,cpuacct
| | |-/sys/fs/cgroup/pids cgroup cgroup
rw,nosuid,nodev,noexec,relatime,pids
| | |-/sys/fs/cgroup/freezer cgroup cgroup
rw,nosuid,nodev,noexec,relatime,freezer
| | |-/sys/fs/cgroup/cpuset cgroup cgroup
rw,nosuid,nodev,noexec,relatime,cpuset
| | |-/sys/fs/cgroup/blkio cgroup cgroup
rw,nosuid,nodev,noexec,relatime,blkio
| | `-/sys/fs/cgroup/devices cgroup cgroup
rw,nosuid,nodev,noexec,relatime,devices
| |-/sys/fs/pstore pstore pstore
rw,nosuid,nodev,noexec,relatime
| |-/sys/firmware/efi/efivars efivarfs efivarfs
rw,nosuid,nodev,noexec,relatime
| |-/sys/fs/bpf none bpf
rw,nosuid,nodev,noexec,relatime,mode=700
| |-/sys/kernel/debug debugfs debugfs
rw,relatime
| |-/sys/kernel/config configfs configfs
rw,relatime
| `-/sys/fs/fuse/connections fusectl fusectl
rw,relatime
|-/proc proc proc
rw,relatime
| `-/proc/sys/fs/binfmt_misc systemd-1 autofs
rw,relatime,fd=30,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=17726
| `-/proc/sys/fs/binfmt_misc binfmt_misc binfmt_misc
rw,relatime
|-/dev udev devtmpfs
rw,nosuid,relatime,size=10161616k,nr_inodes=2540404,mode=755
| |-/dev/pts devpts devpts
rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000
| |-/dev/shm tmpfs tmpfs
rw,nosuid,nodev
| |-/dev/mqueue mqueue mqueue
rw,relatime
| `-/dev/hugepages hugetlbfs hugetlbfs
rw,relatime,pagesize=2M
|-/run tmpfs tmpfs
rw,nosuid,noexec,relatime,size=2037280k,mode=755
| |-/run/lock tmpfs tmpfs
rw,nosuid,nodev,noexec,relatime,size=5120k
| |-/run/rpc_pipefs sunrpc rpc_pipefs
rw,relatime
| `-/run/user/0 tmpfs tmpfs
rw,nosuid,nodev,relatime,size=2037276k,mode=700
|-/var/lib/lxcfs lxcfs fuse.lxcfs
rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other
`-/etc/pve /dev/fuse fuse
rw,nosuid,nodev,relatime,user_id=0,group_id=0,default_permissions,allow_other
# df --human
Filesystem Size Used Avail Use% Mounted on
udev 9.7G 0 9.7G 0% /dev
tmpfs 2.0G 18M 2.0G 1% /run
/dev/mapper/pve-root 29G 5.4G 23G 20% /
tmpfs 9.8G 63M 9.7G 1% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 9.8G 0 9.8G 0% /sys/fs/cgroup
/dev/fuse 30M 24K 30M 1% /etc/pve
tmpfs 2.0G 0 2.0G 0% /run/user/0
==== info about virtual guests ====
# qm list
VMID NAME STATUS MEM(MB) BOOTDISK(GB) PID
107 Debian stopped 4096 32.00 0
# cat /etc/pve/qemu-server/107.conf
bootdisk: scsi0
cores: 1
ide2: Everything:iso/debian-live-10.5.0-amd64-standard.iso,media=cdrom
kvm: 0
memory: 4096
name: Debian
net0: virtio=CA:30:38:6E:2F:AD,bridge=vmbr0
numa: 0
ostype: l26
scsi0: Everything:107/vm-107-disk-0.qcow2,discard=on,size=32G
scsihw: virtio-scsi-pci
smbios1: uuid=7c3cf693-4282-48c2-a2ef-d2c17cb7a7c8
sockets: 1
vmgenid: 8c0500e6-4a2d-4121-9958-51c90c950eca
# pct list
VMID Status Lock Name
100 stopped DNS
101 stopped nginx
102 running Transmission
103 running Plex
104 stopped TFTP
105 stopped Syncthing
106 running Samba
# cat /etc/pve/lxc/100.conf
arch: amd64
cores: 2
hostname: DNS
memory: 2048
nameserver: 192.168.89.2
net0:
name=eth0,bridge=vmbr0,firewall=1,hwaddr=42:DC:41:A0:7F:50,ip=dhcp,type=veth
ostype: debian
rootfs: local-lvm:vm-100-disk-0,size=4G
searchdomain: dns.nufnufs.sknt.ru
swap: 512
unprivileged: 1
# cat /etc/pve/lxc/101.conf
arch: amd64
cores: 1
hostname: nginx
memory: 1024
nameserver: 192.168.89.2
net0:
name=eth0,bridge=vmbr0,firewall=1,hwaddr=9A:97:D8:8F:E9:67,ip=dhcp,type=veth
ostype: debian
rootfs: local-lvm:vm-101-disk-0,size=8G
searchdomain: nginx.nufnufs.sknt.ru
swap: 1024
unprivileged: 1
# cat /etc/pve/lxc/102.conf
arch: amd64
cores: 1
hostname: Transmission
memory: 2048
mp0: /nnpoolz/home,mp=/mnt/home
nameserver: 192.168.89.2
net0:
name=eth0,bridge=vmbr0,firewall=1,hwaddr=EA:D6:44:92:C0:63,ip=dhcp,type=veth
onboot: 1
ostype: debian
rootfs: local-lvm:vm-102-disk-0,size=12G
searchdomain: transmission.nufsnufs.sknt.ru
swap: 2048
unprivileged: 1
# cat /etc/pve/lxc/103.conf
arch: amd64
cores: 4
hostname: Plex
memory: 2048
mp0: /nnpoolz/home,mp=/mnt/home
nameserver: 192.168.89.2
net0: name=eth0,bridge=vmbr0,hwaddr=5A:F5:38:44:44:17,ip=dhcp,type=veth
onboot: 1
ostype: debian
rootfs: local-lvm:vm-103-disk-0,size=12G
searchdomain: plex.nufnufs.sknt.ru
swap: 2048
unprivileged: 1
# cat /etc/pve/lxc/104.conf
arch: amd64
cores: 1
hostname: TFTP
memory: 512
nameserver: 192.168.89.2
net0: name=eth0,bridge=vmbr0,hwaddr=AA:0B:1C:DE:71:75,ip=dhcp,type=veth
ostype: debian
rootfs: local-lvm:vm-104-disk-0,size=6G
searchdomain: tftp.nufnufs.sknt.ru
swap: 512
unprivileged: 1
# cat /etc/pve/lxc/105.conf
arch: amd64
cores: 1
hostname: Syncthing
memory: 1024
mp0: /nnpoolz/home,mp=/mnt/home
nameserver: 192.168.89.19
net0:
name=eth0,bridge=vmbr0,firewall=1,hwaddr=A2:E1:D4:7F:B2:07,ip=dhcp,type=veth
ostype: debian
rootfs: local-lvm:vm-105-disk-0,size=8G
searchdomain: sync.nufnufs.sknt.ru
swap: 512
# cat /etc/pve/lxc/106.conf
arch: amd64
cores: 2
hostname: Samba
memory: 512
mp0: /nnpoolz/home,mp=/mnt/home
net0:
name=eth0,bridge=vmbr0,firewall=1,hwaddr=A6:B3:4F:6C:15:3C,ip=dhcp,type=veth
onboot: 1
ostype: debian
rootfs: local-lvm:vm-106-disk-0,size=8G
swap: 512
==== info about network ====
# ip -details -statistics address
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group
default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 minmtu
0 maxmtu 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
RX: bytes packets errors dropped overrun mcast
178312 2148 0 0 0 0
TX: bytes packets errors dropped carrier collsns
178312 2148 0 0 0 0
2: enp1s0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc pfifo_fast master
vmbr0 state DOWN group default qlen 1000
link/ether f8:d1:11:02:a1:cc brd ff:ff:ff:ff:ff:ff promiscuity 1 minmtu 60
maxmtu 4080
bridge_slave state disabled priority 32 cost 100 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8001 port_no 0x1
designated_port 32769 designated_cost 0 designated_bridge 8000.f8:d1:11:2:a1:cc
designated_root 8000.f8:d1:11:2:a1:cc hold_timer 0.00 message_age_timer
0.00 forward_delay_timer 0.00 topology_change_ack 0 config_pending 0
proxy_arp off proxy_arp_wifi off mcast_router 1 mcast_fast_leave off
mcast_flood on neigh_suppress off group_fwd_mask 0 group_fwd_mask_str 0x0
vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1 gso_max_size 32000
gso_max_segs 24
RX: bytes packets errors dropped overrun mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
3: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master
vmbr0 state UP group default qlen 1000
link/ether b0:6e:bf:38:21:e1 brd ff:ff:ff:ff:ff:ff promiscuity 1 minmtu 60
maxmtu 9200
bridge_slave state forwarding priority 32 cost 4 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8002 port_no 0x2
designated_port 32770 designated_cost 0 designated_bridge 8000.f8:d1:11:2:a1:cc
designated_root 8000.f8:d1:11:2:a1:cc hold_timer 0.00 message_age_timer
0.00 forward_delay_timer 0.00 topology_change_ack 0 config_pending 0
proxy_arp off proxy_arp_wifi off mcast_router 1 mcast_fast_leave off
mcast_flood on neigh_suppress off group_fwd_mask 0 group_fwd_mask_str 0x0
vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1 gso_max_size 64000
gso_max_segs 64
RX: bytes packets errors dropped overrun mcast
81574201 496117 0 2509 0 46933
TX: bytes packets errors dropped carrier collsns
24718416 186115 0 0 0 0
4: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
group default qlen 1000
link/ether f8:d1:11:02:a1:cc brd ff:ff:ff:ff:ff:ff promiscuity 0 minmtu 68
maxmtu 65535
bridge forward_delay 0 hello_time 200 max_age 2000 ageing_time 30000
stp_state 0 priority 32768 vlan_filtering 0 vlan_protocol 802.1Q bridge_id
8000.f8:d1:11:2:a1:cc designated_root 8000.f8:d1:11:2:a1:cc root_port 0
root_path_cost 0 topology_change 0 topology_change_detected 0 hello_timer
0.00 tcn_timer 0.00 topology_change_timer 0.00 gc_timer 163.84
vlan_default_pvid 1 vlan_stats_enabled 0 group_fwd_mask 0 group_address
01:80:c2:00:00:00 mcast_snooping 1 mcast_router 1 mcast_query_use_ifaddr 0
mcast_querier 0 mcast_hash_elasticity 16 mcast_hash_max 4096
mcast_last_member_count 2 mcast_startup_query_count 2
mcast_last_member_interval 100 mcast_membership_interval 26000
mcast_querier_interval 25500 mcast_query_interval 12500
mcast_query_response_interval 1000 mcast_startup_query_interval 3124
mcast_stats_enabled 0 mcast_igmp_version 2 mcast_mld_version 1 nf_call_iptables
0 nf_call_ip6tables 0 nf_call_arptables 0 numtxqueues 1 numrxqueues 1
gso_max_size 32000 gso_max_segs 24
inet 192.168.89.41/25 scope global vmbr0
valid_lft forever preferred_lft forever
inet6 fe80::fad1:11ff:fe02:a1cc/64 scope link
valid_lft forever preferred_lft forever
RX: bytes packets errors dropped overrun mcast
69716482 441778 0 5 0 0
TX: bytes packets errors dropped carrier collsns
9446299 39074 0 0 0 0
5: veth102i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue
master fwbr102i0 state UP group default qlen 1000
link/ether fe:3c:10:c2:e9:51 brd ff:ff:ff:ff:ff:ff link-netnsid 0
promiscuity 1 minmtu 68 maxmtu 65535
veth
bridge_slave state forwarding priority 32 cost 2 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8002 port_no 0x2
designated_port 32770 designated_cost 0 designated_bridge
8000.2e:fe:5b:9c:82:d4 designated_root 8000.2e:fe:5b:9c:82:d4 hold_timer
0.00 message_age_timer 0.00 forward_delay_timer 0.00 topology_change_ack
0 config_pending 0 proxy_arp off proxy_arp_wifi off mcast_router 1
mcast_fast_leave off mcast_flood on neigh_suppress off group_fwd_mask 0
group_fwd_mask_str 0x0 vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1
gso_max_size 65536 gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
5910512 59510 0 0 0 0
TX: bytes packets errors dropped carrier collsns
74540509 448111 0 0 0 0
6: fwbr102i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
group default qlen 1000
link/ether 2e:fe:5b:9c:82:d4 brd ff:ff:ff:ff:ff:ff promiscuity 0 minmtu 68
maxmtu 65535
bridge forward_delay 0 hello_time 200 max_age 2000 ageing_time 30000
stp_state 0 priority 32768 vlan_filtering 0 vlan_protocol 802.1Q bridge_id
8000.2e:fe:5b:9c:82:d4 designated_root 8000.2e:fe:5b:9c:82:d4 root_port 0
root_path_cost 0 topology_change 0 topology_change_detected 0 hello_timer
0.00 tcn_timer 0.00 topology_change_timer 0.00 gc_timer 163.84
vlan_default_pvid 1 vlan_stats_enabled 0 group_fwd_mask 0 group_address
01:80:c2:00:00:00 mcast_snooping 1 mcast_router 1 mcast_query_use_ifaddr 0
mcast_querier 0 mcast_hash_elasticity 16 mcast_hash_max 4096
mcast_last_member_count 2 mcast_startup_query_count 2
mcast_last_member_interval 100 mcast_membership_interval 26000
mcast_querier_interval 25500 mcast_query_interval 12500
mcast_query_response_interval 1000 mcast_startup_query_interval 3124
mcast_stats_enabled 0 mcast_igmp_version 2 mcast_mld_version 1 nf_call_iptables
0 nf_call_ip6tables 0 nf_call_arptables 0 numtxqueues 1 numrxqueues 1
gso_max_size 65536 gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
63439926 396269 0 5 0 0
TX: bytes packets errors dropped carrier collsns
108 2 0 0 0 0
7: fwpr102p0@fwln102i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
noqueue master vmbr0 state UP group default qlen 1000
link/ether 26:94:10:9b:3d:1b brd ff:ff:ff:ff:ff:ff promiscuity 1 minmtu 68
maxmtu 65535
veth
bridge_slave state forwarding priority 32 cost 2 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8003 port_no 0x3
designated_port 32771 designated_cost 0 designated_bridge 8000.f8:d1:11:2:a1:cc
designated_root 8000.f8:d1:11:2:a1:cc hold_timer 0.00 message_age_timer
0.00 forward_delay_timer 0.00 topology_change_ack 0 config_pending 0
proxy_arp off proxy_arp_wifi off mcast_router 1 mcast_fast_leave off
mcast_flood on neigh_suppress off group_fwd_mask 0 group_fwd_mask_str 0x0
vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1 gso_max_size 65536
gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
5910566 59511 0 0 0 0
TX: bytes packets errors dropped carrier collsns
74540882 448114 0 0 0 0
8: fwln102i0@fwpr102p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
noqueue master fwbr102i0 state UP group default qlen 1000
link/ether 2e:fe:5b:9c:82:d4 brd ff:ff:ff:ff:ff:ff promiscuity 1 minmtu 68
maxmtu 65535
veth
bridge_slave state forwarding priority 32 cost 2 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8001 port_no 0x1
designated_port 32769 designated_cost 0 designated_bridge
8000.2e:fe:5b:9c:82:d4 designated_root 8000.2e:fe:5b:9c:82:d4 hold_timer
0.00 message_age_timer 0.00 forward_delay_timer 0.00 topology_change_ack
0 config_pending 0 proxy_arp off proxy_arp_wifi off mcast_router 1
mcast_fast_leave off mcast_flood on neigh_suppress off group_fwd_mask 0
group_fwd_mask_str 0x0 vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1
gso_max_size 65536 gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
74540882 448114 0 0 0 0
TX: bytes packets errors dropped carrier collsns
5910566 59511 0 0 0 0
9: veth103i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue
master vmbr0 state UP group default qlen 1000
link/ether fe:81:2a:60:02:f6 brd ff:ff:ff:ff:ff:ff link-netnsid 1
promiscuity 1 minmtu 68 maxmtu 65535
veth
bridge_slave state forwarding priority 32 cost 2 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8004 port_no 0x4
designated_port 32772 designated_cost 0 designated_bridge 8000.f8:d1:11:2:a1:cc
designated_root 8000.f8:d1:11:2:a1:cc hold_timer 0.00 message_age_timer
0.00 forward_delay_timer 0.00 topology_change_ack 0 config_pending 0
proxy_arp off proxy_arp_wifi off mcast_router 1 mcast_fast_leave off
mcast_flood on neigh_suppress off group_fwd_mask 0 group_fwd_mask_str 0x0
vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1 gso_max_size 65536
gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
9223241 86233 0 0 0 0
TX: bytes packets errors dropped carrier collsns
68945534 398590 0 0 0 0
10: veth106i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue
master fwbr106i0 state UP group default qlen 1000
link/ether fe:9b:1b:95:29:46 brd ff:ff:ff:ff:ff:ff link-netnsid 2
promiscuity 1 minmtu 68 maxmtu 65535
veth
bridge_slave state forwarding priority 32 cost 2 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8002 port_no 0x2
designated_port 32770 designated_cost 0 designated_bridge
8000.16:b2:43:94:36:8e designated_root 8000.16:b2:43:94:36:8e hold_timer
0.00 message_age_timer 0.00 forward_delay_timer 0.00 topology_change_ack
0 config_pending 0 proxy_arp off proxy_arp_wifi off mcast_router 1
mcast_fast_leave off mcast_flood on neigh_suppress off group_fwd_mask 0
group_fwd_mask_str 0x0 vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1
gso_max_size 65536 gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
94714 509 0 0 0 0
TX: bytes packets errors dropped carrier collsns
69285225 400198 0 0 0 0
11: fwbr106i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state
UP group default qlen 1000
link/ether 16:b2:43:94:36:8e brd ff:ff:ff:ff:ff:ff promiscuity 0 minmtu 68
maxmtu 65535
bridge forward_delay 0 hello_time 200 max_age 2000 ageing_time 30000
stp_state 0 priority 32768 vlan_filtering 0 vlan_protocol 802.1Q bridge_id
8000.16:b2:43:94:36:8e designated_root 8000.16:b2:43:94:36:8e root_port 0
root_path_cost 0 topology_change 0 topology_change_detected 0 hello_timer
0.00 tcn_timer 0.00 topology_change_timer 0.00 gc_timer 0.00
vlan_default_pvid 1 vlan_stats_enabled 0 group_fwd_mask 0 group_address
01:80:c2:00:00:00 mcast_snooping 1 mcast_router 1 mcast_query_use_ifaddr 0
mcast_querier 0 mcast_hash_elasticity 16 mcast_hash_max 4096
mcast_last_member_count 2 mcast_startup_query_count 2
mcast_last_member_interval 100 mcast_membership_interval 26000
mcast_querier_interval 25500 mcast_query_interval 12500
mcast_query_response_interval 1000 mcast_startup_query_interval 3124
mcast_stats_enabled 0 mcast_igmp_version 2 mcast_mld_version 1 nf_call_iptables
0 nf_call_ip6tables 0 nf_call_arptables 0 numtxqueues 1 numrxqueues 1
gso_max_size 65536 gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
63435167 396235 0 5 0 0
TX: bytes packets errors dropped carrier collsns
108 2 0 0 0 0
12: fwpr106p0@fwln106i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
noqueue master vmbr0 state UP group default qlen 1000
link/ether 5a:75:0d:55:7b:ac brd ff:ff:ff:ff:ff:ff promiscuity 1 minmtu 68
maxmtu 65535
veth
bridge_slave state forwarding priority 32 cost 2 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8005 port_no 0x5
designated_port 32773 designated_cost 0 designated_bridge 8000.f8:d1:11:2:a1:cc
designated_root 8000.f8:d1:11:2:a1:cc hold_timer 0.00 message_age_timer
0.00 forward_delay_timer 0.00 topology_change_ack 0 config_pending 0
proxy_arp off proxy_arp_wifi off mcast_router 1 mcast_fast_leave off
mcast_flood on neigh_suppress off group_fwd_mask 0 group_fwd_mask_str 0x0
vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1 gso_max_size 65536
gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
94768 510 0 0 0 0
TX: bytes packets errors dropped carrier collsns
69288941 400215 0 0 0 0
13: fwln106i0@fwpr106p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
noqueue master fwbr106i0 state UP group default qlen 1000
link/ether 16:b2:43:94:36:8e brd ff:ff:ff:ff:ff:ff promiscuity 1 minmtu 68
maxmtu 65535
veth
bridge_slave state forwarding priority 32 cost 2 hairpin off guard off
root_block off fastleave off learning on flood on port_id 0x8001 port_no 0x1
designated_port 32769 designated_cost 0 designated_bridge
8000.16:b2:43:94:36:8e designated_root 8000.16:b2:43:94:36:8e hold_timer
0.00 message_age_timer 0.00 forward_delay_timer 0.00 topology_change_ack
0 config_pending 0 proxy_arp off proxy_arp_wifi off mcast_router 1
mcast_fast_leave off mcast_flood on neigh_suppress off group_fwd_mask 0
group_fwd_mask_str 0x0 vlan_tunnel off isolated off numtxqueues 1 numrxqueues 1
gso_max_size 65536 gso_max_segs 65535
RX: bytes packets errors dropped overrun mcast
69288941 400215 0 0 0 0
TX: bytes packets errors dropped carrier collsns
94768 510 0 0 0 0
# ip -details -4 route show
unicast default via 192.168.89.2 dev vmbr0 proto kernel scope global onlink
unicast 192.168.89.0/25 dev vmbr0 proto kernel scope link src 192.168.89.41
# ip -details -6 route show
unicast ::1 dev lo proto kernel scope global metric 256 pref medium
unicast fe80::/64 dev vmbr0 proto kernel scope global metric 256 pref medium
# cat /etc/network/interfaces
# network interface settings; autogenerated
# Please do NOT modify this file directly, unless you know what
# you're doing.
#
# If you want to manage parts of the network configuration manually,
# please utilize the 'source' or 'source-directory' directives to do
# so.
# PVE will preserve these directives, but will NOT read its network
# configuration from sourced files, so do not attempt to move any of
# the PVE managed interfaces into external files!
auto lo
iface lo inet loopback
iface enp2s0 inet manual
iface enp1s0 inet manual
auto vmbr0
iface vmbr0 inet static
address 192.168.89.41/25
gateway 192.168.89.2
bridge-ports enp1s0 enp2s0
bridge-stp off
bridge-fd 0
#Main IP
==== info about firewall ====
# cat /etc/pve/local/host.fw
cat: /etc/pve/local/host.fw: No such file or directory
# iptables-save
# Generated by iptables-save v1.8.2 on Fri Sep 25 10:28:33 2020
*raw
:PREROUTING ACCEPT [655078:169190107]
:OUTPUT ACCEPT [40101:9045639]
COMMIT
# Completed on Fri Sep 25 10:28:33 2020
# Generated by iptables-save v1.8.2 on Fri Sep 25 10:28:33 2020
*filter
:INPUT ACCEPT [530153:155380893]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [40109:9046633]
COMMIT
# Completed on Fri Sep 25 10:28:33 2020
==== info about cluster ====
# pvecm nodes
Membership information
----------------------
Nodeid Votes Name
1 1 pve (local)
# pvecm status
Cluster information
-------------------
Name: NufNuf
Config Version: 1
Transport: knet
Secure auth: on
Quorum information
------------------
Date: Fri Sep 25 10:28:34 2020
Quorum provider: corosync_votequorum
Nodes: 1
Node ID: 0x00000001
Ring ID: 1.28
Quorate: Yes
Votequorum information
----------------------
Expected votes: 1
Highest expected: 1
Total votes: 1
Quorum: 1
Flags: Quorate
Membership information
----------------------
Nodeid Votes Name
0x00000001 1 192.168.89.41 (local)
# cat /etc/pve/corosync.conf 2>/dev/null
logging {
debug: off
to_syslog: yes
}
nodelist {
node {
name: pve
nodeid: 1
quorum_votes: 1
ring0_addr: 192.168.89.41
}
}
quorum {
provider: corosync_votequorum
}
totem {
cluster_name: NufNuf
config_version: 1
interface {
linknumber: 0
}
ip_version: ipv4-6
link_mode: passive
secauth: on
version: 2
}
==== info about bios ====
# dmidecode -t bios
# dmidecode 3.2
Getting SMBIOS data from sysfs.
SMBIOS 3.0.0 present.
Handle 0x0000, DMI type 0, 24 bytes
BIOS Information
Vendor: American Megatrends Inc.
Version: 1001
Release Date: 12/11/2017
Address: 0xF0000
Runtime Size: 64 kB
ROM Size: 16 MB
Characteristics:
PCI is supported
APM is supported
BIOS is upgradeable
BIOS shadowing is allowed
Boot from CD is supported
Selectable boot is supported
BIOS ROM is socketed
EDD is supported
5.25"/1.2 MB floppy services are supported (int 13h)
3.5"/720 kB floppy services are supported (int 13h)
3.5"/2.88 MB floppy services are supported (int 13h)
Print screen service is supported (int 5h)
8042 keyboard services are supported (int 9h)
Serial services are supported (int 14h)
Printer services are supported (int 17h)
ACPI is supported
USB legacy is supported
BIOS boot specification is supported
Targeted content distribution is supported
UEFI is supported
BIOS Revision: 5.12
==== info about pci ====
# lspci -nnk
00:00.0 Host bridge [0600]: Intel Corporation Skylake Host Bridge/DRAM
Registers [8086:190f] (rev 07)
Subsystem: ASUSTeK Computer Inc. Xeon E3-1200 v5/E3-1500 v5/6th Gen
Core Processor Host Bridge/DRAM Registers [1043:8694]
Kernel driver in use: skl_uncore
00:02.0 VGA compatible controller [0300]: Intel Corporation HD Graphics 530
[8086:1912] (rev 06)
Subsystem: ASUSTeK Computer Inc. HD Graphics 530 [1043:8694]
Kernel driver in use: i915
Kernel modules: i915
00:14.0 USB controller [0c03]: Intel Corporation 200 Series/Z370 Chipset Family
USB 3.0 xHCI Controller [8086:a2af]
Subsystem: ASUSTeK Computer Inc. 200 Series/Z370 Chipset Family USB 3.0
xHCI Controller [1043:8694]
Kernel driver in use: xhci_hcd
Kernel modules: xhci_pci
00:16.0 Communication controller [0780]: Intel Corporation 200 Series PCH CSME
HECI [8086:a2ba]
Subsystem: ASUSTeK Computer Inc. 200 Series PCH CSME HECI [1043:8694]
Kernel driver in use: mei_me
Kernel modules: mei_me
00:17.0 SATA controller [0106]: Intel Corporation 200 Series PCH SATA
controller [AHCI mode] [8086:a282]
Subsystem: ASUSTeK Computer Inc. 200 Series PCH SATA controller [AHCI
mode] [1043:8694]
Kernel driver in use: ahci
Kernel modules: ahci
00:1c.0 PCI bridge [0604]: Intel Corporation 200 Series PCH PCI Express Root
Port [8086:a294] (rev f0)
Kernel driver in use: pcieport
00:1c.7 PCI bridge [0604]: Intel Corporation 200 Series PCH PCI Express Root
Port [8086:a297] (rev f0)
Kernel driver in use: pcieport
00:1d.0 PCI bridge [0604]: Intel Corporation 200 Series PCH PCI Express Root
Port [8086:a298] (rev f0)
Kernel driver in use: pcieport
00:1f.0 ISA bridge [0601]: Intel Corporation 200 Series PCH LPC Controller
(B250) [8086:a2c8]
Subsystem: ASUSTeK Computer Inc. 200 Series PCH LPC Controller (B250)
[1043:8694]
00:1f.2 Memory controller [0580]: Intel Corporation 200 Series/Z370 Chipset
Family Power Management Controller [8086:a2a1]
Subsystem: ASUSTeK Computer Inc. 200 Series/Z370 Chipset Family Power
Management Controller [1043:8694]
00:1f.3 Audio device [0403]: Intel Corporation 200 Series PCH HD Audio
[8086:a2f0]
Subsystem: ASUSTeK Computer Inc. 200 Series PCH HD Audio [1043:86c7]
Kernel driver in use: snd_hda_intel
Kernel modules: snd_hda_intel
00:1f.4 SMBus [0c05]: Intel Corporation 200 Series/Z370 Chipset Family SMBus
Controller [8086:a2a3]
Subsystem: ASUSTeK Computer Inc. 200 Series/Z370 Chipset Family SMBus
Controller [1043:8694]
Kernel driver in use: i801_smbus
Kernel modules: i2c_i801
01:00.0 Ethernet controller [0200]: Realtek Semiconductor Co., Ltd.
RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller [10ec:8168] (rev 01)
Subsystem: Realtek Semiconductor Co., Ltd. RTL8111/8168 PCI Express
Gigabit Ethernet controller [10ec:8168]
Kernel driver in use: r8169
Kernel modules: r8169
02:00.0 Ethernet controller [0200]: Realtek Semiconductor Co., Ltd.
RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller [10ec:8168] (rev 15)
Subsystem: ASUSTeK Computer Inc. RTL8111/8168/8411 PCI Express Gigabit
Ethernet Controller [1043:8677]
Kernel driver in use: r8169
Kernel modules: r8169
03:00.0 Non-Volatile memory controller [0108]: Intel Corporation SSD Pro
7600p/760p/E 6100p Series [8086:f1a6] (rev 03)
Subsystem: Intel Corporation SSD Pro 7600p/760p/E 6100p Series
[8086:390b]
Kernel driver in use: nvme
==== info about disks ====
# lsblk --ascii
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 1.8T 0 disk
|-sda1 8:1 0 1.8T 0 part
`-sda9 8:9 0 8M 0 part
sdb 8:16 0 1.8T 0 disk
|-sdb1 8:17 0 1.8T 0 part
`-sdb9 8:25 0 8M 0 part
nvme0n1 259:0 0 119.2G 0 disk
|-nvme0n1p1 259:1 0 1007K 0 part
|-nvme0n1p2 259:2 0 512M 0 part
`-nvme0n1p3 259:3 0 118.8G 0 part
|-pve-swap 253:0 0 8G 0 lvm [SWAP]
|-pve-root 253:1 0 29.5G 0 lvm /
|-pve-data_tmeta 253:2 0 1G 0 lvm
| `-pve-data-tpool 253:4 0 64.5G 0 lvm
| |-pve-data 253:5 0 64.5G 0 lvm
| |-pve-vm--102--disk--0 253:6 0 12G 0 lvm
| |-pve-vm--103--disk--0 253:7 0 12G 0 lvm
| |-pve-vm--101--disk--0 253:8 0 8G 0 lvm
| |-pve-vm--100--disk--0 253:9 0 4G 0 lvm
| |-pve-vm--104--disk--0 253:10 0 6G 0 lvm
| |-pve-vm--105--disk--0 253:11 0 8G 0 lvm
| `-pve-vm--106--disk--0 253:12 0 8G 0 lvm
`-pve-data_tdata 253:3 0 64.5G 0 lvm
`-pve-data-tpool 253:4 0 64.5G 0 lvm
|-pve-data 253:5 0 64.5G 0 lvm
|-pve-vm--102--disk--0 253:6 0 12G 0 lvm
|-pve-vm--103--disk--0 253:7 0 12G 0 lvm
|-pve-vm--101--disk--0 253:8 0 8G 0 lvm
|-pve-vm--100--disk--0 253:9 0 4G 0 lvm
|-pve-vm--104--disk--0 253:10 0 6G 0 lvm
|-pve-vm--105--disk--0 253:11 0 8G 0 lvm
`-pve-vm--106--disk--0 253:12 0 8G 0 lvm
# ls -l /dev/disk/by-*/
/dev/disk/by-id/:
total 0
lrwxrwxrwx 1 root root 9 Sep 24 13:50 ata-ST2000DM001-1ER164_Z4Z0GGKL ->
../../sdb
lrwxrwxrwx 1 root root 10 Sep 24 13:50 ata-ST2000DM001-1ER164_Z4Z0GGKL-part1 ->
../../sdb1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 ata-ST2000DM001-1ER164_Z4Z0GGKL-part9 ->
../../sdb9
lrwxrwxrwx 1 root root 9 Sep 24 13:50 ata-WDC_WD20EZRZ-00Z5HB0_WD-WCC4N2KCNF9U
-> ../../sda
lrwxrwxrwx 1 root root 10 Sep 24 13:50
ata-WDC_WD20EZRZ-00Z5HB0_WD-WCC4N2KCNF9U-part1 -> ../../sda1
lrwxrwxrwx 1 root root 10 Sep 24 13:50
ata-WDC_WD20EZRZ-00Z5HB0_WD-WCC4N2KCNF9U-part9 -> ../../sda9
lrwxrwxrwx 1 root root 10 Sep 24 13:50 dm-name-pve-root -> ../../dm-1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 dm-name-pve-swap -> ../../dm-0
lrwxrwxrwx 1 root root 10 Sep 24 13:50 dm-name-pve-vm--100--disk--0 ->
../../dm-9
lrwxrwxrwx 1 root root 10 Sep 24 13:50 dm-name-pve-vm--101--disk--0 ->
../../dm-8
lrwxrwxrwx 1 root root 10 Sep 24 13:50 dm-name-pve-vm--102--disk--0 ->
../../dm-6
lrwxrwxrwx 1 root root 10 Sep 24 13:50 dm-name-pve-vm--103--disk--0 ->
../../dm-7
lrwxrwxrwx 1 root root 11 Sep 24 13:50 dm-name-pve-vm--104--disk--0 ->
../../dm-10
lrwxrwxrwx 1 root root 11 Sep 24 13:50 dm-name-pve-vm--105--disk--0 ->
../../dm-11
lrwxrwxrwx 1 root root 11 Sep 24 13:50 dm-name-pve-vm--106--disk--0 ->
../../dm-12
lrwxrwxrwx 1 root root 11 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJ3wIyqJCfsJgV5xcCiWJNZDoXFixenl31 ->
../../dm-12
lrwxrwxrwx 1 root root 11 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJ4Cer7egCglsqmxrhlgKfY5Rmcyhr7NsD ->
../../dm-11
lrwxrwxrwx 1 root root 10 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJAWPTaNexIiLNIaxRBEJtCLWyUlpdsKUN ->
../../dm-6
lrwxrwxrwx 1 root root 11 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJH0uRVnzwHIeVsKMxlmlHEIxkK9dH2GOq ->
../../dm-10
lrwxrwxrwx 1 root root 10 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJOFjvQ52oMiUYpaf8TkXbYRO4e35yzeSc ->
../../dm-7
lrwxrwxrwx 1 root root 10 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJVZFWEJRDjlrcKUyDFLFKsHdJ5IO40BQn ->
../../dm-0
lrwxrwxrwx 1 root root 10 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJgevvVqmrFslEG9Pd9FaOAMIiJeXkb3EA ->
../../dm-8
lrwxrwxrwx 1 root root 10 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJuiHnSMehF8e82uBr6lY91quZwNsKE8sd ->
../../dm-9
lrwxrwxrwx 1 root root 10 Sep 24 13:50
dm-uuid-LVM-2n9jVYjeFLz26zFA8edesmDOJQLNZovJwkoN7qOk3JvePKqQw2NBnWgnbMgYUjPi ->
../../dm-1
lrwxrwxrwx 1 root root 15 Sep 24 13:50
lvm-pv-uuid-Ngn9BS-l7oV-cR9w-x6au-jb12-gMWR-ujBs35 -> ../../nvme0n1p3
lrwxrwxrwx 1 root root 13 Sep 24 13:50
nvme-INTEL_SSDPEKKW128G8_PHHH9251012J128A -> ../../nvme0n1
lrwxrwxrwx 1 root root 15 Sep 24 13:50
nvme-INTEL_SSDPEKKW128G8_PHHH9251012J128A-part1 -> ../../nvme0n1p1
lrwxrwxrwx 1 root root 15 Sep 24 13:50
nvme-INTEL_SSDPEKKW128G8_PHHH9251012J128A-part2 -> ../../nvme0n1p2
lrwxrwxrwx 1 root root 15 Sep 24 13:50
nvme-INTEL_SSDPEKKW128G8_PHHH9251012J128A-part3 -> ../../nvme0n1p3
lrwxrwxrwx 1 root root 13 Sep 24 13:50
nvme-eui.0000000001000000e4d25c79b9ef5001 -> ../../nvme0n1
lrwxrwxrwx 1 root root 15 Sep 24 13:50
nvme-eui.0000000001000000e4d25c79b9ef5001-part1 -> ../../nvme0n1p1
lrwxrwxrwx 1 root root 15 Sep 24 13:50
nvme-eui.0000000001000000e4d25c79b9ef5001-part2 -> ../../nvme0n1p2
lrwxrwxrwx 1 root root 15 Sep 24 13:50
nvme-eui.0000000001000000e4d25c79b9ef5001-part3 -> ../../nvme0n1p3
lrwxrwxrwx 1 root root 9 Sep 24 13:50 wwn-0x5000c50078dff321 -> ../../sdb
lrwxrwxrwx 1 root root 10 Sep 24 13:50 wwn-0x5000c50078dff321-part1 ->
../../sdb1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 wwn-0x5000c50078dff321-part9 ->
../../sdb9
lrwxrwxrwx 1 root root 9 Sep 24 13:50 wwn-0x50014ee20d82c0be -> ../../sda
lrwxrwxrwx 1 root root 10 Sep 24 13:50 wwn-0x50014ee20d82c0be-part1 ->
../../sda1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 wwn-0x50014ee20d82c0be-part9 ->
../../sda9
/dev/disk/by-label/:
total 0
lrwxrwxrwx 1 root root 10 Sep 24 13:50 nnpoolz -> ../../sdb1
/dev/disk/by-partlabel/:
total 0
lrwxrwxrwx 1 root root 10 Sep 24 13:50 zfs-3153ea9e3014b968 -> ../../sda1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 zfs-471b64aa981fc7a7 -> ../../sdb1
/dev/disk/by-partuuid/:
total 0
lrwxrwxrwx 1 root root 10 Sep 24 13:50 18a50985-bc08-6347-9394-a09d67e50a3c ->
../../sda1
lrwxrwxrwx 1 root root 15 Sep 24 13:50 23960d11-51eb-4b03-a20a-aefe0718c810 ->
../../nvme0n1p2
lrwxrwxrwx 1 root root 10 Sep 24 13:50 5cb933eb-7a3f-ac4d-ba83-41b7d7e4804b ->
../../sda9
lrwxrwxrwx 1 root root 15 Sep 24 13:50 666373f0-f688-4003-91d3-f490db21e16c ->
../../nvme0n1p3
lrwxrwxrwx 1 root root 15 Sep 24 13:50 747e1cad-eab7-47c7-906f-4dca9c3a4f66 ->
../../nvme0n1p1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 b65dccc6-dd53-314f-baf8-06117149cbde ->
../../sdb1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 d1f93ecd-31bb-4140-a31b-a649a051112f ->
../../sdb9
/dev/disk/by-path/:
total 0
lrwxrwxrwx 1 root root 9 Sep 24 13:50 pci-0000:00:17.0-ata-5 -> ../../sda
lrwxrwxrwx 1 root root 10 Sep 24 13:50 pci-0000:00:17.0-ata-5-part1 ->
../../sda1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 pci-0000:00:17.0-ata-5-part9 ->
../../sda9
lrwxrwxrwx 1 root root 9 Sep 24 13:50 pci-0000:00:17.0-ata-6 -> ../../sdb
lrwxrwxrwx 1 root root 10 Sep 24 13:50 pci-0000:00:17.0-ata-6-part1 ->
../../sdb1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 pci-0000:00:17.0-ata-6-part9 ->
../../sdb9
lrwxrwxrwx 1 root root 13 Sep 24 13:50 pci-0000:03:00.0-nvme-1 -> ../../nvme0n1
lrwxrwxrwx 1 root root 15 Sep 24 13:50 pci-0000:03:00.0-nvme-1-part1 ->
../../nvme0n1p1
lrwxrwxrwx 1 root root 15 Sep 24 13:50 pci-0000:03:00.0-nvme-1-part2 ->
../../nvme0n1p2
lrwxrwxrwx 1 root root 15 Sep 24 13:50 pci-0000:03:00.0-nvme-1-part3 ->
../../nvme0n1p3
/dev/disk/by-uuid/:
total 0
lrwxrwxrwx 1 root root 10 Sep 24 13:50 0cc8fb0c-4caa-492d-99cf-f2550efb7c88 ->
../../dm-7
lrwxrwxrwx 1 root root 10 Sep 24 13:50 17901406786448779187 -> ../../sdb1
lrwxrwxrwx 1 root root 11 Sep 24 13:50 363e3682-4f5d-476d-bf96-8281abdb1f65 ->
../../dm-11
lrwxrwxrwx 1 root root 11 Sep 24 13:50 4dc0b6d8-a260-46e3-be55-d62ca8dbcea3 ->
../../dm-12
lrwxrwxrwx 1 root root 11 Sep 24 13:50 78bebbc3-a506-4065-8e96-fb9435d60100 ->
../../dm-10
lrwxrwxrwx 1 root root 10 Sep 24 13:50 8291aa9e-95e8-4a6b-9f89-f34b58459c48 ->
../../dm-1
lrwxrwxrwx 1 root root 10 Sep 24 13:50 86805845-8653-4444-bf3c-29860ddb86d5 ->
../../dm-9
lrwxrwxrwx 1 root root 10 Sep 24 13:50 8afbb735-1b95-4782-bc13-680132bc9514 ->
../../dm-0
lrwxrwxrwx 1 root root 10 Sep 24 13:50 8c52b39b-f0a1-4b58-ba7b-586d5dc43e31 ->
../../dm-6
lrwxrwxrwx 1 root root 15 Sep 24 13:50 B43B-D4FA -> ../../nvme0n1p2
lrwxrwxrwx 1 root root 10 Sep 24 13:50 cff3e810-8717-4634-a9fb-ee71b6a21e7f ->
../../dm-8
# iscsiadm -m node
iscsiadm: No records found
# iscsiadm -m session
iscsiadm: No active sessions.
==== info about volumes ====
# pvs
PV VG Fmt Attr PSize PFree
/dev/nvme0n1p3 pve lvm2 a-- <118.74g 14.75g
# lvs
LV VG Attr LSize Pool Origin Data% Meta% Move Log
Cpy%Sync Convert
data pve twi-aotz-- <64.49g 28.40 2.44
root pve -wi-ao---- 29.50g
swap pve -wi-ao---- 8.00g
vm-100-disk-0 pve Vwi-a-tz-- 4.00g data 30.23
vm-101-disk-0 pve Vwi-a-tz-- 8.00g data 21.58
vm-102-disk-0 pve Vwi-aotz-- 12.00g data 14.64
vm-103-disk-0 pve Vwi-aotz-- 12.00g data 78.67
vm-104-disk-0 pve Vwi-a-tz-- 6.00g data 21.75
vm-105-disk-0 pve Vwi-a-tz-- 8.00g data 14.57
vm-106-disk-0 pve Vwi-aotz-- 8.00g data 21.35
# vgs
VG #PV #LV #SN Attr VSize VFree
pve 1 10 0 wz--n- <118.74g 14.75g
# zpool status
pool: nnpoolz
state: ONLINE
scan: scrub repaired 0B in 0 days 00:31:39 with 0 errors on Sun Sep 13
00:55:40 2020
config:
NAME STATE READ WRITE CKSUM
nnpoolz ONLINE 0 0 0
wwn-0x50014ee20d82c0be ONLINE 0 0 0
wwn-0x5000c50078dff321 ONLINE 0 0 0
errors: No known data errors
# zpool list -v
NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP
DEDUP HEALTH ALTROOT
nnpoolz 3.62T 569G 3.07T - - 0% 15%
1.00x ONLINE -
wwn-0x50014ee20d82c0be 1.81T 230G 1.59T - - 0% 12.4%
- ONLINE
wwn-0x5000c50078dff321 1.81T 339G 1.48T - - 1% 18.3%
- ONLINE
# zfs list
NAME USED AVAIL REFER MOUNTPOINT
nnpoolz 569G 2.96T 569G /nnpoolz
root@pve:~#
--- End Message ---
