Author: sephe
Date: Thu Jan  5 03:20:00 2017
New Revision: 311353
URL: https://svnweb.freebsd.org/changeset/base/311353

Log:
  MFC 308664,308742,308743
  
  308664
      hyperv/vss: Add driver and tools for VSS
  
      VSS stands for "Volume Shadow Copy Service".  Unlike virtual machine
      snapshot, it only takes snapshot for the virtual disks, so both
      filesystem and applications have to aware of it, and cooperate the
      whole VSS process.
  
      This driver exposes two device files to the userland:
  
          /dev/hv_fsvss_dev
  
          Normally userland programs should _not_ mess with this device file.
          It is currently used by the hv_vss_daemon(8), which freezes and
          thaws the filesystem.  NOTE: currently only UFS is supported, if
          the system mounts _any_ other filesystems, the hv_vss_daemon(8)
          will veto the VSS process.
  
          If hv_vss_daemon(8) was disabled, then this device file must be
          opened, and proper ioctls must be issued to keep the VSS working.
  
          /dev/hv_appvss_dev
  
          Userland application can opened this device file to receive the
          VSS freeze notification, hold the VSS for a while (mainly to flush
          application data to filesystem), release the VSS process, and
          receive the VSS thaw notification i.e. applications can run again.
  
          The VSS will still work, even if this device file is not opened.
          However, only filesystem consistency is promised, if this device
          file is not opened or is not operated properly.
  
      hv_vss_daemon(8) is started by devd(8) by default.  It can be disabled
      by editting /etc/devd/hyperv.conf.
  
      Submitted by:   Hongjiang Zhang <honzhan microsoft com>
      Reviewed by:    kib, mckusick
      Sponsored by:   Microsoft
      Differential Revision:  https://reviews.freebsd.org/D8224
  
  308742
      hyperv/vss: Nuke unused variables.
  
      Submitted by:   markj
      Reported by:    markj
      Sponsored by:   Microsoft
  
  308743
      hyperv/vss: Install the userland daemon to /usr/sbin instead of /
  
      Submitted by:   markj
      Reported by:    markj
      Sponsored by:   Microsoft

Added:
  stable/11/contrib/hyperv/tools/hv_vss_daemon.8
     - copied unchanged from r308664, head/contrib/hyperv/tools/hv_vss_daemon.8
  stable/11/contrib/hyperv/tools/hv_vss_daemon.c
     - copied, changed from r308664, head/contrib/hyperv/tools/hv_vss_daemon.c
  stable/11/share/man/man4/hv_vss.4
     - copied unchanged from r308664, head/share/man/man4/hv_vss.4
  stable/11/sys/dev/hyperv/utilities/hv_snapshot.c
     - copied unchanged from r308664, 
head/sys/dev/hyperv/utilities/hv_snapshot.c
  stable/11/sys/dev/hyperv/utilities/hv_snapshot.h
     - copied unchanged from r308664, 
head/sys/dev/hyperv/utilities/hv_snapshot.h
  stable/11/usr.sbin/hyperv/tools/Makefile.inc
     - copied unchanged from r308743, head/usr.sbin/hyperv/tools/Makefile.inc
  stable/11/usr.sbin/hyperv/tools/kvp/
     - copied from r308664, head/usr.sbin/hyperv/tools/kvp/
  stable/11/usr.sbin/hyperv/tools/vss/
     - copied from r308664, head/usr.sbin/hyperv/tools/vss/
Deleted:
  stable/11/usr.sbin/hyperv/tools/Makefile
  stable/11/usr.sbin/hyperv/tools/Makefile.depend
Modified:
  stable/11/etc/devd/hyperv.conf
  stable/11/etc/mtree/BSD.include.dist
  stable/11/include/Makefile
  stable/11/share/man/man4/Makefile
  stable/11/sys/conf/files.amd64
  stable/11/sys/conf/files.i386
  stable/11/sys/modules/hyperv/utilities/Makefile
  stable/11/usr.sbin/hyperv/Makefile
Directory Properties:
  stable/11/   (props changed)

Copied: stable/11/contrib/hyperv/tools/hv_vss_daemon.8 (from r308664, 
head/contrib/hyperv/tools/hv_vss_daemon.8)
==============================================================================
--- /dev/null   00:00:00 1970   (empty, because file is newly added)
+++ stable/11/contrib/hyperv/tools/hv_vss_daemon.8      Thu Jan  5 03:20:00 
2017        (r311353, copy of r308664, 
head/contrib/hyperv/tools/hv_vss_daemon.8)
@@ -0,0 +1,88 @@
+.\" Copyright (c) 2016 Microsoft Corp.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.Dd October 12, 2016
+.Dt HV_VSS_DAEMON 8
+.Os
+.Sh NAME
+.Nm hv_vss_daemon
+.Nd Hyper-V Volume Shadow Copy Service Daemon
+.Sh SYNOPSIS
+.Nm
+.Op Fl dn
+.Sh DESCRIPTION
+The
+.Nm
+daemon provides the ability to freeze and thaw the file system for
+.Fx
+guest partitions running on Hyper-V.
+.Pp
+Hyper-V allows administrators to backup or restore the 
+.Fx
+guest partition.
+Administrators can
+use Windows Powershell scripts to backup or restore the
+.Fx
+VM.
+.Pp
+The
+.Nm
+accepts file system freeze and thaw requests from the
+.Xr hv_utils 4
+driver and performs the actual file-system operation.
+.Pp
+The file system freeze and thaw functionality is
+useful when the Hyper-V host wants to do live backup of
+.Fx
+guest. Hyper-V host sends file system freezing request to
+.Nm
+which conducts the real operation. After successfully freezing file
+system, Hyper-V host takes a snapshot of the VM. In the future,
+Hyper-V host can restore the
+.Fx
+VM through that snapshot.
+.Pp
+The options are as follows:
+.Bl -tag -width indent
+.It Fl d
+Run as regular process instead of a daemon for debugging purpose.
+.It Fl n
+Generate debugging output.
+.El
+.Sh SEE ALSO
+.Xr hv_vmbus 4 ,
+.Xr hv_utils 4 ,
+.Xr hv_netvsc 4 ,
+.Xr hv_storvsc 4 ,
+.Xr hv_kvp 4
+.Sh HISTORY
+The daemon was introduced in October 2016 and developed by Microsoft Corp.
+.Sh AUTHORS
+.An -nosplit
+.Fx
+support for
+.Nm
+was first added by
+.An Microsoft BSD Integration Services Team Aq Mt bs...@microsoft.com .

Copied and modified: stable/11/contrib/hyperv/tools/hv_vss_daemon.c (from 
r308664, head/contrib/hyperv/tools/hv_vss_daemon.c)
==============================================================================
--- head/contrib/hyperv/tools/hv_vss_daemon.c   Tue Nov 15 02:36:12 2016        
(r308664, copy source)
+++ stable/11/contrib/hyperv/tools/hv_vss_daemon.c      Thu Jan  5 03:20:00 
2017        (r311353)
@@ -158,10 +158,9 @@ main(int argc, char* argv[])
 
        struct pollfd hv_vss_poll_fd[1];
        uint32_t op;
-       int ch, r, len, error;
+       int ch, r, error;
        int hv_vss_dev_fd;
 
-       int freeze_thaw = UNDEF_FREEZE_THAW;
        while ((ch = getopt(argc, argv, "dnh")) != -1) {
                switch (ch) {
                case 'n':

Modified: stable/11/etc/devd/hyperv.conf
==============================================================================
--- stable/11/etc/devd/hyperv.conf      Thu Jan  5 03:13:45 2017        
(r311352)
+++ stable/11/etc/devd/hyperv.conf      Thu Jan  5 03:20:00 2017        
(r311353)
@@ -17,3 +17,19 @@ notify 10 {
        match "cdev"            "hv_kvp_dev";
        action "pkill -x hv_kvp_daemon";
 };
+
+notify 11 {
+       match "system"          "DEVFS";
+       match "subsystem"       "CDEV";
+       match "type"            "CREATE";
+       match "cdev"            "hv_fsvss_dev";
+       action "/usr/sbin/hv_vss_daemon";
+};
+
+notify 11 {
+       match "system"          "DEVFS";
+       match "subsystem"       "CDEV";
+       match "type"            "DESTROY";
+       match "cdev"            "hv_fsvss_dev";
+       action "pkill -x hv_vss_daemon";
+};

Modified: stable/11/etc/mtree/BSD.include.dist
==============================================================================
--- stable/11/etc/mtree/BSD.include.dist        Thu Jan  5 03:13:45 2017        
(r311352)
+++ stable/11/etc/mtree/BSD.include.dist        Thu Jan  5 03:20:00 2017        
(r311353)
@@ -118,6 +118,8 @@
         ..
         hwpmc
         ..
+        hyperv
+        ..
         ic
         ..
         iicbus

Modified: stable/11/include/Makefile
==============================================================================
--- stable/11/include/Makefile  Thu Jan  5 03:13:45 2017        (r311352)
+++ stable/11/include/Makefile  Thu Jan  5 03:20:00 2017        (r311353)
@@ -44,7 +44,7 @@ LDIRS=        bsm cam geom net net80211 netgrap
 
 LSUBDIRS=      cam/ata cam/nvme cam/scsi \
        dev/acpica dev/agp dev/an dev/bktr dev/ciss dev/filemon dev/firewire \
-       dev/hwpmc \
+       dev/hwpmc dev/hyperv \
        dev/ic dev/iicbus dev/io dev/lmc dev/mfi dev/nvme \
        dev/ofw dev/pbio dev/pci ${_dev_powermac_nvram} dev/ppbus dev/smbus \
        dev/speaker dev/utopia dev/vkbd dev/wi \
@@ -154,7 +154,7 @@ copies: .PHONY .META
                done; \
        fi
 .endfor
-.for i in ${LDIRS} 
${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/nand:Ndev/pci} 
${LSUBSUBDIRS}
+.for i in ${LDIRS} 
${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/hyperv:Ndev/nand:Ndev/pci}
 ${LSUBSUBDIRS}
        cd ${.CURDIR}/../sys; \
        ${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 $i/*.h \
            ${DESTDIR}${INCLUDEDIR}/$i
@@ -184,6 +184,9 @@ copies: .PHONY .META
            ${DESTDIR}${INCLUDEDIR}/dev/evdev; \
        ${INSTALL} -C -o ${BINOWN} -g ${BINGRP} -m 444 uinput.h \
            ${DESTDIR}${INCLUDEDIR}/dev/evdev
+       cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
+       ${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 
hv_snapshot.h \
+           ${DESTDIR}${INCLUDEDIR}/dev/hyperv
        cd ${.CURDIR}/../sys/dev/pci; \
        ${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 pcireg.h \
            ${DESTDIR}${INCLUDEDIR}/dev/pci
@@ -256,7 +259,7 @@ symlinks: .PHONY .META
                ${INSTALL_SYMLINK} ${TAG_ARGS} ../../../sys/$i/$$h 
${DESTDIR}${INCLUDEDIR}/$i; \
        done
 .endfor
-.for i in 
${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/nand:Ndev/pci}
+.for i in 
${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/hyperv:Ndev/nand:Ndev/pci}
        cd ${.CURDIR}/../sys/$i; \
        for h in *.h; do \
                ${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/$i/$$h 
${DESTDIR}${INCLUDEDIR}/$i; \
@@ -289,6 +292,11 @@ symlinks: .PHONY .META
                ln -fs ../../../../sys/dev/evdev/$$h \
                    ${DESTDIR}${INCLUDEDIR}/dev/evdev; \
        done
+       cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
+       for h in hv_snapshot.h; do \
+               ${INSTALL_SYMLINK} ${TAG_ARGS} 
../../../../sys/dev/hyperv/utilities/$$h \
+                   ${DESTDIR}${INCLUDEDIR}/dev/hyperv; \
+       done
        cd ${.CURDIR}/../sys/dev/pci; \
        for h in pcireg.h; do \
                ${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/pci/$$h \

Modified: stable/11/share/man/man4/Makefile
==============================================================================
--- stable/11/share/man/man4/Makefile   Thu Jan  5 03:13:45 2017        
(r311352)
+++ stable/11/share/man/man4/Makefile   Thu Jan  5 03:20:00 2017        
(r311353)
@@ -194,6 +194,7 @@ MAN=        aac.4 \
        ${_hv_storvsc.4} \
        ${_hv_utils.4} \
        ${_hv_vmbus.4} \
+       hv_vss.4 \
        hwpmc.4 \
        ichsmb.4 \
        ${_ichwd.4} \

Copied: stable/11/share/man/man4/hv_vss.4 (from r308664, 
head/share/man/man4/hv_vss.4)
==============================================================================
--- /dev/null   00:00:00 1970   (empty, because file is newly added)
+++ stable/11/share/man/man4/hv_vss.4   Thu Jan  5 03:20:00 2017        
(r311353, copy of r308664, head/share/man/man4/hv_vss.4)
@@ -0,0 +1,366 @@
+.\" Copyright (c) 2016 Microsoft Corp.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.Dd October 12, 2016
+.Dt HV_VSS 4
+.Os
+.Sh NAME
+.Nm hv_vss
+.Nd Hyper-V Volume Shadow Copy Service API
+.Sh SYNOPSIS
+.In dev/hyperv/hv_snapshot.h
+.Bd -literal
+#define VSS_SUCCESS            0x00000000
+#define VSS_FAIL               0x00000001
+
+enum hv_vss_op_t {
+       HV_VSS_NONE = 0,
+       HV_VSS_CHECK,
+       HV_VSS_FREEZE,
+       HV_VSS_THAW,
+       HV_VSS_COUNT
+};
+
+struct hv_vss_opt_msg {
+       uint32_t        opt;            /* operation */
+       uint32_t        status;         /* 0 for success, 1 for error */
+       uint64_t        msgid;          /* an ID used to identify the 
transaction */
+       uint8_t         reserved[48];   /* reserved values are all zeroes */
+};
+.Ed
+.Sh DESCRIPTION
+The freeze or thaw functionality of application is important to guarantee
+the application consistent backup. On windows platform, VSS is defined to do
+live backup. But for VM guest running on Hyper-V, the corresponding VSS is
+not defined yet. For example, a running database server instance, it knows 
when the
+applications' freeze/thaw should start or finish. But it is not aware of
+the freeze/thaw notification from Hyper-V host. The
+.Nm
+is designed to notify application freeze/thaw request.
+Thus, it plays a role of broker to forward the freeze/thaw command from 
Hyper-V host
+to userland application if it registered VSS service on
+.Fx
+VM, and sends the result back to Hyper-V host.
+.Pp
+Generally,
+.Xr hv_vss_daemon 8
+takes the responsiblity to freeze/thaw UFS file system,
+and it is automatically launched after system boots. When Hyper-V host wants to
+take a snapshot of the
+.Fx
+VM, it will first send VSS capability check to
+.Fx
+VM. The
+.Nm
+received the request and forward the request to userland application if it is
+registered. Only after
+.Nm
+received the VSS_SUCCESS response from application, the
+.Xr hv_vss_daemon 8
+will be informed to check whether file system freeze/thaw is supported. Any 
error
+occurs during this period,
+.Nm
+will inform Hyper-V host that VSS is not supported. In addition, there is a 
default
+timeout limit before sending response to Hyper-V host.
+If the total response time from application and
+.Xr hv_vss_daemon 8
+exceeds this value, timeout
+will occurs and VSS unsupported is responsed to Hyper-V host.
+.Pp
+After Hyper-V host confirmed the
+.Fx
+VM supports VSS, it will send freeze request to VM, and
+.Nm
+will first forward it to application. After application finished freezing, it 
should
+inform
+.Nm
+and file system level freezing will be triggered by
+.Xr hv_vss_daemon 8 . After all freezing
+on both application and
+.Xr hv_vss_daemon 8
+were finished, the
+.Nm
+will inform Hyper-V host that freezing is done. Of course, there is a timeout 
limit as
+same as VSS capability is set to make sure freezing on
+.Fx
+VM is not hang. If there is any error occurs or timeout happened, the freezing 
is failed
+on Hyper-V side.
+.Pp
+Hyper-V host will send thaw request after taking the snapshot, typically, this 
period is
+very short in order not to block the running application.
+.Nm
+firstly thaw the file system by notifying
+.Xr hv_vss_daemon 8 ,
+then notifies user registered
+application. There is also a timeout check before sending response to Hyper-V 
host.
+.Pp
+All the default timeout limit used in VSS capability check, freeze or thaw is 
the same.
+It is 15 seconds currently.
+.Sh NOTES
+.Nm
+only support UFS currently. If any of file system partition is non UFS, the 
VSS capability
+check will fail. If application does not register VSS,
+.Nm
+only support backup for file system level consistent. The device should be 
closed before it
+was opened again. If you want to simultaneously open "/dev/hv_appvss_dev" two 
or more times,
+an error (-1) will be returned, and errno was set.
+.Pp
+If
+.Xr hv_vss_daemon 8
+was killed after system boots, the VSS functionality will not work.
+.Sh EXAMPLES
+The following is a complete example which does nothing except for waiting 2 
seconds when
+receiving those notifications from
+.Nm
+.Bd -literal
+#include <string.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <sys/param.h>
+#include <sys/ucred.h>
+#include <sys/mount.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <poll.h>
+#include <stdint.h>
+#include <syslog.h>
+#include <errno.h>
+#include <err.h>
+#include <fcntl.h>
+#include <ufs/ffs/fs.h>
+#include <paths.h>
+#include <sys/ioccom.h>
+#include <dev/hyperv/hv_snapshot.h>
+
+#define UNDEF_FREEZE_THAW      (0)
+#define FREEZE                 (1)
+#define THAW                   (2)
+#define CHECK                  (3)
+
+#define        VSS_LOG(priority, format, args...) do   {                       
        \\
+               if (is_debugging == 1) {                                        
\\
+                       if (is_daemon == 1)                                     
\\
+                               syslog(priority, format, ## args);              
\\
+                       else                                                    
\\
+                               printf(format, ## args);                        
\\
+               } else {                                                        
\\
+                       if (priority < LOG_DEBUG) {                             
\\
+                               if (is_daemon == 1)                             
\\
+                                       syslog(priority, format, ## args);      
\\
+                               else                                            
\\
+                                       printf(format, ## args);                
\\
+                       }                                                       
\\
+               }                                                               
\\
+       } while(0)
+
+#define CHECK_TIMEOUT          1
+#define CHECK_FAIL             2
+#define FREEZE_TIMEOUT         1
+#define FREEZE_FAIL            2
+#define THAW_TIMEOUT           1
+#define THAW_FAIL              2
+
+static int is_daemon        = 1;
+static int is_debugging     = 0;
+static int simu_opt_waiting = 2; // seconds
+
+#define GENERIC_OPT(TIMEOUT, FAIL)                                             
\\
+       do {                                                                    
\\
+               sleep(simu_opt_waiting);                                        
\\
+               if (opt == CHECK_TIMEOUT) {                                     
\\
+                       sleep(simu_opt_waiting * 10);                           
\\
+                       VSS_LOG(LOG_INFO, "%s timeout simulation\\n",           
\\
+                           __func__);                                          
\\
+                       return (0);                                             
\\
+               } else if (opt == CHECK_FAIL) {                                 
\\
+                       VSS_LOG(LOG_INFO, "%s failure simulation\\n",           
\\
+                           __func__);                                          
\\
+                       return (CHECK_FAIL);                                    
\\
+               } else {                                                        
\\
+                       VSS_LOG(LOG_INFO, "%s success simulation\\n",           
\\
+                           __func__);                                          
\\
+                       return (0);                                             
\\
+               }                                                               
\\
+       } while (0)
+
+static int
+check(int opt)
+{
+       GENERIC_OPT(CHECK_TIMEOUT, CHECK_FAIL);
+}
+
+static int
+freeze(int opt)
+{
+       GENERIC_OPT(FREEZE_TIMEOUT, FREEZE_FAIL);
+}
+
+static int
+thaw(int opt)
+{
+       GENERIC_OPT(THAW_TIMEOUT, THAW_FAIL);
+}
+
+static void usage(const char* cmd) {
+       fprintf(stderr,
+           "%s -f <0|1|2>: simulate app freeze."
+           " 0: successful, 1: freeze timeout, 2: freeze failed\\n"
+           " -c <0|1|2>: simulate vss feature check"
+           " -t <0|1|2>: simulate app thaw."
+           " 0: successful, 1: freeze timeout, 2: freeze failed\\n"
+           " -d : enable debug mode\\n"
+           " -n : run this tool under non-daemon mode\\n", cmd);
+}
+
+int
+main(int argc, char* argv[]) {
+       int ch, freezesimuop = 0, thawsimuop = 0, checksimuop = 0, fd, r, error;
+       uint32_t op;
+       struct pollfd app_vss_fd[1];
+       struct hv_vss_opt_msg  userdata;
+
+       while ((ch = getopt(argc, argv, "f:c:t:dnh")) != -1) {
+               switch (ch) {
+               case 'f':
+                       /* Run as regular process for debugging purpose. */
+                       freezesimuop = (int)strtol(optarg, NULL, 10);
+                       break;
+               case 't':
+                       thawsimuop = (int)strtol(optarg, NULL, 10);
+                       break;
+               case 'c':
+                       checksimuop = (int)strtol(optarg, NULL, 10);
+                       break;
+               case 'd':
+                       is_debugging = 1;
+                       break;
+               case 'n':
+                       is_daemon = 0;
+                       break;
+               case 'h':
+               default:
+                       usage(argv[0]);
+                       exit(0);
+               }
+       }
+
+       openlog("APPVSS", 0, LOG_USER);
+       /* Become daemon first. */
+       if (is_daemon == 1)
+               daemon(1, 0);
+       else
+               VSS_LOG(LOG_DEBUG, "Run as regular process.\\n");
+
+       VSS_LOG(LOG_INFO, "HV_VSS starting; pid is: %d\\n", getpid());
+
+       fd = open(VSS_DEV(APP_VSS_DEV_NAME), O_RDWR);
+       if (fd < 0) {
+               VSS_LOG(LOG_ERR, "Fail to open %s, error: %d %s\\n",
+                   VSS_DEV(APP_VSS_DEV_NAME), errno, strerror(errno));
+               exit(EXIT_FAILURE);
+       }
+       app_vss_fd[0].fd     = fd;
+       app_vss_fd[0].events = POLLIN | POLLRDNORM;
+
+       while (1) {
+               r = poll(app_vss_fd, 1, INFTIM);
+
+               VSS_LOG(LOG_DEBUG, "poll returned r = %d, revent = 0x%x\\n",
+                   r, app_vss_fd[0].revents);
+
+               if (r == 0 || (r < 0 && errno == EAGAIN) ||
+                   (r < 0 && errno == EINTR)) {
+                       /* Nothing to read */
+                       continue;
+               }
+
+               if (r < 0) {
+                       /*
+                        * For poll return failure other than EAGAIN,
+                        * we want to exit.
+                        */
+                       VSS_LOG(LOG_ERR, "Poll failed.\\n");
+                       perror("poll");
+                       exit(EIO);
+               }
+
+               /* Read from character device */
+               error = ioctl(fd, IOCHVVSSREAD, &userdata);
+               if (error < 0) {
+                       VSS_LOG(LOG_ERR, "Read failed.\\n");
+                       perror("pread");
+                       exit(EIO);
+               }
+
+               if (userdata.status != 0) {
+                       VSS_LOG(LOG_ERR, "data read error\\n");
+                       continue;
+               }
+
+               op = userdata.opt;
+
+               switch (op) {
+               case HV_VSS_CHECK:
+                       error = check(checksimuop);
+                       break;
+               case HV_VSS_FREEZE:
+                       error = freeze(freezesimuop);
+                       break;
+               case HV_VSS_THAW:
+                       error = thaw(thawsimuop);
+                       break;
+               default:
+                       VSS_LOG(LOG_ERR, "Illegal operation: %d\\n", op);
+                       error = VSS_FAIL;
+               }
+               if (error)
+                       userdata.status = VSS_FAIL;
+               else
+                       userdata.status = VSS_SUCCESS;
+               error = ioctl(fd, IOCHVVSSWRITE, &userdata);
+               if (error != 0) {
+                       VSS_LOG(LOG_ERR, "Fail to write to device\\n");
+                       exit(EXIT_FAILURE);
+               } else {
+                       VSS_LOG(LOG_INFO, "Send response %d for %s to 
kernel\\n",
+                           userdata.status, op == HV_VSS_FREEZE ? "Freeze" :
+                           (op == HV_VSS_THAW ? "Thaw" : "Check"));
+               }
+       }
+       return 0;
+}
+.Sh SEE ALSO
+.Xr hv_vss_daemon 8 ,
+.Xr hv_utils 4
+.Sh HISTORY
+The daemon was introduced in October 2016 and developed by Microsoft Corp.
+.Sh AUTHORS
+.An -nosplit
+.Fx
+support for
+.Nm
+was first added by
+.An Microsoft BSD Integration Services Team Aq Mt bs...@microsoft.com .

Modified: stable/11/sys/conf/files.amd64
==============================================================================
--- stable/11/sys/conf/files.amd64      Thu Jan  5 03:13:45 2017        
(r311352)
+++ stable/11/sys/conf/files.amd64      Thu Jan  5 03:20:00 2017        
(r311353)
@@ -299,6 +299,7 @@ dev/hyperv/netvsc/if_hn.c                           
optional        hy
 dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c            optional        hyperv
 dev/hyperv/utilities/hv_heartbeat.c                    optional        hyperv
 dev/hyperv/utilities/hv_kvp.c                          optional        hyperv
+dev/hyperv/utilities/hv_snapshot.c                     optional        hyperv
 dev/hyperv/utilities/hv_shutdown.c                     optional        hyperv
 dev/hyperv/utilities/hv_timesync.c                     optional        hyperv
 dev/hyperv/utilities/hv_util.c                         optional        hyperv

Modified: stable/11/sys/conf/files.i386
==============================================================================
--- stable/11/sys/conf/files.i386       Thu Jan  5 03:13:45 2017        
(r311352)
+++ stable/11/sys/conf/files.i386       Thu Jan  5 03:20:00 2017        
(r311353)
@@ -256,6 +256,7 @@ dev/hyperv/netvsc/if_hn.c                           
optional        hy
 dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c            optional        hyperv
 dev/hyperv/utilities/hv_heartbeat.c                    optional        hyperv
 dev/hyperv/utilities/hv_kvp.c                          optional        hyperv
+dev/hyperv/utilities/hv_snapshot.c                     optional        hyperv
 dev/hyperv/utilities/hv_shutdown.c                     optional        hyperv
 dev/hyperv/utilities/hv_timesync.c                     optional        hyperv
 dev/hyperv/utilities/hv_util.c                         optional        hyperv

Copied: stable/11/sys/dev/hyperv/utilities/hv_snapshot.c (from r308664, 
head/sys/dev/hyperv/utilities/hv_snapshot.c)
==============================================================================
--- /dev/null   00:00:00 1970   (empty, because file is newly added)
+++ stable/11/sys/dev/hyperv/utilities/hv_snapshot.c    Thu Jan  5 03:20:00 
2017        (r311353, copy of r308664, 
head/sys/dev/hyperv/utilities/hv_snapshot.c)
@@ -0,0 +1,1061 @@
+/*-
+ * Copyright (c) 2016 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/conf.h>
+#include <sys/uio.h>
+#include <sys/bus.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/taskqueue.h>
+#include <sys/selinfo.h>
+#include <sys/sysctl.h>
+#include <sys/poll.h>
+#include <sys/proc.h>
+#include <sys/queue.h>
+#include <sys/kthread.h>
+#include <sys/syscallsubr.h>
+#include <sys/sysproto.h>
+#include <sys/un.h>
+#include <sys/endian.h>
+#include <sys/sema.h>
+#include <sys/signal.h>
+#include <sys/syslog.h>
+#include <sys/systm.h>
+#include <sys/mutex.h>
+#include <sys/callout.h>
+
+#include <dev/hyperv/include/hyperv.h>
+#include <dev/hyperv/utilities/hv_utilreg.h>
+#include <dev/hyperv/utilities/vmbus_icreg.h>
+
+#include "hv_util.h"
+#include "hv_snapshot.h"
+#include "vmbus_if.h"
+
+#define VSS_MAJOR              5
+#define VSS_MINOR              0
+#define VSS_MSGVER             VMBUS_IC_VERSION(VSS_MAJOR, VSS_MINOR)
+
+#define VSS_FWVER_MAJOR                3
+#define VSS_FWVER              VMBUS_IC_VERSION(VSS_FWVER_MAJOR, 0)
+
+#define TIMEOUT_LIMIT          (15)    // seconds
+enum hv_vss_op {
+       VSS_OP_CREATE = 0,
+       VSS_OP_DELETE,
+       VSS_OP_HOT_BACKUP,
+       VSS_OP_GET_DM_INFO,
+       VSS_OP_BU_COMPLETE,
+       /*
+        * Following operations are only supported with IC version >= 5.0
+        */
+       VSS_OP_FREEZE, /* Freeze the file systems in the VM */
+       VSS_OP_THAW, /* Unfreeze the file systems */
+       VSS_OP_AUTO_RECOVER,
+       VSS_OP_COUNT /* Number of operations, must be last */
+};
+
+/*
+ * Header for all VSS messages.
+ */
+struct hv_vss_hdr {
+       struct vmbus_icmsg_hdr  ic_hdr;
+       uint8_t                 operation;
+       uint8_t                 reserved[7];
+} __packed;
+
+
+/*
+ * Flag values for the hv_vss_check_feature. Here supports only
+ * one value.
+ */
+#define VSS_HBU_NO_AUTO_RECOVERY               0x00000005
+
+struct hv_vss_check_feature {
+       uint32_t flags;
+} __packed;
+
+struct hv_vss_check_dm_info {
+       uint32_t flags;
+} __packed;
+
+struct hv_vss_msg {
+       union {
+               struct hv_vss_hdr vss_hdr;
+       } hdr;
+       union {
+               struct hv_vss_check_feature vss_cf;
+               struct hv_vss_check_dm_info dm_info;
+       } body;
+} __packed;
+
+struct hv_vss_req {
+       struct hv_vss_opt_msg   opt_msg;        /* used to communicate with 
daemon */
+       struct hv_vss_msg       msg;            /* used to communicate with 
host */
+} __packed;
+
+/* hv_vss debug control */
+static int hv_vss_log = 0;
+
+#define        hv_vss_log_error(...)   do {                            \
+       if (hv_vss_log > 0)                                     \
+               log(LOG_ERR, "hv_vss: " __VA_ARGS__);           \
+} while (0)
+
+#define        hv_vss_log_info(...) do {                               \
+       if (hv_vss_log > 1)                                     \
+               log(LOG_INFO, "hv_vss: " __VA_ARGS__);          \
+} while (0)
+
+static const struct vmbus_ic_desc vmbus_vss_descs[] = {
+       {
+               .ic_guid = { .hv_guid = {
+                   0x29, 0x2e, 0xfa, 0x35, 0x23, 0xea, 0x36, 0x42,
+                   0x96, 0xae, 0x3a, 0x6e, 0xba, 0xcb, 0xa4,  0x40} },
+               .ic_desc = "Hyper-V VSS"
+       },
+       VMBUS_IC_DESC_END
+};
+
+static const char * vss_opt_name[] = {"None", "VSSCheck", "Freeze", "Thaw"};
+
+/* character device prototypes */
+static d_open_t                hv_vss_dev_open;
+static d_close_t       hv_vss_dev_close;
+static d_poll_t                hv_vss_dev_daemon_poll;
+static d_ioctl_t       hv_vss_dev_daemon_ioctl;
+
+static d_open_t                hv_appvss_dev_open;
+static d_close_t       hv_appvss_dev_close;
+static d_poll_t                hv_appvss_dev_poll;
+static d_ioctl_t       hv_appvss_dev_ioctl;
+
+/* hv_vss character device structure */
+static struct cdevsw hv_vss_cdevsw =
+{
+       .d_version      = D_VERSION,
+       .d_open         = hv_vss_dev_open,
+       .d_close        = hv_vss_dev_close,
+       .d_poll         = hv_vss_dev_daemon_poll,
+       .d_ioctl        = hv_vss_dev_daemon_ioctl,
+       .d_name         = FS_VSS_DEV_NAME,
+};
+
+static struct cdevsw hv_appvss_cdevsw =
+{
+       .d_version      = D_VERSION,
+       .d_open         = hv_appvss_dev_open,
+       .d_close        = hv_appvss_dev_close,
+       .d_poll         = hv_appvss_dev_poll,
+       .d_ioctl        = hv_appvss_dev_ioctl,
+       .d_name         = APP_VSS_DEV_NAME,
+};
+
+struct hv_vss_sc;
+/*
+ * Global state to track cdev
+ */
+struct hv_vss_dev_sc {
+       /*
+        * msg was transferred from host to notify queue, and
+        * ack queue. Finally, it was recyled to free list.
+        */
+       STAILQ_HEAD(, hv_vss_req_internal)      to_notify_queue;
+       STAILQ_HEAD(, hv_vss_req_internal)      to_ack_queue;
+       struct hv_vss_sc                        *sc;
+       struct proc                             *proc_task;
+       struct selinfo                          hv_vss_selinfo;
+};
+/*
+ * Global state to track and synchronize the transaction requests from the 
host.
+ * The VSS allows user to register their function to do freeze/thaw for 
application.
+ * VSS kernel will notify both vss daemon and user application if it is 
registered.
+ * The implementation state transition is illustrated by:
+ * https://clovertrail.github.io/assets/vssdot.png
+ */
+typedef struct hv_vss_sc {
+       struct hv_util_sc                       util_sc;
+       device_t                                dev;
+
+       struct task                             task;
+
+       /*
+        * mutex is used to protect access of list/queue,
+        * callout in request is also used this mutex.
+        */
+       struct mtx                              pending_mutex;
+       /*
+        * req_free_list contains all free items
+        */
+       LIST_HEAD(, hv_vss_req_internal)        req_free_list;
+
+       /* Indicates if daemon registered with driver */
+       boolean_t                               register_done;
+
+       boolean_t                               app_register_done;
+
+       /* cdev for file system freeze/thaw */
+       struct cdev                             *hv_vss_dev;
+       /* cdev for application freeze/thaw */
+       struct cdev                             *hv_appvss_dev;
+
+       /* sc for app */
+       struct hv_vss_dev_sc                    app_sc;
+       /* sc for deamon */
+       struct hv_vss_dev_sc                    daemon_sc;
+} hv_vss_sc;
+
+typedef struct hv_vss_req_internal {
+       LIST_ENTRY(hv_vss_req_internal)         link;
+       STAILQ_ENTRY(hv_vss_req_internal)       slink;
+       struct hv_vss_req                       vss_req;
+
+       /* Rcv buffer for communicating with the host*/
+       uint8_t                                 *rcv_buf;
+       /* Length of host message */
+       uint32_t                                host_msg_len;
+       /* Host message id */
+       uint64_t                                host_msg_id;
+
+       hv_vss_sc                               *sc;
+
+       struct callout                          callout;
+} hv_vss_req_internal;
+
+#define SEARCH_REMOVE_REQ_LOCKED(reqp, queue, link, tmp, id)           \
+       do {                                                            \
+               STAILQ_FOREACH_SAFE(reqp, queue, link, tmp) {           \
+                       if (reqp->vss_req.opt_msg.msgid == id) {        \
+                               STAILQ_REMOVE(queue,                    \
+                                   reqp, hv_vss_req_internal, link);   \
+                               break;                                  \
+                       }                                               \
+               }                                                       \
+       } while (0)
+
+static bool
+hv_vss_is_daemon_killed_after_launch(hv_vss_sc *sc)
+{
+       return (!sc->register_done && sc->daemon_sc.proc_task);
+}
+
+/*
+ * Callback routine that gets called whenever there is a message from host
+ */
+static void
+hv_vss_callback(struct vmbus_channel *chan __unused, void *context)
+{
+       hv_vss_sc *sc = (hv_vss_sc*)context;
+       if (hv_vss_is_daemon_killed_after_launch(sc))
+               hv_vss_log_info("%s: daemon was killed!\n", __func__);
+       if (sc->register_done || sc->daemon_sc.proc_task) {
+               hv_vss_log_info("%s: Queuing work item\n", __func__);
+               if (hv_vss_is_daemon_killed_after_launch(sc))
+                       hv_vss_log_info("%s: daemon was killed!\n", __func__);
+               taskqueue_enqueue(taskqueue_thread, &sc->task);
+       } else {
+               hv_vss_log_info("%s: daemon has never been registered\n", 
__func__);
+       }
+       hv_vss_log_info("%s: received msg from host\n", __func__);
+}
+/*
+ * Send the response back to the host.
+ */
+static void
+hv_vss_respond_host(uint8_t *rcv_buf, struct vmbus_channel *ch,
+    uint32_t recvlen, uint64_t requestid, uint32_t error)
+{
+       struct vmbus_icmsg_hdr *hv_icmsg_hdrp;
+
+       hv_icmsg_hdrp = (struct vmbus_icmsg_hdr *)rcv_buf;
+
+       hv_icmsg_hdrp->ic_status = error;
+       hv_icmsg_hdrp->ic_flags = HV_ICMSGHDRFLAG_TRANSACTION | 
HV_ICMSGHDRFLAG_RESPONSE;
+
+       error = vmbus_chan_send(ch, VMBUS_CHANPKT_TYPE_INBAND, 0,
+           rcv_buf, recvlen, requestid);
+       if (error)
+               hv_vss_log_info("%s: hv_vss_respond_host: sendpacket 
error:%d\n",
+                   __func__, error);
+}
+
+static void
+hv_vss_notify_host_result_locked(struct hv_vss_req_internal *reqp, uint32_t 
status)
+{
+       struct hv_vss_msg* msg = (struct hv_vss_msg *)reqp->rcv_buf;
+       hv_vss_sc *sc = reqp->sc;
+       if (reqp->vss_req.opt_msg.opt == HV_VSS_CHECK) {
+               msg->body.vss_cf.flags = VSS_HBU_NO_AUTO_RECOVERY;
+       }
+       hv_vss_log_info("%s, %s response %s to host\n", __func__,
+           vss_opt_name[reqp->vss_req.opt_msg.opt],
+           status == HV_S_OK ? "Success" : "Fail");
+       hv_vss_respond_host(reqp->rcv_buf, vmbus_get_channel(reqp->sc->dev),
+           reqp->host_msg_len, reqp->host_msg_id, status);
+       /* recycle the request */
+       LIST_INSERT_HEAD(&sc->req_free_list, reqp, link);
+}
+
+static void
+hv_vss_notify_host_result(struct hv_vss_req_internal *reqp, uint32_t status)
+{
+       mtx_lock(&reqp->sc->pending_mutex);
+       hv_vss_notify_host_result_locked(reqp, status);
+       mtx_unlock(&reqp->sc->pending_mutex);
+}
+
+static void
+hv_vss_cp_vssreq_to_user(struct hv_vss_req_internal *reqp,
+    struct hv_vss_opt_msg *userdata)
+{
+       struct hv_vss_req *hv_vss_dev_buf;
+       hv_vss_dev_buf = &reqp->vss_req;
+       hv_vss_dev_buf->opt_msg.opt = HV_VSS_NONE;
+       switch (reqp->vss_req.msg.hdr.vss_hdr.operation) {
+       case VSS_OP_FREEZE:
+               hv_vss_dev_buf->opt_msg.opt = HV_VSS_FREEZE;
+               break;
+       case VSS_OP_THAW:
+               hv_vss_dev_buf->opt_msg.opt = HV_VSS_THAW;
+               break;
+       case VSS_OP_HOT_BACKUP:
+               hv_vss_dev_buf->opt_msg.opt = HV_VSS_CHECK;
+               break;
+       }
+       *userdata = hv_vss_dev_buf->opt_msg;
+       hv_vss_log_info("%s, read data from user for "
+           "%s (%ju) \n", __func__, vss_opt_name[userdata->opt],
+           (uintmax_t)userdata->msgid);
+}
+
+/**
+ * Remove the request id from app notifiy or ack queue,
+ * and recyle the request by inserting it to free list.
+ *
+ * When app was notified but not yet sending ack, the request
+ * should locate in either notify queue or ack queue.
+ */
+static struct hv_vss_req_internal*
+hv_vss_drain_req_queue_locked(hv_vss_sc *sc, uint64_t req_id)
+{
+       struct hv_vss_req_internal *reqp, *tmp;
+       SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->daemon_sc.to_notify_queue,
+           slink, tmp, req_id);
+       if (reqp == NULL)
+               SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->daemon_sc.to_ack_queue,
+                   slink, tmp, req_id);
+       if (reqp == NULL)
+               SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->app_sc.to_notify_queue,
+                   slink, tmp, req_id);
+       if (reqp == NULL)
+               SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->app_sc.to_ack_queue, slink,
+                   tmp, req_id);
+       return (reqp);
+}
+/**
+ * Actions for daemon who has been notified.
+ */
+static void
+hv_vss_notified(struct hv_vss_dev_sc *dev_sc, struct hv_vss_opt_msg *userdata)
+{
+       struct hv_vss_req_internal *reqp;
+       mtx_lock(&dev_sc->sc->pending_mutex);
+       if (!STAILQ_EMPTY(&dev_sc->to_notify_queue)) {
+               reqp = STAILQ_FIRST(&dev_sc->to_notify_queue);
+               hv_vss_cp_vssreq_to_user(reqp, userdata);
+               STAILQ_REMOVE_HEAD(&dev_sc->to_notify_queue, slink);
+               /* insert the msg to queue for write */
+               STAILQ_INSERT_TAIL(&dev_sc->to_ack_queue, reqp, slink);
+               userdata->status = VSS_SUCCESS;
+       } else {
+               /* Timeout occur, thus request was removed from queue. */
+               hv_vss_log_info("%s: notify queue is empty!\n", __func__);
+               userdata->status = VSS_FAIL;
+       }
+       mtx_unlock(&dev_sc->sc->pending_mutex);
+}
+
+static void
+hv_vss_notify(struct hv_vss_dev_sc *dev_sc, struct hv_vss_req_internal *reqp)

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"

Reply via email to