Federico Simoncelli has uploaded a new change for review. Change subject: [wip] import/export methods ......................................................................
[wip] import/export methods Change-Id: Ib208bb20d88633167b5c66b42febe912439a022d Signed-off-by: Federico Simoncelli <[email protected]> --- M client/vdsClient.py M vdsm.spec.in M vdsm/API.py M vdsm/BindingXMLRPC.py M vdsm/storage/Makefile.am M vdsm/storage/hsm.py M vdsm/storage/image.py A vdsm/storage/imageSharing.py M vdsm/storage/outOfProcess.py M vdsm/storage/remoteFileHandler.py M vdsm/storage/sp.py 11 files changed, 261 insertions(+), 7 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/55/14955/1 diff --git a/client/vdsClient.py b/client/vdsClient.py index 332b438..4d089f1 100644 --- a/client/vdsClient.py +++ b/client/vdsClient.py @@ -18,6 +18,7 @@ # import sys +import ast import getopt import traceback import xmlrpclib @@ -1296,6 +1297,30 @@ return 0, image['uuid'] + def exportImage(self, args): + spUUID, sdUUID, imgUUID, volUUID, exportInfoString = args + exportInfo = ast.literal_eval(exportInfoString) + + image = self.s.exportImage(exportInfo, spUUID, sdUUID, imgUUID, + volUUID) + + if image['status']['code']: + return image['status']['code'], image['status']['message'] + + return 0, image['uuid'] + + def importImage(self, args): + spUUID, sdUUID, imgUUID, volUUID, importInfoString = args + importInfo = ast.literal_eval(importInfoString) + + image = self.s.importImage(importInfo, spUUID, sdUUID, imgUUID, + volUUID) + + if image['status']['code']: + return image['status']['code'], image['status']['message'] + + return 0, image['uuid'] + def moveMultiImage(self, args): spUUID = args[0] srcDomUUID = args[1] @@ -2291,6 +2316,14 @@ 'Synchronize image data between storage domains ' 'within same pool.' )), + 'exportImage': (serv.exportImage, + ('<spUUID> <sdUUID> <imgUUID> <volUUID> <exportInfo>', + '' + )), + 'importImage': (serv.importImage, + ('<spUUID> <sdUUID> <imgUUID> <volUUID> <importInfo>', + '' + )), 'moveMultiImage': (serv.moveMultiImage, ('<spUUID> <srcDomUUID> <dstDomUUID> ' '<imgList>({imgUUID=postzero,' diff --git a/vdsm.spec.in b/vdsm.spec.in index 2f8c735..9c4e4df 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -811,6 +811,7 @@ %{_datadir}/%{vdsm_name}/storage/hba.py* %{_datadir}/%{vdsm_name}/storage/hsm.py* %{_datadir}/%{vdsm_name}/storage/image.py* +%{_datadir}/%{vdsm_name}/storage/imageSharing.py* %{_datadir}/%{vdsm_name}/storage/iscsiadm.py* %{_datadir}/%{vdsm_name}/storage/iscsi.py* %{_datadir}/%{vdsm_name}/storage/localFsSD.py* diff --git a/vdsm/API.py b/vdsm/API.py index ee72116..c9fd67f 100644 --- a/vdsm/API.py +++ b/vdsm/API.py @@ -794,6 +794,14 @@ return self._irs.syncImageData(self._spUUID, self._sdUUID, self._UUID, dstSdUUID, syncType) + def send(self, exportInfo, volUUID=None): + return self._irs.exportImage(self._spUUID, self._sdUUID, self._UUID, + volUUID, exportInfo) + + def receive(self, importInfo, volUUID=None): + return self._irs.importImage(self._spUUID, self._sdUUID, self._UUID, + volUUID, importInfo) + class LVMVolumeGroup(APIBase): ctorArgs = ['lvmvolumegroupID'] diff --git a/vdsm/BindingXMLRPC.py b/vdsm/BindingXMLRPC.py index 9a4db12..fa12dcd 100644 --- a/vdsm/BindingXMLRPC.py +++ b/vdsm/BindingXMLRPC.py @@ -502,6 +502,14 @@ image = API.Image(imgUUID, spUUID, sdUUID) return image.syncData(dstSdUUID, syncType) + def imageExport(self, exportInfo, spUUID, sdUUID, imgUUID, volUUID=None): + image = API.Image(imgUUID, spUUID, sdUUID) + return image.send(exportInfo, volUUID) + + def imageImport(self, importInfo, spUUID, sdUUID, imgUUID, volUUID=None): + image = API.Image(imgUUID, spUUID, sdUUID) + return image.receive(importInfo, volUUID) + def poolConnect(self, spUUID, hostID, scsiKey, msdUUID, masterVersion, options=None): pool = API.StoragePool(spUUID) @@ -837,6 +845,8 @@ (self.imageMove, 'moveImage'), (self.imageCloneStructure, 'cloneImageStructure'), (self.imageSyncData, 'syncImageData'), + (self.imageExport, 'exportImage'), + (self.imageImport, 'importImage'), (self.poolConnect, 'connectStoragePool'), (self.poolConnectStorageServer, 'connectStorageServer'), (self.poolCreate, 'createStoragePool'), diff --git a/vdsm/storage/Makefile.am b/vdsm/storage/Makefile.am index 5173277..bcfd9c7 100644 --- a/vdsm/storage/Makefile.am +++ b/vdsm/storage/Makefile.am @@ -38,6 +38,7 @@ hba.py \ hsm.py \ image.py \ + imageSharing.py \ iscsiadm.py \ iscsi.py \ localFsSD.py \ diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index 15b8331..ae5c609 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -1602,6 +1602,28 @@ self._spmSchedule(spUUID, "syncImageData", pool.syncImageData, sdUUID, imgUUID, dstSdUUID, syncType) + @public + def exportImage(self, spUUID, sdUUID, imgUUID, volUUID, exportInfo): + """ + """ + self.validateSdUUID(sdUUID) + + pool = self.getPool(spUUID) + # NOTE: this could become an hsm task + self._spmSchedule(spUUID, "exportImage", pool.exportImage, sdUUID, + imgUUID, volUUID, exportInfo) + + @public + def importImage(self, spUUID, sdUUID, imgUUID, volUUID, importInfo): + """ + """ + self.validateSdUUID(sdUUID) + + pool = self.getPool(spUUID) + # NOTE: this could become an hsm task + self._spmSchedule(spUUID, "importImage", pool.importImage, sdUUID, + imgUUID, volUUID, importInfo) + @deprecated @public def moveMultipleImages(self, spUUID, srcDomUUID, dstDomUUID, imgDict, diff --git a/vdsm/storage/image.py b/vdsm/storage/image.py index 5f39c91..8f81c8a 100644 --- a/vdsm/storage/image.py +++ b/vdsm/storage/image.py @@ -1265,3 +1265,41 @@ self.log.info("Merge src=%s with dst=%s was successfully finished.", srcVol.getVolumePath(), dstVol.getVolumePath()) + + def _activateVolumeForImportExport(self, domain, imgUUID, volUUID=None): + chain = self.getChain(domain.sdUUID, imgUUID, volUUID) + template = chain[0].getParentVolume() + + if template or len(chain) > 1: + self.log.error("Importing and exporting an image with more " + "than one volume is not supported") + raise se.CopyImageError() + + domain.activateVolumes(imgUUID, volUUIDs=[chain[0].volUUID]) + return chain[0] + + def send(self, exportInfo, sdUUID, imgUUID, volUUID=None): + domain = sdCache.produce(sdUUID) + expVol = self._activateVolumeForImportExport(domain, imgUUID, volUUID) + + try: + expVolPath = expVol.getVolumePath() + # In the future based on the exportInfo data we'll select + # an appropriate backend to export to. Since now we're just + # supporting glance we hardwire it directly. + return domain.oop.imageSharing.exportImage(expVolPath, exportInfo) + finally: + domain.deactivateVolumes(imgUUID, volUUIDs=[expVol.volUUID]) + + def receive(self, importInfo, sdUUID, imgUUID, volUUID=None): + domain = sdCache.produce(sdUUID) + impVol = self._activateVolumeForImportExport(domain, imgUUID, volUUID) + + try: + impVolPath = impVol.getVolumePath() + # In the future based on the importInfo data we'll select + # an appropriate backend to import from. Since now we're just + # supporting glance we hardwire it directly. + return domain.oop.imageSharing.importImage(impVolPath, importInfo) + finally: + domain.deactivateVolumes(imgUUID, volUUIDs=[impVol.volUUID]) diff --git a/vdsm/storage/imageSharing.py b/vdsm/storage/imageSharing.py new file mode 100644 index 0000000..92d2122 --- /dev/null +++ b/vdsm/storage/imageSharing.py @@ -0,0 +1,120 @@ +# Copyright 2013 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +import types +import logging +import httplib +from urlparse import urlparse + + +log = logging.getLogger("Storage.ImageSharing") + + +# Decorators used to mark the import/export methods +class ImportMethod(object): + def __init__(self, method): + self.method = method + + def __call__(self, fun): + fun._importMethod = self.method + return fun + + +class ExportMethod(object): + def __init__(self, method): + self.method = method + + def __call__(self, fun): + fun._exportMethod = self.method + return fun + + +# HTTP import/export section +HTTP_READ_BUFFER = 262144 # 256K + + +def _httpClientConnect(reqAction, shareInfo): + urlInfo = urlparse(shareInfo.get('url')) + + if urlInfo.scheme == "https": + client = httplib.HTTPSConnection(urlInfo.hostname, urlInfo.port) + elif urlInfo.scheme == "http": + client = httplib.HTTPConnection(urlInfo.hostname, urlInfo.port) + else: + raise RuntimeError("Unsupported scheme %s" % urlInfo.scheme) + + client.connect() + client.putrequest(reqAction, urlInfo.path) + + headers = shareInfo.get("headers", {}) + + for headerName, headerValue in headers.iteritems(): + client.putheader(headerName, headerValue) + + client.endheaders() + return client + + +@ImportMethod("http") +def httpImportImage(dstImgPath, importInfo): + client = _httpClientConnect("GET", importInfo) + response = client.getresponse() + + with open(dstImgPath, 'wb') as image: + while True: + data = response.read(HTTP_READ_BUFFER) + if len(data) == 0: + break # completed successfully + image.write(data) + + +@ExportMethod("http") +def httpExportImage(srcImgPath, exportInfo): + client = _httpClientConnect("POST", exportInfo) + response = client.getresponse() + + with open(srcImgPath, 'rb') as image: + while True: + data = image.read(HTTP_READ_BUFFER) + if len(data) == 0: + break # completed successfully + response.write(HTTP_READ_BUFFER) + + +# Official entry points for importing and exporting +def importImage(dstImgPath, importInfo): + # NOTE: don't use iteritems as the dictionary might change size + # during the iteration. + reqMethod = importInfo.get("method") + for functionName, functionImpl in globals().items(): + if (isinstance(functionImpl, types.FunctionType) and + getattr(functionImpl, "_importMethod", None) == reqMethod): + return functionImpl(dstImgPath, importInfo) + raise RuntimeError("Import method %s not found" % reqMethod) + + +def exportImage(srcImgPath, exportInfo): + # NOTE: don't use iteritems as the dictionary might change size + # during the iteration. + reqMethod = exportInfo.get("method") + for functionName, functionImpl in globals().items(): + if (isinstance(functionImpl, types.FunctionType) and + getattr(functionImpl, "_exportMethod", None) == reqMethod): + return functionImpl(srcImgPath, exportInfo) + raise RuntimeError("Export method %s not found" % reqMethod) diff --git a/vdsm/storage/outOfProcess.py b/vdsm/storage/outOfProcess.py index bf61d54..8e0377a 100644 --- a/vdsm/storage/outOfProcess.py +++ b/vdsm/storage/outOfProcess.py @@ -79,9 +79,7 @@ def OopWrapper(procPool): - return _ModuleWrapper("oop", procPool, DEFAULT_TIMEOUT, - (("os", - ("path",)), - "glob", - "fileUtils", - "utils")) + return _ModuleWrapper("oop", procPool, DEFAULT_TIMEOUT, ( + ("os", ("path",)), "glob", "fileUtils", "utils", "fileUtils", + "imageSharing") + ) diff --git a/vdsm/storage/remoteFileHandler.py b/vdsm/storage/remoteFileHandler.py index 9c19b1c..c69c136 100644 --- a/vdsm/storage/remoteFileHandler.py +++ b/vdsm/storage/remoteFileHandler.py @@ -46,6 +46,7 @@ import misc import fileUtils +import imageSharing import zombieReaper from vdsm import utils @@ -404,7 +405,7 @@ server.registerFunction(func) - for mod in (os, glob, fileUtils): + for mod in (os, glob, fileUtils, imageSharing): server.registerModule(mod) server.registerModule(utils, name="utils") except Exception: diff --git a/vdsm/storage/sp.py b/vdsm/storage/sp.py index ed79808..caeee0f 100644 --- a/vdsm/storage/sp.py +++ b/vdsm/storage/sp.py @@ -1864,6 +1864,28 @@ image.Image(self.poolPath).syncData( sdUUID, imgUUID, dstSdUUID, syncType) + def exportImage(self, sdUUID, imgUUID, volUUID, exportInfo): + """ + """ + imgResourceLock = rmanager.acquireResource( + sd.getNamespace(sdUUID, IMAGE_NAMESPACE), imgUUID, + rm.LockType.shared) + + with imgResourceLock: + return image.Image(self.poolPath).send( + exportInfo, sdUUID, imgUUID, volUUID) + + def importImage(self, sdUUID, imgUUID, volUUID, importInfo): + """ + """ + imgResourceLock = rmanager.acquireResource( + sd.getNamespace(sdUUID, IMAGE_NAMESPACE), imgUUID, + rm.LockType.exclusive) + + with imgResourceLock: + return image.Image(self.poolPath).receive( + importInfo, sdUUID, imgUUID, volUUID) + def moveMultipleImages(self, srcDomUUID, dstDomUUID, imgDict, vmUUID, force): """ -- To view, visit http://gerrit.ovirt.org/14955 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ib208bb20d88633167b5c66b42febe912439a022d Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Federico Simoncelli <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
