Adam Litke has uploaded a new change for review. Change subject: StorageDomainManifest: move image and volume enumeration ......................................................................
StorageDomainManifest: move image and volume enumeration Change-Id: I56214a7024b1f894b8e4e0b951faf0567a1606c9 Signed-off-by: Adam Litke <[email protected]> --- M vdsm/storage/blockSD.py M vdsm/storage/fileSD.py M vdsm/storage/sd.py 3 files changed, 137 insertions(+), 132 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/69/42269/1 diff --git a/vdsm/storage/blockSD.py b/vdsm/storage/blockSD.py index 273cf51..d685313 100644 --- a/vdsm/storage/blockSD.py +++ b/vdsm/storage/blockSD.py @@ -628,6 +628,43 @@ deleteVolumes(sdUUID, toDel) self.rmDCImgDir(imgUUID, volsImgs) + def getAllVolumesImages(self): + """ + Return all the images that depend on a volume. + + Return dicts: + vols = {volUUID: ([imgUUID1, imgUUID2], parentUUID)]} + for complete images. + remnants (same) for broken imgs, orphan volumes, etc. + """ + vols = {} # The "legal" volumes: not half deleted/removed volumes. + remnants = {} # Volumes which are part of failed image deletes. + allVols = getAllVolumes(self.sdUUID) + for volName, ip in allVols.iteritems(): + if (volName.startswith(sd.REMOVED_IMAGE_PREFIX) or + ip.imgs[0].startswith(sd.REMOVED_IMAGE_PREFIX)): + remnants[volName] = ip + else: + # Deleted images are not dependencies of valid volumes. + images = [img for img in ip.imgs + if not img.startswith(sd.REMOVED_IMAGE_PREFIX)] + vols[volName] = sd.ImgsPar(images, ip.parent) + return vols, remnants + + def getAllVolumes(self): + vols, rems = self.getAllVolumesImages() + return vols + + def getAllImages(self): + """ + Get the set of all images uuids in the SD. + """ + vols = self.getAllVolumes() # {volName: ([imgs], parent)} + images = set() + for imgs, parent in vols.itervalues(): + images.update(imgs) + return images + class BlockStorageDomain(sd.StorageDomain): manifestClass = BlockStorageDomainManifest @@ -1020,16 +1057,6 @@ mdavalid=vgMetadataStatus['mdavalid'], mdathreshold=vgMetadataStatus['mdathreshold']) - def getAllImages(self): - """ - Get the set of all images uuids in the SD. - """ - vols = self.getAllVolumes() # {volName: ([imgs], parent)} - images = set() - for imgs, parent in vols.itervalues(): - images.update(imgs) - return images - def rmDCImgDir(self, imgUUID, volsImgs): return self._manifest.rmDCImgDir(imgUUID, volsImgs) @@ -1057,37 +1084,6 @@ allVols = self.getAllVolumes() volUUIDs = self._getImgExclusiveVols(imgUUID, allVols) lvm.deactivateLVs(self.sdUUID, volUUIDs) - - def getAllVolumesImages(self): - """ - Return all the images that depend on a volume. - - Return dicts: - vols = {volUUID: ([imgUUID1, imgUUID2], parentUUID)]} - for complete images. - remnants (same) for broken imgs, orphan volumes, etc. - """ - vols = {} # The "legal" volumes: not half deleted/removed volumes. - remnants = {} # Volumes which are part of failed image deletes. - allVols = getAllVolumes(self.sdUUID) - for volName, ip in allVols.iteritems(): - if (volName.startswith(sd.REMOVED_IMAGE_PREFIX) or - ip.imgs[0].startswith(sd.REMOVED_IMAGE_PREFIX)): - remnants[volName] = ip - else: - # Deleted images are not dependencies of valid volumes. - images = [img for img in ip.imgs - if not img.startswith(sd.REMOVED_IMAGE_PREFIX)] - vols[volName] = sd.ImgsPar(images, ip.parent) - return vols, remnants - - def getAllVolumes(self): - vols, rems = self.getAllVolumesImages() - return vols - - def getAllRemnants(self): - vols, rems = self.getAllVolumesImages() - return rems def linkBCImage(self, imgPath, imgUUID): dst = self.getLinkBCImagePath(imgUUID) diff --git a/vdsm/storage/fileSD.py b/vdsm/storage/fileSD.py index 9f44342..07fc1c6 100644 --- a/vdsm/storage/fileSD.py +++ b/vdsm/storage/fileSD.py @@ -213,6 +213,87 @@ self.log.error("removed image dir: %s can't be removed", toDelDir) raise se.ImageDeleteError("%s %s" % (imgUUID, str(e))) + def getAllVolumes(self): + """ + Return dict {volUUID: ((imgUUIDs,), parentUUID)} of the domain. + + (imgUUIDs,) is a tuple of all the images that contain a certain + volUUID. For non-templates volumes, this tuple consists of a single + image. For template volume it consists of all the images that are + based on the template volume. In that case, the first imgUUID in the + tuple is the self-image of the template. + + The parent of a non-template volume cannot be determined in file domain + without reading the metadata. However, in order to have an output + compatible to block domain, we report parent as None. + + Template volumes have no parent, and thus we report BLANK_UUID as their + parentUUID. + """ + volMetaPattern = os.path.join(self.mountpoint, self.sdUUID, + sd.DOMAIN_IMAGES, "*", "*.meta") + volMetaPaths = self.oop.glob.glob(volMetaPattern) + + # First create mapping from images to volumes + images = collections.defaultdict(list) + for metaPath in volMetaPaths: + head, tail = os.path.split(metaPath) + volUUID, volExt = os.path.splitext(tail) + imgUUID = os.path.basename(head) + images[imgUUID].append(volUUID) + + # Using images to volumes mapping, we can create volumes to images + # mapping, detecting template volumes and template images, based on + # these rules: + # + # Template volumes are hard linked in every image directory + # which is derived from that template, therefore: + # + # 1. A template volume which is in use will appear at least twice + # (in the template image dir and in the derived image dir) + # + # 2. Any volume which appears more than once in the dir tree is + # by definition a template volume. + # + # 3. Any image which has more than 1 volume is not a template + # image. + + volumes = {} + for imgUUID, volUUIDs in images.iteritems(): + for volUUID in volUUIDs: + if volUUID in volumes: + # This must be a template volume (rule 2) + volumes[volUUID]['parent'] = sd.BLANK_UUID + if len(volUUIDs) > 1: + # This image is not a template (rule 3) + volumes[volUUID]['imgs'].append(imgUUID) + else: + # This image is a template (rule 3) + volumes[volUUID]['imgs'].insert(0, imgUUID) + else: + volumes[volUUID] = {'imgs': [imgUUID], 'parent': None} + + return dict((k, sd.ImgsPar(tuple(v['imgs']), v['parent'])) + for k, v in volumes.iteritems()) + + def getAllImages(self): + """ + Fetch the set of the Image UUIDs in the SD. + """ + # Get Volumes of an image + pattern = os.path.join(sd.storage_repository, + # ISO domains don't have images, + # we can assume single domain + self.getPools()[0], + self.sdUUID, sd.DOMAIN_IMAGES) + pattern = os.path.join(pattern, constants.UUID_GLOB_PATTERN) + files = self.oop.glob.glob(pattern) + images = set() + for i in files: + if self.oop.os.path.isdir(i): + images.add(os.path.basename(i)) + return images + class FileStorageDomain(sd.StorageDomain): manifestClass = FileStorageDomainManifest @@ -390,24 +471,6 @@ def validateMasterMount(self): return self.oop.fileUtils.pathExists(self.getMasterDir()) - def getAllImages(self): - """ - Fetch the set of the Image UUIDs in the SD. - """ - # Get Volumes of an image - pattern = os.path.join(self.storage_repository, - # ISO domains don't have images, - # we can assume single domain - self.getPools()[0], - self.sdUUID, sd.DOMAIN_IMAGES) - pattern = os.path.join(pattern, constants.UUID_GLOB_PATTERN) - files = self.oop.glob.glob(pattern) - images = set() - for i in files: - if self.oop.os.path.isdir(i): - images.add(os.path.basename(i)) - return images - def zeroImage(self, sdUUID, imgUUID, volsImgs): self.log.warning("image %s on a fileSD %s won't be zeroed." % (imgUUID, sdUUID)) @@ -420,69 +483,6 @@ imgUUID: the image to be deactivated. """ pass - - def getAllVolumes(self): - """ - Return dict {volUUID: ((imgUUIDs,), parentUUID)} of the domain. - - (imgUUIDs,) is a tuple of all the images that contain a certain - volUUID. For non-templates volumes, this tuple consists of a single - image. For template volume it consists of all the images that are - based on the template volume. In that case, the first imgUUID in the - tuple is the self-image of the template. - - The parent of a non-template volume cannot be determined in file domain - without reading the metadata. However, in order to have an output - compatible to block domain, we report parent as None. - - Template volumes have no parent, and thus we report BLANK_UUID as their - parentUUID. - """ - volMetaPattern = os.path.join(self.mountpoint, self.sdUUID, - sd.DOMAIN_IMAGES, "*", "*.meta") - volMetaPaths = self.oop.glob.glob(volMetaPattern) - - # First create mapping from images to volumes - images = collections.defaultdict(list) - for metaPath in volMetaPaths: - head, tail = os.path.split(metaPath) - volUUID, volExt = os.path.splitext(tail) - imgUUID = os.path.basename(head) - images[imgUUID].append(volUUID) - - # Using images to volumes mapping, we can create volumes to images - # mapping, detecting template volumes and template images, based on - # these rules: - # - # Template volumes are hard linked in every image directory - # which is derived from that template, therefore: - # - # 1. A template volume which is in use will appear at least twice - # (in the template image dir and in the derived image dir) - # - # 2. Any volume which appears more than once in the dir tree is - # by definition a template volume. - # - # 3. Any image which has more than 1 volume is not a template - # image. - - volumes = {} - for imgUUID, volUUIDs in images.iteritems(): - for volUUID in volUUIDs: - if volUUID in volumes: - # This must be a template volume (rule 2) - volumes[volUUID]['parent'] = sd.BLANK_UUID - if len(volUUIDs) > 1: - # This image is not a template (rule 3) - volumes[volUUID]['imgs'].append(imgUUID) - else: - # This image is a template (rule 3) - volumes[volUUID]['imgs'].insert(0, imgUUID) - else: - volumes[volUUID] = {'imgs': [imgUUID], 'parent': None} - - return dict((k, sd.ImgsPar(tuple(v['imgs']), v['parent'])) - for k, v in volumes.iteritems()) def linkBCImage(self, imgPath, imgUUID): # Nothing to do here other than returning the path diff --git a/vdsm/storage/sd.py b/vdsm/storage/sd.py index 873c8fa..9e81917 100644 --- a/vdsm/storage/sd.py +++ b/vdsm/storage/sd.py @@ -351,6 +351,18 @@ def getFormat(self): return str(self.getVersion()) + def getPools(self): + try: + pools = self.getMetaParam(key=DMDK_POOLS) + except KeyError: + pools = [] + else: + # Old pool MD marked SDs not belonging to any pool with + # BLANK_UUID as the pool uuid. + if BLANK_UUID in pools: + pools.remove(BLANK_UUID) + return pools + class StorageDomain(object): log = logging.getLogger("Storage.StorageDomain") @@ -406,6 +418,12 @@ def deleteImage(self, sdUUID, imgUUID, volsImgs): self._manifest.deleteImage(sdUUID, imgUUID, volsImgs) + + def getAllImages(self): + return self._manifest.getAllImages() + + def getAllVolumes(self): + return self._manifest.getAllVolumes() def prepareMailbox(self): """ @@ -542,16 +560,7 @@ return self._manifest.getFormat() def getPools(self): - try: - pools = self.getMetaParam(key=DMDK_POOLS) - except KeyError: - pools = [] - else: - # Old pool MD marked SDs not belonging to any pool with - # BLANK_UUID as the pool uuid. - if BLANK_UUID in pools: - pools.remove(BLANK_UUID) - return pools + return self._manifest.getPools() def getIdsFilePath(self): return self._manifest.getIdsFilePath() -- To view, visit https://gerrit.ovirt.org/42269 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I56214a7024b1f894b8e4e0b951faf0567a1606c9 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Adam Litke <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
