commit: 691dc445082cc455159d699498a714d40754fedd
Author: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 7 18:27:24 2014 +0000
Commit: Devan Franchini <twitch153 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 15 20:57:38 2014 +0000
URL:
http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=691dc445
Centralizes common archive overlay code
This commit brings archive overlay code to one common place, in
an attempt to make maintaining and adding archive overlay types
to layman much easier.
config.py: Alters the clean_tar option to clean_archive.
tar.py: Removes doctest tests from tar.py as there is already an
existing external.py test.
---
layman/config.py | 2 +-
layman/constants.py | 11 +++
layman/overlays/archive.py | 197 +++++++++++++++++++++++++++++++++++++++++++++
layman/overlays/tar.py | 179 ++++++----------------------------------
4 files changed, 235 insertions(+), 154 deletions(-)
diff --git a/layman/config.py b/layman/config.py
index 4849d5e..91c179e 100644
--- a/layman/config.py
+++ b/layman/config.py
@@ -100,7 +100,7 @@ class BareConfig(object):
'check_official': 'Yes',
'conf_type': 'make.conf',
'require_repoconfig': 'Yes',
- 'clean_tar': 'yes',
+ 'clean_archive': 'yes',
'make_conf' : '%(storage)s/make.conf',
'repos_conf': path([self.root,
EPREFIX,'/etc/portage/repos.conf/layman.conf']),
'conf_module': ['make_conf', 'repos_conf'],
diff --git a/layman/constants.py b/layman/constants.py
index 24d1f36..c526cb6 100644
--- a/layman/constants.py
+++ b/layman/constants.py
@@ -66,3 +66,14 @@ SUCCEED = 0
COMPONENT_DEFAULTS = ['name', 'descriptions', 'owner', 'type', 'sources']
POSSIBLE_COMPONENTS = ['name', 'descriptions', 'homepage', 'owner', 'quality',
'priority', 'sources', 'branch', 'irc', 'feeds']
+
+
+###############################################################################
+##
+## Archive overlay possible file extensions
+##
+###############################################################################
+
+FILE_EXTENSIONS = {'Tar': ('bz2', 'gz', 'lzma', 'xz', 'Z', 'tgz', 'tbz', 'taz',
+ 'tlz', 'txz')
+ }
diff --git a/layman/overlays/archive.py b/layman/overlays/archive.py
new file mode 100644
index 0000000..68c8b47
--- /dev/null
+++ b/layman/overlays/archive.py
@@ -0,0 +1,197 @@
+#!/usr/bin/python
+from __future__ import unicode_literals
+
+import os
+import sys
+import shutil
+import tempfile
+
+import xml.etree.ElementTree as ET # Python 2.5
+
+from layman.constants import MOUNT_TYPES
+from layman.compatibility import fileopen
+from layman.overlays.source import OverlaySource, require_supported
+from layman.utils import path
+from layman.version import VERSION
+from sslfetch.connections import Connector
+
+USERAGENT = "Layman-" + VERSION
+
+class ArchiveOverlay(OverlaySource):
+
+ type = 'Archive'
+ type_key = 'archive'
+
+ def __init__(self, parent, config, _location, ignore = 0):
+
+ super(ArchiveOverlay, self).__init__(parent,
+ config, _location, ignore)
+
+ self.clean_archive = config['clean_archive']
+ self.output = config['output']
+ self.proxies = config.proxies
+ self.branch = self.parent.branch
+ self.mount_me = bool(self.type in MOUNT_TYPES)
+
+
+ def _fetch(self, base, archive_url, dest_dir):
+ '''
+ Fetches overlay source archive.
+
+ @params base: string of directory base for installed overlays.
+ @params archive_url: string of URL where archive is located.
+ @params dest_dir: string of destination of extracted archive.
+ @rtype tuple (str of package location, bool to clean_archive)
+ '''
+ ext = self.get_extension()
+
+ if 'file://' not in archive_url:
+ # set up ssl-fetch output map
+ connector_output = {
+ 'info': self.output.debug,
+ 'error': self.output.error,
+ 'kwargs-info': {'level': 2},
+ 'kwargs-error': {'level': None},
+ }
+
+ fetcher = Connector(connector_output, self.proxies, USERAGENT)
+
+ success, archive, timestamp = fetcher.fetch_content(archive_url)
+
+ pkg = path([base, self.parent.name + ext])
+
+ try:
+ with fileopen(pkg, 'w+b') as out_file:
+ out_file.write(archive)
+
+ except Exception as error:
+ raise Exception('Failed to store archive package in '\
+ '%(pkg)s\nError was: %(error)s'\
+ % ({'pkg': pkg, 'error': error}))
+
+ else:
+ self.clean_archive = False
+ pkg = archive_url.replace('file://', '')
+
+ return pkg
+
+
+ def _add_unchecked(self, base):
+ def try_to_wipe(folder):
+ if not os.path.exists(folder):
+ return
+
+ try:
+ self.output.info('Deleting directory %(dir)s'\
+ % ({'dir': folder}), 2)
+ shutil.rmtree(folder)
+ except Exception as error:
+ raise Exception('Failed to remove unneccessary archive '\
+ 'structure %(dir)s\nError was: %(err)s'\
+ % ({'dir': folder, 'err': error}))
+
+ final_path = path([base, self.parent.name])
+ try:
+ if not self.mount_me:
+ temp_path = tempfile.mkdtemp(dir=base)
+ else:
+ temp_path = final_path
+ if not os.path.exists(temp_path):
+ os.mkdir(temp_path)
+ pkg = self._fetch(base=base, archive_url=self.src,
+ dest_dir=temp_path)
+ result = self.post_fetch(pkg, temp_path)
+ if self.clean_archive:
+ os.unlink(pkg)
+ except Exception as error:
+ try_to_wipe(temp_path)
+ raise error
+
+ if result == 0 and not self.mount_me:
+ if self.branch:
+ source = temp_path + os.path.sep + self.branch
+ else:
+ source = temp_path
+
+ if os.path.exists(source):
+ if os.path.exists(final_path):
+ self.delete(base)
+
+ try:
+ os.rename(source, final_path)
+ except Exception as error:
+ raise Exception('Failed to rename archive subdirectory '\
+ '%(src)s to %(path)s\nError was: %(err)s'\
+ % ({'src': source, 'path': final_path, 'err': error}))
+ os.chmod(final_path, 0o755)
+ else:
+ raise Exception('The given path (branch setting in the xml)\n'\
+ ' %(src)s does not exist in this archive package!'\
+ % ({'src': source}))
+
+ if not self.mount_me:
+ try_to_wipe(temp_path)
+
+ return result
+
+
+ def add(self, base):
+ '''
+ Add overlay.
+
+ @params base: string location where overlays are installed.
+ @rtype bool
+ '''
+
+ if not self.supported():
+ return 1
+
+ target = path([base, self.parent.name])
+
+ if os.path.exists(target):
+ raise Exception('Directory %(dir)s already exists. Will not '\
+ 'overwrite its contents!' % ({'dir': target}))
+
+ return self.postsync(
+ self._add_unchecked(base),
+ cwd=target)
+
+
+ def sync(self, base):
+ '''
+ Sync overlay.
+
+ @params base: string location where overlays are installed.
+ @rtype bool
+ '''
+
+ if not self.supported():
+ return 1
+
+ target = path([base, self.parent.name])
+
+ return self.postsync(
+ self._add_unchecked(base),
+ cwd=target)
+
+
+ def supported(self):
+ '''
+ Determines if overlay type is supported.
+
+ @rtype bool
+ '''
+
+ return self.is_supported()
+
+
+if __name__ == '__main__':
+ import doctest
+
+ # Ignore warnings here. We are just testing.
+ from warnings import filterwarnings, resetwarnings
+ filterwarnings('ignore')
+
+ doctest.testmod(sys.modules[__name__])
+
+ resetwarnings()
diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py
index 4999e20..7d7bf89 100644
--- a/layman/overlays/tar.py
+++ b/layman/overlays/tar.py
@@ -26,21 +26,11 @@ __version__ = "$Id: tar.py 310 2007-04-09 16:30:40Z wrobel
$"
#
#-------------------------------------------------------------------------------
-import os
-import os.path
import sys
-import shutil
-import tempfile
-import xml.etree.ElementTree as ET # Python 2.5
-
-from layman.compatibility import fileopen
-from layman.overlays.source import OverlaySource, require_supported
-from layman.utils import path
-from layman.version import VERSION
-from sslfetch.connections import Connector
-
-USERAGENT = "Layman" + VERSION
+from layman.constants import FILE_EXTENSIONS
+from layman.overlays.archive import ArchiveOverlay
+from layman.overlays.source import require_supported
#===============================================================================
#
@@ -48,174 +38,57 @@ USERAGENT = "Layman" + VERSION
#
#-------------------------------------------------------------------------------
-class TarOverlay(OverlaySource):
- ''' Handles tar overlays.
-
- >>> from layman.output import Message
- >>> import xml.etree.ElementTree as ET # Python 2.5
- >>> repo = ET.Element('repo')
- >>> repo_name = ET.Element('name')
- >>> repo_name.text = 'dummy'
- >>> desc = ET.Element('description')
- >>> desc.text = 'Dummy description'
- >>> owner = ET.Element('owner')
- >>> owner_email = ET.Element('email')
- >>> owner_email.text = 'du...@example.org'
- >>> owner[:] = [owner_email]
- >>> source = ET.Element('source', type='tar')
- >>> here = os.path.dirname(os.path.realpath(__file__))
- >>> source.text = 'file://' + here +
'/../tests/testfiles/layman-test.tar.bz2'
- >>> branch = ET.Element('branch')
- >>> branch.text = 'layman-test'
- >>> repo[:] = [repo_name, desc, owner, source, branch]
- >>> from layman.config import BareConfig
- >>> config = BareConfig()
- >>> import tempfile
- >>> testdir = tempfile.mkdtemp(prefix="laymantmp_")
- >>> from layman.overlays.overlay import Overlay
- >>> a = Overlay(config, repo)
- >>> config['output'].set_colorize(False)
- >>> a.add(testdir)
- 0
- >>> os.listdir(testdir + '/dummy/')
- ['layman-test']
- >>> sorted(os.listdir(testdir + '/dummy/layman-test/'))
- ['app-admin', 'app-portage']
- >>> shutil.rmtree(testdir)
- '''
+class TarOverlay(ArchiveOverlay):
+ '''Handles tar overlays.'''
type = 'Tar'
type_key = 'tar'
- def __init__(self, parent, config, _location, ignore = 0):
+ def __init__(self, parent, config, _location, ignore=0):
super(TarOverlay, self).__init__(parent,
config, _location, ignore)
self.output = config['output']
- self.proxies = config.proxies
- self.branch = self.parent.branch
- def _extract(self, base, tar_url, dest_dir):
+ def get_extension(self):
+ '''
+ Determines tar file extension.
+
+ @rtype str
+ '''
ext = '.tar.noidea'
- clean_tar = self.config['clean_tar']
- for i in [('tar.%s' % e) for e in ('bz2', 'gz', 'lzma', 'xz', 'Z')] \
- + ['tgz', 'tbz', 'taz', 'tlz', 'txz']:
+ for i in [('tar.%s' % e) for e in FILE_EXTENSIONS[self.type]:
candidate_ext = '.%s' % i
if self.src.endswith(candidate_ext):
ext = candidate_ext
break
- if 'file://' not in tar_url:
- # setup the ssl-fetch output map
- connector_output = {
- 'info': self.output.debug,
- 'error': self.output.error,
- 'kwargs-info': {'level': 2},
- 'kwargs-error':{'level': None},
- }
-
- fetcher = Connector(connector_output, self.proxies, USERAGENT)
+ return ext
- success, tar, timestamp = fetcher.fetch_content(tar_url)
- pkg = path([base, self.parent.name + ext])
-
- try:
- with fileopen(pkg, 'w+b') as out_file:
- out_file.write(tar)
-
- except Exception as error:
- raise Exception('Failed to store tar package in '
- + pkg + '\nError was:' + str(error))
- else:
- clean_tar = False
- pkg = tar_url.replace('file://', '')
+ def post_fetch(self, pkg, dest_dir):
+ '''
+ Extracts tar archive.
+ @params pkg: string location where tar archive is located.
+ @params dest_dir: string of destination of extracted archive.
+ @rtype bool
+ '''
# tar -v -x -f SOURCE -C TARGET
args = ['-v', '-x', '-f', pkg, '-C', dest_dir]
result = self.run_command(self.command(), args, cmd=self.type)
- if clean_tar:
- os.unlink(pkg)
- return result
-
- def _add_unchecked(self, base):
- def try_to_wipe(folder):
- if not os.path.exists(folder):
- return
-
- try:
- self.output.info('Deleting directory "%s"' % folder, 2)
- shutil.rmtree(folder)
- except Exception as error:
- raise Exception('Failed to remove unnecessary tar structure "'
- + folder + '"\nError was:' + str(error))
-
- final_path = path([base, self.parent.name])
- temp_path = tempfile.mkdtemp(dir=base)
- try:
- result = self._extract(base=base, tar_url=self.src,
- dest_dir=temp_path)
- except Exception as error:
- try_to_wipe(temp_path)
- raise error
-
- if result == 0:
- if self.branch:
- source = temp_path + '/' + self.branch
- else:
- source = temp_path
-
- if os.path.exists(source):
- if os.path.exists(final_path):
- self.delete(base)
-
- try:
- os.rename(source, final_path)
- except Exception as error:
- raise Exception('Failed to rename tar subdirectory ' +
- source + ' to ' + final_path +
- '\nError was:' + str(error))
- os.chmod(final_path, 0o755)
- else:
- raise Exception('The given path (branch setting in the xml)\n'
+ \
- '"%(source)s" does not exist in the tar package!' %
({'source': source}))
-
- try_to_wipe(temp_path)
return result
- def add(self, base):
- '''Add overlay.'''
-
- if not self.supported():
- return 1
-
- target = path([base, self.parent.name])
-
- if os.path.exists(target):
- raise Exception('Directory ' + target + ' already exists.' +\
- ' Will not overwrite its contents!')
-
- return self.postsync(
- self._add_unchecked(base),
- cwd=target)
-
- def sync(self, base):
- '''Sync overlay.'''
-
- if not self.supported():
- return 1
-
- target = path([base, self.parent.name])
- return self.postsync(
- self._add_unchecked(base),
- cwd=target)
+ def is_supported(self):
+ '''
+ Determines if overlay type is supported.
- def supported(self):
- '''Overlay type supported?'''
+ @rtype bool
+ '''
return require_supported(
[(self.command(), 'tar', 'app-arch/tar'), ],
