Hi Ido, On Dec 4, 2012, at 10:26 AM, Ido Tamir wrote:
> Hi, > how do I get tools from one local toolshed (or the main galaxy toolshed) > into another tool-shed? Is it possible to clone the content of a complete > tool-shed - all repositories > at once into another tool-shed? Not at the current time, but I've attached a script that I use in my development environment that sort of does what you want. When you run it for the first time, make sure it installs to a tool shed you don't care about destroying just so you can see how it works. It uses the configuration settings in commnity_wsgi.ini, so make sure you set things accordingly. here's what you do: 1. Place the 3 attached files in ~/scripts/tool_shed 2. Define the tool shed you want to copy repositories form in copy_repository.xml 3. Define any number of valid <repository> tags in copy_repository.xml 4. run the copy_repositoyr.sh shell script from the ~/scripts/tool_shed directory 5. When all repositories have been copied into you local tool shed, start it up and make sure to reset all metadata on all repositories as the script does not do so. You can dfo this from the Tool Shed's admin interface. I do not have time to support it, so you're on your own if you want to use it as is. If you enhance it to the point that it would be useful to the community, please contribute it back to me and I'll make sure it is included in the Galaxy code base.
#!/usr/bin/env python
'''
This script requires the Galaxy instance to use Postgres for database storage.
To run this script, use "sh copy_repository.sh" from this directory
'''
import sys, os, ConfigParser
assert sys.version_info[:2] >= ( 2, 4 )
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
from galaxy import eggs
import pkg_resources
pkg_resources.require( "psycopg2" )
import psycopg2
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree, ElementInclude
import galaxy.webapps.community.app
from galaxy import util
from mercurial import hg, ui, httprepo, commands
def directory_hash_id( id ):
s = str( id )
l = len( s )
# Shortcut -- ids 0-999 go under ../000/
if l < 4:
return [ "000" ]
# Pad with zeros until a multiple of three
padded = ( ( ( 3 - len( s ) ) % 3 ) * "0" ) + s
# Drop the last three digits -- 1000 files per directory
padded = padded[:-3]
# Break into chunks of three
return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ]
def add_hgweb_config_entry( repository, repository_path ):
# Add an entry in the hgweb.config file for a new repository. This enables calls to repository.repo_path.
# An entry looks something like: repos/test/mira_assembler = database/community_files/000/repo_123
hgweb_config = "%s/hgweb.config" % os.getcwd()
entry = "repos/%s/%s = %s" % ( repository.user.username, repository.name, repository_path.lstrip( './' ) )
if os.path.exists( hgweb_config ):
output = open( hgweb_config, 'a' )
else:
output = open( hgweb_config, 'w' )
output.write( '[paths]\n' )
output.write( "%s\n" % entry )
output.close()
def create_hgrc_file( repository ):
# At this point, an entry for the repository is required to be in the hgweb.config
# file so we can call repository.repo_path.
# Create a .hg/hgrc file that looks something like this:
# [web]
# allow_push = test
# name = convert_characters1
# push_ssl = False
# Upon repository creation, only the owner can push to it ( allow_push setting ),
# and since we support both http and https, we set push_ssl to False to override
# the default (which is True) in the mercurial api.
hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) )
output = open( hgrc_file, 'w' )
output.write( '[web]\n' )
output.write( 'allow_push = %s\n' % repository.user.username )
output.write( 'name = %s\n' % repository.name )
output.write( 'push_ssl = false\n' )
output.flush()
output.close()
def clone_repository( repository_clone_url, repository_path ):
"""Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository."""
commands.clone( get_configured_ui(),
str( repository_clone_url ),
dest=str( repository_path ),
pull=True,
noupdate=False,
rev=[] )
def contains( containing_str, contained_str ):
return containing_str.lower().find( contained_str.lower() ) >= 0
def copy_repository( app, repository_clone_url, name, description, user_id ):
sa_session = app.model.context
# Add the repository record to the db
print "Adding new database record for repository: ", name
repository = app.model.Repository( name=name,
description=description,
long_description=description,
user_id=user_id )
# Flush to get the id
sa_session.add( repository )
sa_session.flush()
# Determine the repository's repo_path on disk
clone_dir = os.path.join( app.config.file_path, *directory_hash_id( repository.id ) )
# Create directory if it does not exist
if not os.path.exists( clone_dir ):
os.makedirs( clone_dir )
# Define repo name inside hashed directory
repository_path = os.path.join( clone_dir, "repo_%d" % repository.id )
# Create local repository directory
if not os.path.exists( repository_path ):
os.makedirs( repository_path )
print "Creating local repository at: ", repository_path
# Clone the repository
clone_repository( repository_clone_url, os.path.abspath( repository_path ) )
repo = hg.repository( get_configured_ui(), repository_path )
# Update the repository files for browsing.
print "Updating repository"
update_repository( repo )
# Add an entry in the hgweb.config file for the local repository. This enables calls to repository.repo_path
print "Adding entry to hgweb.config"
add_hgweb_config_entry( repository, repository_path )
# Create a .hg/hgrc file for the local repository
print "Creating hgrc file"
create_hgrc_file( repository )
print "Repository '%s' has been copied." % repository.name
def create_user( app, email, password, username ):
sa_session = app.model.context
user = app.model.User( email=email )
user.set_password_cleartext( password )
user.username = username
sa_session.add( user )
sa_session.flush()
return user
def get_configured_ui():
# Configure any desired ui settings.
_ui = ui.ui()
# The following will suppress all messages. This is
# the same as adding the following setting to the repo
# hgrc file' [ui] section:
# quiet = True
_ui.setconfig( 'ui', 'quiet', True )
return _ui
def get_user( app, email ):
sa_session = app.model.context
return sa_session.query( app.model.User ) \
.filter( app.model.User.table.c.email == email ) \
.first()
def update_repository( repo, ctx_rev=None ):
commands.update( get_configured_ui(),
repo,
rev=ctx_rev )
def main():
if len( sys.argv ) < 3:
print "Usage: python %s <Tool shed config file> copy_repository.xml" % sys.argv[0]
sys.exit( 0 )
email = 't...@bx.psu.edu'
password = 'testuser'
username = 'test'
# community_wsgi.ini file
ini_file = sys.argv[ 1 ]
repository_xml_file = sys.argv[ 2 ]
conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } )
conf_parser.read( ini_file )
try:
db_conn_str = conf_parser.get( "app:main", "database_connection" )
except ConfigParser.NoOptionError, e:
db_conn_str = conf_parser.get( "app:main", "database_file" )
print 'DB Connection: ', db_conn_str
# Determine db connection - only postgres is supported
if contains( db_conn_str, '///' ) and contains( db_conn_str, '?' ) and contains( db_conn_str, '&' ):
# postgres:///galaxy_test?user=postgres&password=postgres
db_str = db_conn_str.split( '///' )[ 1 ]
db_name = db_str.split( '?' )[ 0 ]
db_user = db_str.split( '?' )[ 1 ].split( '&' )[ 0 ].split( '=' )[ 1 ]
db_password = db_str.split( '?' )[ 1 ].split( '&' )[ 1 ].split( '=' )[ 1 ]
elif contains( db_conn_str, '//' ) and contains( db_conn_str, ':' ):
# dialect://user:password@host/db_name
db_name = db_conn_str.split( '/' )[ -1 ]
db_user = db_conn_str.split( '//' )[ 1 ].split( ':' )[ 0 ]
# Instantiate app
configuration = {}
for key, value in conf_parser.items( "app:main" ):
configuration[ key ] = value
app = galaxy.webapps.community.app.UniverseApplication( global_conf=dict( __file__=ini_file ), **configuration )
sa_session = app.model.context
# Make sure we have a user.
user = get_user( app, email )
if user is None:
user = create_user( app, email, password, username )
hgweb_config = "%s/hgweb.config" % os.getcwd()
# Parse the copy_repository.xml file to determine what repositories should be copied.
tree = util.parse_xml( repository_xml_file )
root = tree.getroot()
tool_shed_url = root.get( 'url' )
for repository_elem in root:
repository_name = repository_elem.get( 'name' )
repository_description = repository_elem.get( 'description' )
repository_owner = repository_elem.get( 'owner' )
repository_clone_url = os.path.join( tool_shed_url, 'repos', repository_owner, repository_name )
copy_repository( app, repository_clone_url, repository_name, repository_description, user.id )
app.shutdown()
sys.exit(0)
if __name__ == "__main__":
main()
copy_repository.sh
Description: Binary data
copy_repository.xml
Description: XML document
> Can I then import all the tools at once into a local galaxy instance? Not currently, but this is planned for the near future ( within a few months ). > > We have a group with their own tool-shed. Unfortunately, I accessed it > directly > without a proxy on its port which leads to a problem described in issue 825 > in galaxy-central. > > I want to > a) clone the tool-shed as it is into my own tool-shed (rsync?) > b) selectively or bulk install these tools into my galaxy server. > > thank you very much, > ido > > > > ___________________________________________________________ > Please keep all replies on the list by using "reply all" > in your mail client. To manage your subscriptions to this > and other Galaxy lists, please use the interface at: > > http://lists.bx.psu.edu/
___________________________________________________________ Please keep all replies on the list by using "reply all" in your mail client. To manage your subscriptions to this and other Galaxy lists, please use the interface at: http://lists.bx.psu.edu/
