* Baruch Even <[email protected]> [01.10.2009 20:10]: > Hi, > > Frederik Sdun wrote: >> Hi, >> >> I try to package some maps for navit. Because I don't want to waste hard >> disk space on the SHR server I want to use postinst and postrm and >> download the maps from cloudmade.com [0]. >> The problems: >> * maps are really big and you might get problems if you don't use a >> memory card >> * navit needs a fixed directory for maps in the config file(see PATCH #3) >> * How should the versioning be done (cloudmade updates weekly) >> >> My ideas: >> * The sze of the maps must be known before downloading >> * figure out which partition has enough space left, else bail out >> * create a link to /usr/share/navit/maps >> * remove everything on postrm >> >> My new problems: >> * My test package for andorra is empty >> * Some fights against navit compilation > > > Why bother packaging the maps? They change weekly and unless the repo is > updated fast enough the users will be left behind. My thinking for this > was to create a script that downloads the data for the user and updates > it for him. > > Attached is the script I wrote, I'm not even sure at what state it is. I > think it works. > > My plan for the script was to run it from if-up.d so that it will update > whenever the user connects to the network, but that's pretty excessive. > > Baruch
Your script is nice, but I don't want an external app. Either a package or we
can contribute a
plugin to navit for downloading maps from cloudmade? Anyone has
experience with this?
> #!/usr/bin/python
>
> import pickle, sys, os, time, urllib2, re, zipfile, stat
>
> #MAP_LOCATION = '/media/card/Maps/navit'
> MAP_LOCATION = '.'
> CONFIG_FILE = 'navit-osmmap.pickle'
> mapname_re_str = r'cloudmade_map_([A-Za-z]+)/([A-Za-z]+).navit.bin$'
> mapname_re = re.compile(mapname_re_str)
> map_updated = False
>
> if 1:
> def debug(msg):
> print 'debug:', msg
> def info(msg):
> print 'INFO:', msg
> def missing(msg):
> print 'MISSING FEATURE:', msg
>
> class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
> def http_error_default(self, req, fp, code, msg, headers):
> result = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
> result.status = code
> return result
>
> def get_map_name_from_gsm():
> missing('Fetching info from GSM is not yet implemented, using a default')
> return ('asia', 'israel')
>
> def safe_file_save(filename, writer):
> tmp_fname = filename + '.tmp'
> debug('Open file %s' % tmp_fname)
> f = open(tmp_fname, 'w')
> debug('Writing content of tmp file')
> writer(f)
> debug('fsync content to disk')
> os.fsync(f.fileno())
> debug('close file')
> f.close()
> debug('Rename %s to %s' % (tmp_fname, filename))
> os.rename(tmp_fname, filename)
> debug('Done safe file save')
>
> def load_config():
> conf = None
> try:
> f = open(CONFIG_FILE, 'r')
> conf = pickle.load(f)
> f.close()
> except OSError, e:
> info('Config file "%s" failed to load due to %s' % (CONFIG_FILE, e))
> except IOError, e:
> info('Config file "%s" not read due to %s' % (CONFIG_FILE, e))
>
> if not conf:
> conf = dict(last_check = 0, url=dict())
>
> return conf
>
> def save_config():
> safe_file_save(CONFIG_FILE, lambda f: pickle.dump(conf, f))
>
> def regular_file_iter(root):
> for root, dirs, files in os.walk(root):
> for file in files:
> fname = os.path.join(root, file)
> st = os.stat(fname)
> if stat.S_ISREG(st[stat.ST_MODE]):
> yield fname
>
> def map_to_filename(p):
> return os.path.join(MAP_LOCATION, 'cloudmade_map_%s.%s.navit.bin' % p)
>
> def download_country(p, map_fname = None):
> if not map_fname:
> map_fname = map_to_filename(p)
>
> # Get URL from filename
> url = 'http://download.cloudmade.com/%s/%s/%s.navit.bin.zip' % (p[0],
> p[1], p[1])
>
> global conf
> lastModified = conf['url'].get(url, None)
>
> # Download from url to a temporary file (only download if changed)
> info('Starting request for %s last modified %s' % (url, lastModified))
>
> request = urllib2.Request(url)
> if lastModified:
> request.add_header('If-Modified-Since', lastModified)
> opener = urllib2.build_opener(DefaultErrorHandler())
> stream = opener.open(request)
>
> try:
> a = stream.status
> except AttributeError:
> stream.status = 200
> info('Reuqest status is %d' % stream.status)
>
> if stream.status == 200:
> conf['url'][url] = stream.headers.get('Last-Modified')
>
> if stream.status != 200:
> info('File not retrievent http response=%d' % stream.status)
>
> tmp_fname = 'tmp.navit.bin'
> info('Saving zipfile to %s' % tmp_fname)
> f = open(tmp_fname, 'w')
> f.write(stream.read())
> f.close()
> stream.close()
>
> # extract file from zip to temporary location
> def save_as_zipfile(f):
> debug('Writing zip content')
> zf = zipfile.ZipFile(tmp_fname)
> names = zf.namelist()
> assert(len(names) == 1)
> print names
> f.write(zf.read(names[0]))
> zf.close()
>
> info('Save the file inside the zipfile')
> safe_file_save(map_fname, save_as_zipfile)
>
> # Mark maps as updated
> global map_updated
> map_updated = True
>
> conf = load_config()
>
> force_download = False
>
> gsm_map = get_map_name_from_gsm()
> if gsm_map and not os.path.exists(map_to_filename(gsm_map)):
> force_download = True
>
> # If not forced and last update was less than 24 hours ago, do not try to
> download anything
> now = time.time()
> debug('now=%d last_check=%d diff=%d' % (now, conf['last_check'],
> now-conf['last_check']))
> if not force_download and now - conf['last_check'] > 24*60*60:
> info('Not forced download and last check is less than 24 hours ago')
> sys.exit(0)
>
> downloaded = []
>
> # List all cloudmade maps
> for map_fname in regular_file_iter(MAP_LOCATION):
> match = mapname_re.match(map_fname)
>
> # Is it a cloudmade map?
> if match is None:
> debug('%s is not a cloudmade map' % map_fname)
> continue
>
> p = (match[1], match[2])
>
> if p in downloaded:
> os.unlink(map_fname)
> else:
> downloaded.append(p)
>
> download_country(p, map_fname)
>
> if gsm_map not in downloaded:
> download_country(gsm_map, None)
> downloaded.append(gsm_map)
>
> # If any map was updated, notify navit somehow
> if map_updated:
> missing('Dont know how to update navit that a map was changed')
>
> # Update configuration
> conf['last_check'] = now
> save_config()
--
IRC: playya @ Freenode, Gimpnet
xmpp: [email protected]
signature.asc
Description: Digital signature
_______________________________________________ Shr-devel mailing list [email protected] http://lists.shr-project.org/mailman/listinfo/shr-devel
