Michael Hall has proposed merging
lp:~mhall119/loco-directory/data-recovery-script into lp:loco-directory.
Requested reviews:
loco-directory-dev (loco-directory-dev)
For more details, see:
https://code.launchpad.net/~mhall119/loco-directory/data-recovery-script/+merge/73409
Adds a new management command that will compare a json file, produced by
running manage.py dumpdata, against the current database contents and attempts
to recreate any records from the json file that don't exist.
Once we deploy this, we will need to provide IS with a copy of a json file to
run it against.
--
https://code.launchpad.net/~mhall119/loco-directory/data-recovery-script/+merge/73409
Your team loco-directory-dev is requested to review the proposed merge of
lp:~mhall119/loco-directory/data-recovery-script into lp:loco-directory.
=== added directory 'loco_directory/common/management'
=== added file 'loco_directory/common/management/__init__.py'
--- loco_directory/common/management/__init__.py 1970-01-01 00:00:00 +0000
+++ loco_directory/common/management/__init__.py 2011-08-30 16:30:30 +0000
@@ -0,0 +1,1 @@
+
=== added directory 'loco_directory/common/management/commands'
=== added file 'loco_directory/common/management/commands/__init__.py'
--- loco_directory/common/management/commands/__init__.py 1970-01-01 00:00:00 +0000
+++ loco_directory/common/management/commands/__init__.py 2011-08-30 16:30:30 +0000
@@ -0,0 +1,1 @@
+
=== added file 'loco_directory/common/management/commands/recoverdata.py'
--- loco_directory/common/management/commands/recoverdata.py 1970-01-01 00:00:00 +0000
+++ loco_directory/common/management/commands/recoverdata.py 2011-08-30 16:30:30 +0000
@@ -0,0 +1,269 @@
+'''
+import data from a 0.1 dump
+'''
+
+import simplejson
+
+from django.core.management.base import BaseCommand
+
+from django.contrib.auth.models import User
+from teams.models import *
+from userprofiles.models import *
+from events.models import *
+from venues.models import *
+from meetings.models import *
+
+class Command(BaseCommand):
+ args = 'dump_file'
+ help = 'Attempt to recover missing data from a json dump'
+
+ def handle(self, *args, **options):
+ if len(args) < 1:
+ print "You must supply a dumpfile"
+ return
+ dumpfile = args[0]
+
+ try:
+ json = open(dumpfile)
+ except:
+ print "Unable to open %s" % dumpfile
+ return
+
+ try:
+ decoder = simplejson.JSONDecoder()
+ data = decoder.decode(json.read())
+ except:
+ print "Failed to parse %s" % dumpfile
+ return
+ finally:
+ json.close()
+
+ print "Loading %s" % dumpfile
+ users = dict()
+ profiles = dict()
+ teams = dict()
+ baseevents = dict()
+ globalevents = dict()
+ teamevents = dict()
+ venues = dict()
+ basemeetings=dict()
+ teammeetings = dict()
+ agendaitems = dict()
+ agendaitems_needparent = dict()
+
+ # first pass we can collect user, teams and venues
+ for record in data:
+ # recover users
+ if record['model'] == 'auth.user':
+# print "Checking user: %s" % record['fields']['username']
+ users[record['pk']], user_created = User.objects.get_or_create(
+ username=record['fields']['username'],
+ defaults={
+ 'first_name': record['fields']['first_name'],
+ 'last_name': record['fields']['last_name'],
+ }
+ )
+ if user_created:
+ print "Created User: %s" % users[record['pk']].username
+
+ # recover teams
+ if record['model'] == 'teams.team':
+# print "Checking team %s (%s)" % (record['fields']['name'], record['fields']['lp_name'])
+ teams[record['pk']], team_created = Team.objects.get_or_create(
+ lp_name=record['fields']['lp_name'],
+ defaults={
+ 'name': record['fields']['name'],
+ }
+ )
+ if team_created:
+ print "Created Team: %s" % teams[record['pk']].lp_name
+
+ # recover venues
+ if record['model'] == 'venues.venue':
+# print "Checking venue %s" % (record['fields']['name'],)
+ try:
+ venue_country = Country.objects.get(pk=record['fields']['country'])
+ except:
+ venue_country = None
+ try:
+ venues[record['pk']], venue_created = Venue.objects.get_or_create(
+ name=record['fields']['name'],
+ country=venue_country,
+ city=record['fields']['city'],
+ longitude=record['fields']['longitude'],
+ latitude=record['fields']['latitude'],
+ defaults={
+ 'spr': record['fields']['spr'],
+ 'venue_url': record['fields']['venue_url'],
+ 'tz': record['fields']['tz'],
+ }
+ )
+ if venue_created:
+ print u"Created Venue %s: %s" % (venues[record['pk']].pk, unicode(venues[record['pk']].name).encode('ascii', 'ignore'))
+ except Exception, e:
+ print u"Failed to create missing venue: %s\n%s" % (unicode(record['fields']['name']).encode('ascii', 'ignore'), e)
+
+ # second pass we need to save baseevents and basemeetings, we'll need
+ # their data for creating global events, team event and team meetings
+ # we also collect userprofiles for later use
+ for record in data:
+ # save base event
+ if record['model'] == 'events.baseevent':
+# print "Caching baseevent %s" % (record['fields']['name'],)
+ baseevents[record['pk']] = record['fields']
+
+ # save base meeting
+ if record['model'] == 'meetings.basemeeting':
+# print "Caching basemeeting %s" % (record['fields']['name'],)
+ basemeetings[record['pk']] = record['fields']
+
+ # recover userprofile
+ if record['model'] == 'userprofiles.userprofile':
+ if record['fields']['user'] in users:
+ try:
+ profiles[record['pk']], profile_created = UserProfile.objects.get_or_create(
+ user=users[record['fields']['user']],
+ defaults={
+ 'realname': record['fields']['realname'],
+ }
+ )
+ if profile_created:
+ print u"Created UserProfile: %s (%s)" % (profiles[record['pk']].realname, profiles[record['pk']].user.username)
+ except Exception, e:
+ print u"Failed to create missing userprofile: %s\n%s" % (record, users[record['fields']['user']])
+ else:
+ print u"Failed to create missing userprofile, no such user: %s" % record['fields']['user']
+
+
+ # third pass we can collect global events and team meetings, using the
+ # baseevent and basemeeting data cached in the previous pass
+ for record in data:
+ # recover global event
+ if record['model'] == 'events.globalevent':
+ baseevent = baseevents[record['pk']]
+# print "Checking globalevent %s" % (baseevent['name'],)
+ globalevents[record['pk']], ge_created = GlobalEvent.objects.get_or_create(
+ name=baseevent['name'],
+ date_begin=baseevent['date_begin'],
+ date_end=baseevent['date_end'],
+ defaults={
+ 'description': baseevent['description'],
+ 'announce': baseevent['announce'],
+ 'date_created': baseevent['date_created'],
+ 'microbloghashtag': record['fields']['microbloghashtag'],
+ 'pictag': record['fields']['pictag'],
+ }
+ )
+ if ge_created:
+ print u"Created GlobalEvent %s: %s" % (globalevents[record['pk']].pk, unicode(globalevents[record['pk']].name).encode('ascii', 'ignore'))
+
+ # recover team meeting
+ if record['model'] == 'meetings.teammeeting':
+ basemeeting = basemeetings[record['pk']]
+# print "Checking teammeeting %s\n%s" % (basemeeting['name'], record)
+ teammeetings[record['pk']], meeting_created = TeamMeeting.objects.get_or_create(
+ name=basemeeting['name'],
+ date_begin=basemeeting['date_begin'],
+ date_end=basemeeting['date_end'],
+ defaults={
+ 'channel': basemeeting['channel'],
+ 'logs': basemeeting['logs'],
+ 'minutes': basemeeting['minutes'],
+ 'date_created': basemeeting['date_created'],
+ 'chair': profiles.get(basemeeting['chair'], None),#profile lookup
+ 'meeting_tz': record['fields']['meeting_tz'],
+ }
+ )
+ if meeting_created:
+ add_teams = [teams[tid] for tid in record['fields']['teams'] if tid in teams]
+ teammeetings[record['pk']].teams.add(*add_teams)
+ print u"Created TeamMeeting %s: %s" % (teammeetings[record['pk']].pk, unicode(teammeetings[record['pk']].name).encode('ascii', 'ignore'))
+
+ # fourth pass we can collect team events
+ for record in data:
+ # recover team event
+ if record['model'] == 'events.teamevent':
+ baseevent = baseevents[record['pk']]
+# print u"Checking teamevent %s" % (baseevent['name'],)
+ try:
+ teamevents[record['pk']], te_created = TeamEvent.objects.get_or_create(
+ name=baseevent['name'],
+ date_begin=baseevent['date_begin'],
+ date_end=baseevent['date_end'],
+ venue=venues.get(record['fields']['venue'], None), #venue lookup
+ channel=record['fields']['channel'],
+ teams__in=[teams[tid] for tid in record['fields']['teams'] if tid in teams],
+ defaults={
+ 'description': baseevent['description'],
+ 'announce': baseevent['announce'],
+ 'date_created': baseevent['date_created'],
+ 'contact': profiles.get(record['fields']['contact'], None), #profile lookup
+ 'global_event': globalevents.get(record['fields']['global_event'], None), #global event lookup
+ 'registration': record['fields']['registration'],
+ }
+ )
+ if te_created:
+ add_teams = [teams[tid] for tid in record['fields']['teams'] if tid in teams]
+ teamevents[record['pk']].teams.add(*add_teams)
+ print u"Created TeamEvent %s: %s" % (teamevents[record['pk']].pk, unicode(teamevents[record['pk']].name).encode('ascii', 'ignore'))
+ except Exception, e:
+ print u"Failed to create team event %s: %s" % (record['pk'], unicode(baseevent['name']).encode('ascii', 'ignore'))
+
+ # fifth pass we can collect agenda items, event attendees and comments
+ for record in data:
+ # recover agenda item
+ if record['model'] == 'meetings.agendaitem':
+ agendaitems[record['pk']], ai_created = AgendaItem.objects.get_or_create(
+ meeting=teammeetings.get(record['fields']['meeting'], None),
+ title=record['fields']['title'],
+ owner=profiles.get(record['fields']['owner'], None),
+ created_date=record['fields']['created_date'],
+ defaults={
+ 'description': record['fields']['description'],
+ 'parent': None, #saved for later lookup, since they aren't ordered
+ 'order': record['fields']['order'],
+ 'log': record['fields']['log'],
+ }
+ )
+ if ai_created and record['fields']['parent']:
+ agendaitems_needparent[record['pk']] = record['fields']['parent']
+ print u"Created AgendaItem %s: %s" % (agendaitems[record['pk']].pk, unicode(agendaitems[record['pk']].title).encode('ascii', 'ignore'))
+
+ # recover event attendee
+ if record['model'] == 'events.attendee':
+ if record['fields']['team_event'] in teamevents:
+ attendee, a_created = Attendee.objects.get_or_create(
+ team_event=teamevents.get(record['fields']['team_event'], None),
+ attendee_profile=profiles.get(record['fields']['attendee_profile'], None),
+ defaults={
+ 'promise': record['fields']['promise'],
+ 'guests': record['fields']['guests'],
+ }
+ )
+ if a_created:
+ print u"Created TeamEvent Attendee: %s for %s (%s)" % (attendee.attendee_profile.user.username, unicode(attendee.team_event.name).encode('ascii', 'ignore'), attendee.team_event.pk)
+ else:
+ print u"Failed to create Attendee, no such team event: %s" % record['fields']['team_event']
+
+ # recover event comment
+ if record['model'] == 'events.teameventcomment':
+ if record['fields']['team_event'] in teamevents:
+ comment, c_created = TeamEventComment.objects.get_or_create(
+ team_event=teamevents.get(record['fields']['team_event'], None),
+ commenter_profile=profiles.get(record['fields']['commenter_profile'], None),
+ date_created=record['fields']['date_created'],
+ defaults={
+ 'comment': record['fields']['comment'],
+ }
+ )
+ if c_created:
+ print u"Created TeamEvent Comment: %s on %s (%s)" % (comment.commenter_profile.user.username, unicode(comment.team_event.name).encode('ascii', 'ignore'), comment.team_event.pk)
+ else:
+ print u"Failed to create TeamEventComment, no such team event: %s" % record['fields']['team_event']
+
+ # after fifth pass, set agendaitem parents
+ for agendaitem_id in agendaitems_needparent:
+ parent_id = agendaitems_needparent[agendaitem_id]
+ agendaitem = agendaitems[agendaitem_id]
+ agendaitem.parent = agendaitems[parent_id]
+ agendaitem.save()
_______________________________________________
Mailing list: https://launchpad.net/~loco-directory-dev
Post to : [email protected]
Unsubscribe : https://launchpad.net/~loco-directory-dev
More help : https://help.launchpad.net/ListHelp
