# models.py from google.appengine.ext import db
class CainIncident(db.Model): name = db.StringProperty() status = db.StringProperty() date = db.DateProperty() age = db.IntegerProperty() type = db.StringProperty() narrative = db.StringProperty() blamed = db.StringProperty() lat = db.FloatProperty() lon = db.FloatProperty() created = db.DateTimeProperty(auto_now_add=True) # data_api.py import datetime import os import re import sys sys.path.insert(0, '.') from google.appengine.api import datastore_types from google.appengine.ext import db from google.appengine.tools import bulkloader from utils.parsedatetime.parsedatetime import Calendar from models import CainIncident class CainLoader(bulkloader.Loader): elements = [ ('name', str), ('status', str), ('date', lambda x: datetime.date(*(Calendar().parseDateText(x) [:3])) ), ('age', int), ('type', str), ('narrative', str), ('blamed', str), ('lat', float), ('lon', float) ] def __init__(self): bulkloader.Loader.__init__(self, 'CainIncident', self.elements) loaders = [CainLoader] # cain_raw.csv "Abbott, Anthony","British Army (BA),",24 October 1976,19,nfNI,"Shot by snipers while checking abandoned car, Oakfield Street, Ardoyne, Belfast.",Irish Republican Army (IRA),54.5972686,-5.9301088 "Abercrombie, Aubrey","Ulster Defence Regiment (UDR),",05 February 1980,44,Protestant,"Off duty. Shot at his farm, near Kinawley, County Fermanagh.",Irish Republican Army (IRA),54.22641,-7.649053 "Abernethy, Colin","Civilian Political Activist (CivPA),",09 September 1988,30,Protestant,"Ulster Clubs member. Shot while travelling on train to his workplace, Finaghy, Belfast.",Irish Republican Army (IRA), 54.5553948,-5.9930868 "Abrams, Kieran","Civilian (Civ),",05 July 1992,35,Catholic,"Found beaten to death, North Howard Street, Lower Falls, Belfast.",non- specific Loyalist group (LOY),54.5972686,-5.9301088 "Acheson, Alfred","Civilian (Civ),",03 June 1973,48,Protestant,"Shot while in friend's house, Druse Street, Lower Oldpark, Belfast.",Ulster Volunteer Force (UVF),54.5972686,-5.9301088 "Acheson, Kenneth","Royal Ulster Constabulary (RUC),",02 April 1981,23,Protestant,"Off duty. Killed when detonated booby trap bomb attached to his car, shortly after leaving Bessbrook Royal Ulster Constabulary (RUC) base, County Armagh.",Irish Republican Army (IRA), 54.3475392,-6.6561082 "Adams, Nathaniel","Civilian (Civ),",05 April 1975,29,Protestant,"Killed in bomb attack on Mountainview Tavern, Shankill Road, Belfast.",non-specific Republican group (REP), 54.6040929,-5.9493439 "Adams, Stanley","Ulster Defence Regiment (UDR),",28 October 1976,29,Protestant,"Off duty. Shot while delivering mail, Altmore, near Pomeroy, County Tyrone.",Irish Republican Army (IRA), 54.5686974,-6.9641998 "Adams, Terence","British Army (BA),",06 December 1982,20,nfNI,"Off duty. Killed by time bomb left in disco at Droppin Well Bar, Ballykelly, County Derry.",Irish National Liberation Army (INLA), 54.99405,-7.325508 "Adams, Michael","Ulster Defence Regiment (UDR),",09 April 1990,23,Protestant,"Killed in land mine attack on Ulster Defence Regiment mobile patrol, Ballydugan Road, Downpatrick, County Down.",Irish Republican Army (IRA),54.3237914,-5.7205771 On 23 Sep, 08:57, "Nick Johnson (Google)" <nick.john...@google.com> wrote: > Hi Col, > > Would you be prepared to send me your script, model definitions, and the > first few lines of your CSV file? This looks like it could be a bug in the > SDK. > > -Nick Johnson > > On Wed, Sep 23, 2009 at 7:40 AM, Col Wilson <col.wilson.em...@googlemail.com > > > > > > > wrote: > > > Nick, I went back to my real word code and created a CSV and yes, the > > same problem... > > > [INFO ] Connecting to localhost:8080/remote_api > > [DEBUG ] [Thread-11] ProgressTrackerThread: started > > [DEBUG ] [Thread-12] DataSourceThread: started > > [INFO ] Starting import; maximum 10 entities per post > > [DEBUG ] [Thread-12] DataSourceThread: exiting > > [DEBUG ] Waiting for worker threads to finish... > > [DEBUG ] [Thread-3] Got work item [1-10] > > [ERROR ] Retrying on non-fatal datastore error: Put accepted 10 > > entities but returned 0 keys. > > Traceback (most recent call last): > > File "C:\Program Files\Google\google_appengine\google\appengine\tools > > \bulkloader.py", line 675, in PerformWork > > transfer_time = self._TransferItem(thread_pool) > > File "C:\Program Files\Google\google_appengine\google\appengine\tools > > \bulkloader.py", line 832, in _TransferItem > > self.request_manager.PostEntities(self.content) > > File "C:\Program Files\Google\google_appengine\google\appengine\tools > > \bulkloader.py", line 1239, in PostEntities > > datastore.Put(entities) > > File "C:\Program Files\Google\google_appengine\google\appengine\api > > \datastore.py", line 179, in Put > > (num_entities, num_keys)) > > InternalError: Put accepted 10 entities but returned 0 keys. > > [INFO ] Backing off due to errors: 1.0 seconds > > [DEBUG ] [Thread-3] Got work item [1-10] > > > On 21 Sep, 14:52, "Nick Johnson (Google)" <nick.john...@google.com> > > wrote: > > > Hi Col, > > > Can you try loading from CSV (eg, remove your generate_records method) > > and > > > let us know if you see the same problem? Also, can you please include the > > > complete stacktrace? > > > > -Nick Johnson > > > > On Sat, Sep 19, 2009 at 12:28 PM, Col Wilson < > > > > col.wilson.em...@googlemail.com> wrote: > > > > > loaders = [ThingLoader] > > > > > is what I meant to type. > > > > > On Sep 19, 10:14 am, Col Wilson <col.wilson.em...@googlemail.com> > > > > wrote: > > > > > I'm using Python 2.5 locally and GAE 1.5. > > > > > > I'm trying to load data which is not is CSV format. Here's a > > > > > simplified model which reflects what I get with a more complex real > > > > > life issue: > > > > > > class Thing(db.Model): > > > > > name = db.StringProperty() > > > > > > class ThingLoader(bulkloader.Loader): > > > > > elements = [ > > > > > ('name', str) > > > > > ] > > > > > > def __init__(self): > > > > > bulkloader.Loader.__init__(self, 'Thing', self.elements) > > > > > > def generate_records(self, filename): > > > > > records = load_records_from_some_datasource() > > > > > for record in records: > > > > > yield [ > > > > > record['name'] > > > > > ] > > > > > > loaders = [Thing] > > > > > > When I try to bulkload this (both locally or to google) I get: > > > > > > InternalError: Put accepted 10 entities but returned 0 keys. > > > > > > I can't work out why the datastore will not allocate keys. Any ideas > > > > > what I should do? > > > > -- > > > Nick Johnson, Developer Programs Engineer, App Engine > > > Google Ireland Ltd. :: Registered in Dublin, Ireland, Registration > > Number: > > > 368047 > > -- > Nick Johnson, Developer Programs Engineer, App Engine > Google Ireland Ltd. :: Registered in Dublin, Ireland, Registration Number: > 368047 --~--~---------~--~----~------------~-------~--~----~ You received this message because you are subscribed to the Google Groups "Google App Engine" group. To post to this group, send email to google-appengine@googlegroups.com To unsubscribe from this group, send email to google-appengine+unsubscr...@googlegroups.com For more options, visit this group at http://groups.google.com/group/google-appengine?hl=en -~----------~----~----~----~------~----~------~--~---