# models.py
from google.appengine.ext import db
class CainIncident(db.Model):
name = db.StringProperty()
status = db.StringProperty()
date = db.DateProperty()
age = db.IntegerProperty()
type = db.StringProperty()
narrative = db.StringProperty()
blamed = db.StringProperty()
lat = db.FloatProperty()
lon = db.FloatProperty()
created = db.DateTimeProperty(auto_now_add=True)
# data_api.py
import datetime
import os
import re
import sys
sys.path.insert(0, '.')
from google.appengine.api import datastore_types
from google.appengine.ext import db
from google.appengine.tools import bulkloader
from utils.parsedatetime.parsedatetime import Calendar
from models import CainIncident
class CainLoader(bulkloader.Loader):
elements = [
('name', str),
('status', str),
('date', lambda x: datetime.date(*(Calendar().parseDateText(x)
[:3])) ),
('age', int),
('type', str),
('narrative', str),
('blamed', str),
('lat', float),
('lon', float)
]
def __init__(self):
bulkloader.Loader.__init__(self, 'CainIncident',
self.elements)
loaders = [CainLoader]
# cain_raw.csv
"Abbott, Anthony","British Army (BA),",24 October 1976,19,nfNI,"Shot
by snipers while checking abandoned car, Oakfield Street, Ardoyne,
Belfast.",Irish Republican Army (IRA),54.5972686,-5.9301088
"Abercrombie, Aubrey","Ulster Defence Regiment (UDR),",05 February
1980,44,Protestant,"Off duty. Shot at his farm, near Kinawley, County
Fermanagh.",Irish Republican Army (IRA),54.22641,-7.649053
"Abernethy, Colin","Civilian Political Activist (CivPA),",09 September
1988,30,Protestant,"Ulster Clubs member. Shot while travelling on
train to his workplace, Finaghy, Belfast.",Irish Republican Army (IRA),
54.5553948,-5.9930868
"Abrams, Kieran","Civilian (Civ),",05 July 1992,35,Catholic,"Found
beaten to death, North Howard Street, Lower Falls, Belfast.",non-
specific Loyalist group (LOY),54.5972686,-5.9301088
"Acheson, Alfred","Civilian (Civ),",03 June 1973,48,Protestant,"Shot
while in friend's house, Druse Street, Lower Oldpark, Belfast.",Ulster
Volunteer Force (UVF),54.5972686,-5.9301088
"Acheson, Kenneth","Royal Ulster Constabulary (RUC),",02 April
1981,23,Protestant,"Off duty. Killed when detonated booby trap bomb
attached to his car, shortly after leaving Bessbrook Royal Ulster
Constabulary (RUC) base, County Armagh.",Irish Republican Army (IRA),
54.3475392,-6.6561082
"Adams, Nathaniel","Civilian (Civ),",05 April
1975,29,Protestant,"Killed in bomb attack on Mountainview Tavern,
Shankill Road, Belfast.",non-specific Republican group (REP),
54.6040929,-5.9493439
"Adams, Stanley","Ulster Defence Regiment (UDR),",28 October
1976,29,Protestant,"Off duty. Shot while delivering mail, Altmore,
near Pomeroy, County Tyrone.",Irish Republican Army (IRA),
54.5686974,-6.9641998
"Adams, Terence","British Army (BA),",06 December 1982,20,nfNI,"Off
duty. Killed by time bomb left in disco at Droppin Well Bar,
Ballykelly, County Derry.",Irish National Liberation Army (INLA),
54.99405,-7.325508
"Adams, Michael","Ulster Defence Regiment (UDR),",09 April
1990,23,Protestant,"Killed in land mine attack on Ulster Defence
Regiment mobile patrol, Ballydugan Road, Downpatrick, County
Down.",Irish Republican Army (IRA),54.3237914,-5.7205771
On 23 Sep, 08:57, "Nick Johnson (Google)" <nick.john...@google.com>
wrote:
> Hi Col,
>
> Would you be prepared to send me your script, model definitions, and the
> first few lines of your CSV file? This looks like it could be a bug in the
> SDK.
>
> -Nick Johnson
>
> On Wed, Sep 23, 2009 at 7:40 AM, Col Wilson <col.wilson.em...@googlemail.com
>
>
>
>
>
> > wrote:
>
> > Nick, I went back to my real word code and created a CSV and yes, the
> > same problem...
>
> > [INFO ] Connecting to localhost:8080/remote_api
> > [DEBUG ] [Thread-11] ProgressTrackerThread: started
> > [DEBUG ] [Thread-12] DataSourceThread: started
> > [INFO ] Starting import; maximum 10 entities per post
> > [DEBUG ] [Thread-12] DataSourceThread: exiting
> > [DEBUG ] Waiting for worker threads to finish...
> > [DEBUG ] [Thread-3] Got work item [1-10]
> > [ERROR ] Retrying on non-fatal datastore error: Put accepted 10
> > entities but returned 0 keys.
> > Traceback (most recent call last):
> > File "C:\Program Files\Google\google_appengine\google\appengine\tools
> > \bulkloader.py", line 675, in PerformWork
> > transfer_time = self._TransferItem(thread_pool)
> > File "C:\Program Files\Google\google_appengine\google\appengine\tools
> > \bulkloader.py", line 832, in _TransferItem
> > self.request_manager.PostEntities(self.content)
> > File "C:\Program Files\Google\google_appengine\google\appengine\tools
> > \bulkloader.py", line 1239, in PostEntities
> > datastore.Put(entities)
> > File "C:\Program Files\Google\google_appengine\google\appengine\api
> > \datastore.py", line 179, in Put
> > (num_entities, num_keys))
> > InternalError: Put accepted 10 entities but returned 0 keys.
> > [INFO ] Backing off due to errors: 1.0 seconds
> > [DEBUG ] [Thread-3] Got work item [1-10]
>
> > On 21 Sep, 14:52, "Nick Johnson (Google)" <nick.john...@google.com>
> > wrote:
> > > Hi Col,
> > > Can you try loading from CSV (eg, remove your generate_records method)
> > and
> > > let us know if you see the same problem? Also, can you please include the
> > > complete stacktrace?
>
> > > -Nick Johnson
>
> > > On Sat, Sep 19, 2009 at 12:28 PM, Col Wilson <
>
> > > col.wilson.em...@googlemail.com> wrote:
>
> > > > loaders = [ThingLoader]
>
> > > > is what I meant to type.
>
> > > > On Sep 19, 10:14 am, Col Wilson <col.wilson.em...@googlemail.com>
> > > > wrote:
> > > > > I'm using Python 2.5 locally and GAE 1.5.
>
> > > > > I'm trying to load data which is not is CSV format. Here's a
> > > > > simplified model which reflects what I get with a more complex real
> > > > > life issue:
>
> > > > > class Thing(db.Model):
> > > > > name = db.StringProperty()
>
> > > > > class ThingLoader(bulkloader.Loader):
> > > > > elements = [
> > > > > ('name', str)
> > > > > ]
>
> > > > > def __init__(self):
> > > > > bulkloader.Loader.__init__(self, 'Thing', self.elements)
>
> > > > > def generate_records(self, filename):
> > > > > records = load_records_from_some_datasource()
> > > > > for record in records:
> > > > > yield [
> > > > > record['name']
> > > > > ]
>
> > > > > loaders = [Thing]
>
> > > > > When I try to bulkload this (both locally or to google) I get:
>
> > > > > InternalError: Put accepted 10 entities but returned 0 keys.
>
> > > > > I can't work out why the datastore will not allocate keys. Any ideas
> > > > > what I should do?
>
> > > --
> > > Nick Johnson, Developer Programs Engineer, App Engine
> > > Google Ireland Ltd. :: Registered in Dublin, Ireland, Registration
> > Number:
> > > 368047
>
> --
> Nick Johnson, Developer Programs Engineer, App Engine
> Google Ireland Ltd. :: Registered in Dublin, Ireland, Registration Number:
> 368047
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"Google App Engine" group.
To post to this group, send email to google-appengine@googlegroups.com
To unsubscribe from this group, send email to
google-appengine+unsubscr...@googlegroups.com
For more options, visit this group at
http://groups.google.com/group/google-appengine?hl=en
-~----------~----~----~----~------~----~------~--~---
