I don’t usually fetch all objects but fetch in batches.
You need to recycle the EOEditingContext. I tend to bring the array in batches,
process, then create a new editing context and repeat.
I have some code included below but I think there is also good batching support
in Wonder. The code below fetches raw rows and then converts to an EO when
requested. The code below will automatically use a new editing context for each
batch.
Mark
package com.eldrix.snomedct.search;
import java.util.Iterator;
import org.apache.log4j.Logger;
import com.webobjects.eoaccess.EODatabaseContext;
import com.webobjects.eoaccess.EOEntity;
import com.webobjects.eoaccess.EOSQLExpression;
import com.webobjects.eocontrol.EOEditingContext;
import com.webobjects.eocontrol.EOFetchSpecification;
import com.webobjects.eocontrol.EOSortOrdering;
import com.webobjects.foundation.NSArray;
import com.webobjects.foundation.NSDictionary;
import er.extensions.eof.ERXEC;
import er.extensions.eof.ERXEOAccessUtilities;
import er.extensions.eof.ERXEOControlUtilities;
/**
* This is an optimised memory-efficient method of iterating through enterprise
objects.
* This will iterate through enterprise objects, either returning the primary
key, the object or the raw row.
*
* For example:
* <code>
* <pre>
* for(EOEnterpriseObject eo:
IterableBatchEnterpriseObjects.allObjectsOfEntity("Patient", 300) {
* // process eo
* }
* </pre>
* </code>
*
* @author mark
*
*/
@SuppressWarnings("rawtypes")
public class IterableBatchEnterpriseObjects implements Iterable<Object> {
private static Logger log =
Logger.getLogger(IterableBatchEnterpriseObjects.class);
private final static int DEFAULT_BATCH_SIZE=50;
private EOEditingContext _ec;
private EOFetchSpecification _fs;
private int _rowCount;
private int _start;
private int _batchSize;
private NSArray _currentBatch;
private int _batchPosition;
private IterableBatchMode _mode;
private boolean _refreshEditingContext = true; // we create a
new editing context for each batch by default
private enum IterableBatchMode {
PRIMARY_KEYS {
NSArray resultsInRange(EOEditingContext ec,
EOFetchSpecification fs, int start, int end) {
return
ERXEOControlUtilities.primaryKeyValuesInRange(ec, fs, start, end);
}
},
OBJECTS {
NSArray resultsInRange(EOEditingContext ec,
EOFetchSpecification fs, int start, int end) {
return ERXEOControlUtilities.objectsInRange(ec,
fs, start, end);
}
},
RAW_ROWS {
NSArray resultsInRange(EOEditingContext ec,
EOFetchSpecification fs, int start, int end) {
return rawRowsInRange(ec, fs, start, end);
}
};
abstract NSArray resultsInRange(EOEditingContext ec,
EOFetchSpecification fs, int start, int end);
};
private IterableBatchEnterpriseObjects(EOFetchSpecification fs, int
batchSize, IterableBatchMode mode) {
_fs = fs;
_batchSize = batchSize;
_mode = mode;
_rowCount =
ERXEOAccessUtilities.rowCountForFetchSpecification(editingContext(), _fs);
}
public static IterableBatchEnterpriseObjects allObjectsOfEntity(String
entityName, NSArray<EOSortOrdering> sortOrderings, int batchSize) {
EOFetchSpecification fs = new EOFetchSpecification(entityName,
null, sortOrderings);
return new IterableBatchEnterpriseObjects(fs, batchSize,
IterableBatchMode.OBJECTS);
}
public static IterableBatchEnterpriseObjects allObjectsOfEntity(String
entityName, int batchSize) {
return allObjectsOfEntity(entityName,
sortByPrimaryKey(entityName), batchSize);
}
public static IterableBatchEnterpriseObjects allObjectsOfEntity(String
entityName) {
return allObjectsOfEntity(entityName, DEFAULT_BATCH_SIZE);
}
public static IterableBatchEnterpriseObjects
allPrimaryKeysOfEntity(String entityName, NSArray<EOSortOrdering>
sortOrderings, int batchSize) {
EOFetchSpecification fs = new EOFetchSpecification(entityName,
null, sortOrderings);
return new IterableBatchEnterpriseObjects(fs, batchSize,
IterableBatchMode.PRIMARY_KEYS);
}
public static IterableBatchEnterpriseObjects
allPrimaryKeysOfEntity(String entityName, int batchSize) {
return allPrimaryKeysOfEntity(entityName,
sortByPrimaryKey(entityName), batchSize);
}
public static IterableBatchEnterpriseObjects
allPrimaryKeysOfEntity(String entityName) {
return allPrimaryKeysOfEntity(entityName, DEFAULT_BATCH_SIZE);
}
public static IterableBatchEnterpriseObjects
objects(EOFetchSpecification fs, int batchSize) {
return new IterableBatchEnterpriseObjects(fs, batchSize,
IterableBatchMode.OBJECTS);
}
public static IterableBatchEnterpriseObjects
primaryKeys(EOFetchSpecification fs, int batchSize) {
return new IterableBatchEnterpriseObjects(fs, batchSize,
IterableBatchMode.PRIMARY_KEYS);
}
public static IterableBatchEnterpriseObjects
rawRows(EOFetchSpecification fs, int batchSize) {
return new IterableBatchEnterpriseObjects(fs, batchSize,
IterableBatchMode.RAW_ROWS);
}
public static NSArray<EOSortOrdering> sortByPrimaryKey(String
entityName) {
EOEntity entity = ERXEOAccessUtilities.entityNamed(null,
entityName);
NSArray<String> attrs = entity.primaryKeyAttributeNames();
EOSortOrdering so =
EOSortOrdering.sortOrderingWithKey(attrs.get(0),
EOSortOrdering.CompareAscending);
return new NSArray<EOSortOrdering>(so);
}
/**
* Returns an {@link com.webobjects.foundation.NSArray NSArray}
containing the raw rows for the fetch specification.
* This is based upon ERXEOControlUtilities.primaryKeyValuesInRange
* @param ec
* @param spec
* @param start
* @param end
* @see er.extensions.eof.ERXEOControlUtilities#primaryKeyValuesInRange
* @return
*/
public static NSArray rawRowsInRange(EOEditingContext ec,
EOFetchSpecification spec, int start, int end) {
EOEntity entity = ERXEOAccessUtilities.entityNamed(ec,
spec.entityName());
NSArray attributeNames = (NSArray)
entity.attributes().valueForKey("name");
spec.setFetchesRawRows(true);
spec.setRawRowKeyPaths(attributeNames);
EOFetchSpecification clonedFetchSpec =
(EOFetchSpecification)spec.clone();
EOSQLExpression sql =
ERXEOAccessUtilities.sqlExpressionForFetchSpecification(ec, clonedFetchSpec,
start, end);
NSDictionary<String, EOSQLExpression> hints = new NSDictionary<String,
EOSQLExpression>(sql, EODatabaseContext.CustomQueryExpressionHintKey);
clonedFetchSpec.setHints(hints);
return ec.objectsWithFetchSpecification(clonedFetchSpec);
}
public EOEditingContext editingContext() {
if (_ec == null) {
_ec = ERXEC.newEditingContext();
}
return _ec;
}
public void setEditingContext(EOEditingContext ec) {
_ec = ec;
_refreshEditingContext = false;
}
protected EOFetchSpecification fetchSpecification() {
return _fs;
}
protected NSArray currentBatch() {
if (_currentBatch == null) {
if (_start > _rowCount) return null;
int end = _start + _batchSize;
if (end > _rowCount) end = _rowCount;
log.debug("Fetching batch... start: " + _start + " end:
" + end);
_currentBatch = _mode.resultsInRange(editingContext(),
fetchSpecification(), _start, end);
_batchPosition = 0;
}
return _currentBatch;
}
/**
* Fetch next batch, optionally creating a new editingcontext for this
new batch
* @return
*/
protected NSArray nextBatch() {
_start += _batchSize;
_currentBatch = null;
if (refreshEditingContext()==true) _ec = null;
return currentBatch();
}
public boolean hasNext() {
if (_start + _batchPosition >= _rowCount) return false;
return true;
}
public Object next() {
Object eo = null;
NSArray batch = currentBatch();
if (_batchPosition == batch.size()) {
batch = nextBatch();
}
if (batch != null) {
eo = batch.get(_batchPosition);
_batchPosition++ ;
}
return eo;
}
public void setRefreshEditingContext(boolean refresh) {
_refreshEditingContext = refresh;
}
public boolean refreshEditingContext() {
return _refreshEditingContext;
}
protected class EnterpriseObjectPrimaryKeyIterator implements Iterator {
public boolean hasNext() {
return IterableBatchEnterpriseObjects.this.hasNext();
}
public Object next() {
return IterableBatchEnterpriseObjects.this.next();
}
public void remove() {
throw new UnsupportedOperationException();
}
}
public Iterator<Object> iterator() {
return new EnterpriseObjectPrimaryKeyIterator();
}
}
> On 10 Mar 2016, at 10:51, Daniele Corti <ilde...@gmail.com> wrote:
>
> Hi,
> I have a question, about performance while working with LongRequest and many
> EO.
>
> Here's the situation: I've prepared a class that extends
> ERXLongResponseTask.DefaultImplementation.
> In this class, I operate a database migration, over 10000 record.
> Each of them has, at least, 50/60 objects releated. From them, I create
> others objects, for a total of 60 new objects for each of the starting 10000
> rows.
>
> The procedure works fine, but, I experience a difference in speed during the
> procedure. At first, 300-400 objects are processed in few seconds, then the
> procedure slow down until, from 5000, it processes about 7/8 objects in 5
> seconds.
>
> The procedure is very simple:
> 1. Fetch all objects in a NSArray
> 2. Manipulate each object in a for loop, fetching related objects and
> creating other objects in every iteration.
>
> I use just one EOEditingContext, so, I think the slowness begin when it
> became really full of elements.
>
> I would like to know, if there is a way to "clean up" the manipulated objects
> from the EC, or if a different approach is preferred.
>
> One last thing: it is good to use a NSArray of 1000 object in the for loop to
> do this? I ask this because, I fetch all the objects and save them in memory
> using EOObject.fetchAllObjects(EC) method from the class.
>
> Thanks in advance!
> Daniele C.
> _______________________________________________
> Do not post admin requests to the list. They will be ignored.
> Webobjects-dev mailing list (Webobjects-dev@lists.apple.com)
> Help/Unsubscribe/Update your Subscription:
> https://lists.apple.com/mailman/options/webobjects-dev/mark%40wardle.org
>
> This email sent to m...@wardle.org
_______________________________________________
Do not post admin requests to the list. They will be ignored.
Webobjects-dev mailing list (Webobjects-dev@lists.apple.com)
Help/Unsubscribe/Update your Subscription:
https://lists.apple.com/mailman/options/webobjects-dev/archive%40mail-archive.com
This email sent to arch...@mail-archive.com
