Author: rande
Date: 2010-02-19 10:14:52 +0100 (Fri, 19 Feb 2010)
New Revision: 28136
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/behavior/sfLuceneDoctrineListener.class.php
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneActionIndexer.class.php
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexer.class.php
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexerHandler.class.php
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneIndexer.class.php
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneModelIndexer.class.php
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelSystemTask.class.php
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelTask.class.php
Log:
[sfSolrPlugin] fix indexing task, add more logging information
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/behavior/sfLuceneDoctrineListener.class.php
===================================================================
---
plugins/sfSolrPlugin/branches/sf1.2/lib/behavior/sfLuceneDoctrineListener.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++
plugins/sfSolrPlugin/branches/sf1.2/lib/behavior/sfLuceneDoctrineListener.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -79,9 +79,12 @@
return;
}
- if(sfContext::hasInstance())
+ try{
+ sfProjectConfiguration::getActive()->getEventDispatcher()->notify(new
sfEvent($this, 'indexer.log', array('deleting model "%s" with PK = "%s"',
get_class($node), current($node->identifier()))));
+ }
+ catch(RuntimeException $e)
{
- sfContext::getInstance()->getEventDispatcher()->notify(new
sfEvent($this, 'lucene.log', array('{sfLucene} deleting model "%s" with PK =
"%s"', get_class($node), current($node->identifier()))));
+ // no active ProjectConfiguration
}
foreach ($this->getSearchInstances($node) as $instance)
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneActionIndexer.class.php
===================================================================
---
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneActionIndexer.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneActionIndexer.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -52,7 +52,7 @@
if ($this->deleteGuid( $this->getGuid($params)))
{
- $this->getSearch()->getEventDispatcher()->notify(new sfEvent($this,
'indexer.log', array('Deleted action "%s" of module "%s" from index', $action,
$module)));
+ $this->getSearch()->getEventDispatcher()->notify(new sfEvent($this,
'application.log', array('Deleted action "%s" of module "%s" from index',
$action, $module)));
}
return $this;
@@ -101,7 +101,7 @@
$this->addDocument($doc, $guid, 'action');
- $this->getSearch()->getEventDispatcher()->notify(new sfEvent($this,
'indexer.log', array('Inserted action "%s" of module "%s" to index',
$this->getAction(), $this->getModule())));
+ $this->getSearch()->getEventDispatcher()->notify(new sfEvent($this,
'application.log', array('Inserted action "%s" of module "%s" to index',
$this->getAction(), $this->getModule())));
return $this;
}
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexer.class.php
===================================================================
---
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexer.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexer.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -185,10 +185,11 @@
// build getter by converting from underscore case to camel case
try
{
- $value = $this->getModel()->$getter();
+ $value = call_user_func(array($this->getModel(), $getter));
}
catch(Doctrine_Record_Exception $e)
{
+
$model_properies = $this->getModelProperties();
// some fields can be only used as a definition
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexerHandler.class.php
===================================================================
---
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexerHandler.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneDoctrineIndexerHandler.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -44,8 +44,9 @@
'limit' => $limit
)));
- $offset = $page * $limit;
- $this->batchRebuild(clone $query, $offset, $limit);
+ $offset = ($page - 1) * $limit;
+
+ $this->batchRebuild($query, $offset, $limit);
}
}
@@ -73,24 +74,31 @@
}
public function batchRebuild($query, $offset, $limit)
- {
+ {
+
+ $start = microtime(true);
+ $this->getSearch()->getEventDispatcher()->notify(new
sfEvent($this,'indexer.log', array('batch indexing - offset:%s, limit:%s.',
$offset, $limit)));
+
$collection = $query->limit($limit)->offset($offset)->execute();
+ $this->getSearch()->getEventDispatcher()->notify(new
sfEvent($this,'indexer.log', array(' - fetching and hydrating objects in %s
seconds.', number_format(microtime(true) - $start, 5))));
+
$documents = array();
$pks = array();
+
+ $start = microtime(true);
foreach($collection as $record)
- {
+ {
+ $start1 = microtime(true);
$doc = $this->getFactory()->getModel($record)->getDocument();
+ // $this->getSearch()->getEventDispatcher()->notify(new
sfEvent($this,'indexer.log', array(' - creating one object in %s seconds.',
number_format(microtime(true) - $start1, 5))));
if(!$doc)
{
- $this->getSearch()->getEventDispatcher()->notify(
- new sfEvent($this, 'application.log', array(
- sprintf('invalid document %s [id:%s]: ', get_class($record),
$record->identifier()),
- 'priority' => sfLogger::ALERT
- ))
- );
+ $this->getSearch()->getEventDispatcher()->notify( new sfEvent($this,
'application.log', array( sprintf('invalid document %s [id:%s]: ',
get_class($record), $record->identifier()), 'priority' => sfLogger::ALERT )));
+ $this->getSearch()->getEventDispatcher()->notify( new sfEvent($this,
'indexer.log', array( sprintf(' - invalid document %s [id:%s]: ',
get_class($record), $record->identifier()), 'priority' => sfLogger::ALERT )));
+
continue;
}
@@ -99,44 +107,26 @@
$field = $doc->getField('id');
$pks[] = $field['value']['0'];
-
- unset($record);
}
+
+ $this->getSearch()->getEventDispatcher()->notify(new
sfEvent($this,'indexer.log', array(' - creating solr documents in %s
seconds.', number_format(microtime(true) - $start, 5))));
$search_engine = $this->getSearch()->getSearchService();
try
{
+ $start = microtime(true);
+
$search_engine->deleteByMultipleIds(array_keys($documents));
$search_engine->addDocuments($documents);
$search_engine->commit();
- $this->getSearch()->getEventDispatcher()->notify(
- new sfEvent(
- $this,
- 'indexer.log',
- array('indexing %s objects - primary keys [%s]', count($documents),
implode(', ', $pks))
- )
- );
+ $this->getSearch()->getEventDispatcher()->notify(new
sfEvent($this,'indexer.log', array(' - indexing %s objects in %s seconds.',
count($documents), number_format(microtime(true) - $start, 5))));
}
catch(Exception $e)
{
- $this->getSearch()->getEventDispatcher()->notify(
- new sfEvent(
- $this,
- 'indexer.log',
- array('indexing Failed indexing object - primary keys [%s]',
implode(', ', $pks))
- )
- );
-
- $this->getSearch()->getEventDispatcher()->notify(
- new sfEvent($this, 'application.log', array(
- 'indexing document fail : '.$e->getMessage(),
- 'priority' => sfLogger::ALERT
- ))
- );
+ $this->getSearch()->getEventDispatcher()->notify(new sfEvent($this,
'indexer.log', array('indexing Failed indexing object - primary keys [%s]',
implode(', ', $pks))));
+ $this->getSearch()->getEventDispatcher()->notify(new sfEvent($this,
'application.log', array('indexing document fail :
'.$e->getMessage(),'priority' => sfLogger::ALERT)));
}
-
- unset($collection);
}
}
\ No newline at end of file
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneIndexer.class.php
===================================================================
--- plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneIndexer.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++ plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneIndexer.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -64,12 +64,7 @@
}
catch(Exception $e)
{
- $this->getConfiguration()->getEventDispatcher()->notify(
- new sfEvent($this, 'application.log', array(
- 'indexing document fail : '.$e->getMessage(),
- 'priority' => sfLogger::ALERT
- ))
- );
+ $this->getConfiguration()->getEventDispatcher()->notify(new
sfEvent($this, 'application.log', array('indexing document fail :
'.$e->getMessage(),'priority' => sfLogger::ALERT)));
}
return $this;
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneModelIndexer.class.php
===================================================================
---
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneModelIndexer.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++
plugins/sfSolrPlugin/branches/sf1.2/lib/indexer/sfLuceneModelIndexer.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -18,6 +18,9 @@
abstract class sfLuceneModelIndexer extends sfLuceneIndexer
{
+ static
+ $model_properties = array();
+
private
$instance,
$model_name; // model name used in the search.yml file
@@ -84,8 +87,13 @@
*/
protected function getModelProperties()
{
-
- return
$this->getSearch()->getParameter('models')->get($this->getModelName());
+ $model_name = $this->getModelName();
+ if(!isset(self::$model_properties[$model_name]))
+ {
+ self::$model_properties[$model_name] =
$this->getSearch()->getParameter('models')->get($model_name);
+ }
+
+ return self::$model_properties[$model_name];
}
/**
@@ -191,6 +199,7 @@
{
$field_properties = $properties->get('fields')->get($field);
+
$value = $this->getFieldValue($field, $field_properties);
// do not index null value
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelSystemTask.class.php
===================================================================
---
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelSystemTask.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelSystemTask.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -23,17 +23,15 @@
protected function configure()
{
$this->addArguments(array(
- new sfCommandArgument('application', sfCommandArgument::REQUIRED, 'The
application name'),
- new sfCommandArgument('index', sfCommandArgument::REQUIRED, 'The name of
the index to rebuild'),
- new sfCommandArgument('culture', sfCommandArgument::REQUIRED, 'The name of
the culture to rebuild'),
+ new sfCommandArgument('application', sfCommandArgument::REQUIRED, 'The
application name'),
+ new sfCommandArgument('index', sfCommandArgument::REQUIRED, 'The name of
the index to rebuild'),
+ new sfCommandArgument('culture', sfCommandArgument::REQUIRED, 'The name
of the culture to rebuild'),
));
$this->addOptions(array(
- new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The
environment', 'search'),
- new sfCommandOption('model', null, sfCommandOption::PARAMETER_OPTIONAL,
'The model to reindex', null),
- new sfCommandOption('limit', null, sfCommandOption::PARAMETER_OPTIONAL,
'The environment', 50),
- new sfCommandOption('delete', null, sfCommandOption::PARAMETER_OPTIONAL,
'set to true to delete all related record', false),
-
+ new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED,
'The environment', 'search'),
+ new sfCommandOption('model', null, sfCommandOption::PARAMETER_OPTIONAL,
'The model to reindex', null),
+ new sfCommandOption('delete', null, sfCommandOption::PARAMETER_OPTIONAL,
'set to true to delete all related record', false),
));
$this->aliases = array('lucene-rebuild-system');
@@ -65,12 +63,10 @@
$index = $arguments['index'];
$culture = $arguments['culture'];
- $limit = $options['limit'];
$model = $options['model'];
$delete = $options['delete'];
$this->checkAppExists($app);
- $this->standardBootstrap($app, $options['env']);
if(sfConfig::get('sf_orm') != 'doctrine')
{
@@ -82,39 +78,25 @@
$search = sfLucene::getInstance($index, $culture, $this->configuration);
$search->optimize();
-
- $this->setupEventDispatcher($search);
- $models = $search->getParameter('models')->getAll();
+ $models = $model ? array($model) :
array_keys($search->getParameter('models')->getAll());
-
- if($model)
+ foreach($models as $model)
{
if($delete)
{
$this->deleteModel($search, $model);
}
+
$this->update($app, $index, $culture, $model, $limit);
}
- else
- {
- foreach($models as $model => $params)
- {
- if($delete)
- {
- $this->deleteModel($search, $model);
- }
-
- $this->update($app, $index, $culture, $model, $limit);
- }
- }
$time = microtime(true) - $start;
- $final = $this->formatter->format('Update index done !!', array('fg' =>
'green', 'bold' => true));
- $final .= $this->formatter->format(number_format($time, 5), array('fg' =>
'cyan')) . ' seconds.';
+ $final = $this->formatter->format('Update index done !!', array('fg' =>
'green', 'bold' => true));
+ $final .= $this->formatter->format(number_format($time, 5), array('fg' =>
'cyan')) . ' seconds.';
- $this->dispatcher->notify(new sfEvent($this, 'command.log', array('',
$final)));
+ $this->dispatcher->notify(new sfEvent($this, 'command.log', array('',
$final)));
}
public function deleteModel(sfLucene $lucene, $model)
@@ -122,6 +104,7 @@
$query = 'sfl_model:'.$model;
$lucene->getLucene()->deleteByQuery($query);
$lucene->getLucene()->commit();
+ $lucene->getLucene()->optimize();
}
public function getFilestatePath($model)
@@ -130,7 +113,7 @@
return sprintf(sfConfig::get('sf_data_dir').'/solr_index/update_%s.state',
sfInflector::underscore($model));
}
- public function update($app, $index, $culture, $model, $limit)
+ public function update($app, $index, $culture, $model)
{
$file = $this->getFilestatePath($model);
@@ -150,9 +133,7 @@
$app,
$index,
$culture,
- $model,
- $limit,
- $offset
+ $model
);
try
Modified:
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelTask.class.php
===================================================================
---
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelTask.class.php
2010-02-19 06:18:30 UTC (rev 28135)
+++
plugins/sfSolrPlugin/branches/sf1.2/lib/task/sfLuceneUpdateModelTask.class.php
2010-02-19 09:14:52 UTC (rev 28136)
@@ -69,8 +69,8 @@
$model = $arguments['model'];
$state = $options['state'];
- $offset = $options['offset'];
$limit = $options['limit'];
+ $page = $options['page'];
$delete = $options['delete'];
$this->checkAppExists($app);
@@ -82,7 +82,6 @@
throw new LogicException('This feature is only implemented for Doctrine
ORM');
}
-
if($state)
{
// use state file
@@ -93,17 +92,17 @@
$this->logSection('lucene', sprintf('Loading state page:%s, limit:%s',
$page, $limit));
}
-
$this->configuration->getEventDispatcher()->connect('lucene.indexing_loop',
array($this, 'handleMemoryLimitEvent'));
+ $this->dispatcher->connect('lucene.indexing_loop', array($this,
'handleMemoryLimitEvent'));
$instance = sfLucene::getInstance($index, $culture, $this->configuration);
-
+
$this->setupEventDispatcher($instance);
if($delete)
{
$query = 'sfl_model:'.$model;
- $instance->getLucene()->deleteByQuery($query);
- $instance->getLucene()->commit();
+ $instance->getSearchService()->deleteByQuery($query);
+ $instance->getSearchService()->commit();
}
$this->rebuild($instance, $model, $page, $limit);
@@ -159,8 +158,7 @@
public function saveState($model, $state)
{
-
- $file =
sprintf(sfConfig::get('sf_data_dir').'/solr_index/update_%s.state', $model);
+ $file =
sprintf(sfConfig::get('sf_data_dir').'/solr_index/update_%s.state',
sfInflector::underscore($model));
file_put_contents($file, serialize($state));
}
--
You received this message because you are subscribed to the Google Groups
"symfony SVN" group.
To post to this group, send email to [email protected].
To unsubscribe from this group, send email to
[email protected].
For more options, visit this group at
http://groups.google.com/group/symfony-svn?hl=en.
