Dennis thats the function if you want to have a look at it...
public function executeIndex(sfWebRequest $request) {
ini_set('memory_limit','512M');
set_time_limit(60*5);
sfConfig::set('sf_logging_enabled', false);
sfConfig::set('sf_web_debug', false);
$timer = new Timer;
$timer->start_time();
$this->submited_bulk_stats =
$request->getParameterHolder()->get('bulk_stats');
$this->form = new BulkAddStatsForm();
$this->count_updated_records = 0;
$this->count_inserted_records = 0;
if ($request->isMethod('post'))
{
$this->form->bind($request->getParameter('bulk_stats'),
$request->getFiles('bulk_stats'));
if ($this->form->isValid())
{
$values = $this->form->getValues();
$file = $this->form->getValue('file');
$file_name =
str_replace($file->getOriginalExtension(), '',
$file->getOriginalName());
$save_to =
sfConfig::get('sf_upload_dir').DIRECTORY_SEPARATOR.'uploaded_'.$file_name.'_'.date("dmYHis").$file->getOriginalExtension();
$file->save($save_to);
echo "<br>File saved: $save_to";
echo "<br>File size: ".filesize($save_to);
// get accounts
$accounts = AccountsPeer::doSelect(new
Criteria());
$re_indexed_accounts =
$re_indexed_accounts_by_id = array();
foreach ($accounts as $tmp_account) {
$re_indexed_accounts[$tmp_account->getAffName().$tmp_account->getAffUser()]
= $tmp_account;
$re_indexed_accounts_by_id[$tmp_account->getId()] = $tmp_account;
}
// get parking programs
$parking_programs =
ParkingProgramPeer::doSelect(new Criteria());
$re_indexed_parking_programs = array();
foreach ($parking_programs as $pp) {
$re_indexed_parking_programs[$pp->getId()] = $pp;
}
// Process the uploaded file
$objReader = new PHPExcel_Reader_Excel2007;
$objPHPExcel = $objReader->load($save_to);
// Define allowed columns
$allowed_columns =
array('domain','views','clicks','revenue','currency','date','park','account');
$re_indexed_data = array();
$data =
$objPHPExcel->getActiveSheet()->toArray();
// Get db connections since we are using two
different db's here
$connection_cdb = Propel::getConnection();
$connection_stats =
Propel::getConnection('cdb_stats');
// get domains
$domains_re_indexed = array();
$query = 'SELECT * FROM domains';
$statement = $connection_cdb->prepare($query);
$statement->execute();
while ($dObj = $statement->fetch(PDO::FETCH_OBJ))
{
$domains_re_indexed[$dObj->dn] =
array('domain_id'=>$dObj->domain_ID, 'deal_id'=>$dObj->deal_id);
}
try {
$validated_data = array();
$sfLogger =
sfContext::getInstance()->getLogger();
foreach($data as $row_number => $row) {
$batch_options = array();
// Check its not an empty row
if (!strlen(implode("", $row)))
{
continue;
}
foreach($row as $col_number =>
$col) {
echo "<br>MEMORY:
".memory_get_usage();
if ($row_number == 1) {
$col =
strtolower($col);
$key =
array_search($col, $allowed_columns);
if ($key !==
false) {
$data_indexes[$col_number] = $allowed_columns[$key];
}
} else {
if
(isset($data_indexes[$col_number]) &&
strlen($data_indexes[$col_number]) && strlen($col)) {
if
($data_indexes[$col_number] == "date") {
$col = date("Y-m-d", strtotime(str_replace("/", "-", $col)));
}
$batch_options[$data_indexes[$col_number]] = $col;
}
}
}
if ($row_number > 1) {
// Park info now comes
from the form
if
(!isset($values['parking_program']) ||
!strlen($values['parking_program'])) {
throw new
Exception('Missing or invalid parking program');
} else {
$batch_options['aff_name'] =
strtolower($re_indexed_parking_programs[$values['parking_program']]->getName());
}
// Account info now
comes from the form
if
(!isset($values['accounts']) || !strlen($values['accounts'])) {
throw new
Exception('Missing or invalid parking account');
} else {
$batch_options['aff_user'] =
strtolower($re_indexed_accounts_by_id[$values['accounts']]->getAffUser());
}
// Specify for batch
results to be returned as a serialized array
$batch_results =
array();
$batch_options['return_results'] = true;
$batch_options['row'] =
$row_number;
$batch_options['skip_merge'] = 1;
$output = array();
foreach($batch_options
as $k=>$v) {
$output[] =
"$k:$v";
}
echo "<br>".implode(",
", $output);
flush();
// Validate and set data
$c = new Criteria();
$c->add(DailyStatsTempPeer::DN, $batch_options['domain']);
$c->add(DailyStatsTempPeer::AFF_NAME, $batch_options['aff_name']);
$c->add(DailyStatsTempPeer::AFF_USER, $batch_options['aff_user']);
$c->add(DailyStatsTempPeer::VIEWS, $batch_options['views']);
$c->add(DailyStatsTempPeer::CLICKS, $batch_options['clicks']);
$c->add(DailyStatsTempPeer::REVENUE, $batch_options['revenue']);
$c->add(DailyStatsTempPeer::STAT_DATE, $batch_options['date']);
$c->add(DailyStatsTempPeer::STAT_CURRENCY, $batch_options['currency']);
// Set other data that
needs to be added with the stats
// ~ Set click rate
$c->add(DailyStatsTempPeer::CLICK_RATE,
($c->getValue(DailyStatsTempPeer::VIEWS) ?
$c->getValue(DailyStatsTempPeer::CLICKS)/$c->getValue(DailyStatsTempPeer::VIEWS)
: 0));
// ~ Set revenue per
click
$c->add(DailyStatsTempPeer::REVENUE_PER_CLICK,
($c->getValue(DailyStatsTempPeer::CLICKS) ?
$c->getValue(DailyStatsTempPeer::REVENUE)/$c->getValue(DailyStatsTempPeer::CLICKS)
: 0));
// ~ Set data from
accounts table
if
(isset($re_indexed_accounts[$c->getValue(DailyStatsTempPeer::AFF_NAME).$c->getValue(DailyStatsTempPeer::AFF_USER)]))
{
$account =
$re_indexed_accounts[$c->getValue(DailyStatsTempPeer::AFF_NAME).$c->getValue(DailyStatsTempPeer::AFF_USER)];
$c->add(DailyStatsTempPeer::BONUS, $account->getBonus());
$c->add(DailyStatsTempPeer::REV_SPLIT, $account->getRevSplit());
$c->add(DailyStatsTempPeer::COMPANY, $account->getCompany());
$c->add(DailyStatsTempPeer::ACCOUNT_ID, $account->getId());
} else {
$sfLogger->err("{".basename(__file__)."} Account not found
for Park: {$c->getValue(DailyStatsTempPeer::AFF_NAME)}
Account:{$c->getValue(DailyStatsTempPeer::AFF_USER)} on row
$row_number");
throw new
Exception('Account not found for
Park:'.$c->getValue(DailyStatsTempPeer::AFF_NAME).',
Account:'.$c->getValue(DailyStatsTempPeer::AFF_USER).' on row
'.$row_number);
}
// ~ Set additional
colums data recently added
if
(isset($domains_re_indexed[$batch_options['domain']])) {
$domain =
$domains_re_indexed[$batch_options['domain']];
$c->add(DailyStatsTempPeer::DOMAIN_ID, $domain['domain_id']);
$c->add(DailyStatsTempPeer::DEAL_ID, $domain['deal_id']);
}
if
($batch_options['date'] == '1970-01-01' ||
$batch_options['date'] == '1969-12-31') {
$sfLogger->err("{".basename(__file__)."} Invalid date format:
{$batch_options['date']} on row $row_number");
throw new
Exception('Please change date format in excel file
to dd/mm/yyyy');
}
$checkIfRecordExistsCriteria = new Criteria();
$checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::STAT_DATE));
$checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::DN));
$checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::AFF_NAME));
$checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::AFF_USER));
// Set the daily stat
entree information to be used for logging
$daily_stat_record_info
=
DailyStatsTempPeer::DN.":".$c->getValue(DailyStatsTempPeer::DN).", ".
DailyStatsTempPeer::AFF_NAME.":".$c->getValue(DailyStatsTempPeer::AFF_NAME).",
".
DailyStatsTempPeer::AFF_USER.":".$c->getValue(DailyStatsTempPeer::AFF_USER).",
".
DailyStatsTempPeer::STAT_DATE.":".$c->getValue(DailyStatsTempPeer::STAT_DATE)."
Row: $row_number";
$affected_rows =
DailyStatsTempPeer::doDelete($checkIfRecordExistsCriteria,
$connection_stats);
DailyStatsTempPeer::doInsert($c, $connection_stats);
if ($affected_rows) {
$this->count_updated_records++;
} else {
$this->count_inserted_records++;
}
DealsPeer::clearInstancePool();
DailyStatsTempPeer::clearInstancePool();
AccountsPeer::clearInstancePool();
DomainsPeer::clearInstancePool();
ParkingProgramPeer::clearInstancePool();
}
}
sfBatchHelper::executeBatch('mergeDailyStatsTempToDailyStats', array());
} catch (Exception $e) {
$this->getUser()->setFlash('error',
var_export($e, true), false);
}
}
}
// Remove the uploaded file
if (isset($save_to) && strlen($save_to) &&
file_exists($save_to)) {
unlink($save_to);
}
$this->submited_bulk_stats = $submited_bulk_stats;
}
Works fine with a couple of thousand rows... but its clear to see the
memory going up
Don't know what this will tell you that i haven't already
On Mon, Apr 12, 2010 at 5:28 PM, Dennis Benkert
<spinecras...@googlemail.com> wrote:
> So, how big are these CSV files? How many lines are you processing and
> can you tell us how many entries you're inserting to your DB using
> Propel?
>
> It would be best if you could show us the code that implements what
> you described in your last mail. Investing maybe 30 minutes to write
> an email that desribes the problem you have, what part of your code
> suffers from it and the data you're using will most likely keep you
> away from writing 5 or more emails desribing everything. Plus it will
> stop me asking stupid questions because I have to digg into your
> problem this way. :)
>
> - Dennis
>
> 2010/4/12 Richard U <richard....@gmail.com>:
>> I am uploading a csv file......
>> i require looping over each row... the data in each column has to be
>> processed individually and allocated to the right table...
>>
>>
>>
>>
>> On Mon, Apr 12, 2010 at 3:56 PM, Dennis Benkert
>> <spinecras...@googlemail.com> wrote:
>>> Although I am pretty sure you are frustrated there is no reason to show
>>> disrespect to other people.
>>>
>>> Christian's point is that we cannot help you without getting some
>>> information about your specific problem. Maybe you can provide some code
>>> sample that show us your memory problems. Also it's interesting to know
>>> what you want to do with the code (are you inserting thousands of entries,
>>> looping over many entries and saving them again, etc.).
>>>
>>> - Dennis
>>>
>>> Am 12.04.2010 um 07:41 schrieb Slavka <richard....@gmail.com>:
>>>
>>>> Christian have you just started using symfony? if not then i am sure
>>>> you are aware of the memory leaks i am talking about...
>>>>
>>>> Cheers
>>>>
>>>> On Apr 12, 3:30 pm, Christian Schaefer <cae...@gmail.com> wrote:
>>>>>
>>>>> Hi Slavka,
>>>>>
>>>>> from your tiny description it is not clear whether the leak you
>>>>> experience is a result of Propel or symfony or your usage of them.
>>>>>
>>>>> You should know that symfony 1.2 is no longer supported but replaced
>>>>> by symfony 1.4. The favoured ORM nowadays is Doctrine although Propel
>>>>> is still supported.
>>>>>
>>>>> But most importantly: you posted a complaint rather than a question.
>>>>> This might work with a licenced software from some company but with an
>>>>> open source community a certain netiquette should apply.
>>>>>
>>>>> Cheers
>>>>> /Christian
>>>>>
>>>>> On 12 Apr., 05:51, Slavka <richard....@gmail.com> wrote:
>>>>>
>>>>>
>>>>>
>>>>>> It seems to be pretty clear the symfony 1.2 and propel are still full
>>>>>> of memory leaks... I have a simple for loop selecting the same object
>>>>>> into the same variable... in theory this should not lead to increase
>>>>>> memory use for every loop however that is what is happening...
>>>>>
>>>>>> Has anyone every looked into this... if so what are options to handle
>>>>>> the leaks... , if there is no solution we are going to have to
>>>>>> consider just using mysql functions...
>>>>>
>>>>>> Regards
>>>>
>>>> --
>>>> If you want to report a vulnerability issue on symfony, please send it to
>>>> security at symfony-project.com
>>>>
>>>> You received this message because you are subscribed to the Google
>>>> Groups "symfony users" group.
>>>> To post to this group, send email to symfony-users@googlegroups.com
>>>> To unsubscribe from this group, send email to
>>>> symfony-users+unsubscr...@googlegroups.com
>>>> For more options, visit this group at
>>>> http://groups.google.com/group/symfony-users?hl=en
>>>>
>>>> To unsubscribe, reply using "remove me" as the subject.
>>>
>>> --
>>> If you want to report a vulnerability issue on symfony, please send it to
>>> security at symfony-project.com
>>>
>>> You received this message because you are subscribed to the Google
>>> Groups "symfony users" group.
>>> To post to this group, send email to symfony-users@googlegroups.com
>>> To unsubscribe from this group, send email to
>>> symfony-users+unsubscr...@googlegroups.com
>>> For more options, visit this group at
>>> http://groups.google.com/group/symfony-users?hl=en
>>>
>>
>> --
>> If you want to report a vulnerability issue on symfony, please send it to
>> security at symfony-project.com
>>
>> You received this message because you are subscribed to the Google
>> Groups "symfony users" group.
>> To post to this group, send email to symfony-users@googlegroups.com
>> To unsubscribe from this group, send email to
>> symfony-users+unsubscr...@googlegroups.com
>> For more options, visit this group at
>> http://groups.google.com/group/symfony-users?hl=en
>>
>
> --
> If you want to report a vulnerability issue on symfony, please send it to
> security at symfony-project.com
>
> You received this message because you are subscribed to the Google
> Groups "symfony users" group.
> To post to this group, send email to symfony-users@googlegroups.com
> To unsubscribe from this group, send email to
> symfony-users+unsubscr...@googlegroups.com
> For more options, visit this group at
> http://groups.google.com/group/symfony-users?hl=en
>
--
If you want to report a vulnerability issue on symfony, please send it to
security at symfony-project.com
You received this message because you are subscribed to the Google
Groups "symfony users" group.
To post to this group, send email to symfony-users@googlegroups.com
To unsubscribe from this group, send email to
symfony-users+unsubscr...@googlegroups.com
For more options, visit this group at
http://groups.google.com/group/symfony-users?hl=en
