Thanks Dennis, awesome feedback!
On Mon, Apr 12, 2010 at 5:53 PM, Richard U <richard....@gmail.com> wrote:
> Dennis thats the function if you want to have a look at it...
>
> public function executeIndex(sfWebRequest $request) {
>
> ini_set('memory_limit','512M');
> set_time_limit(60*5);
> sfConfig::set('sf_logging_enabled', false);
> sfConfig::set('sf_web_debug', false);
>
> $timer = new Timer;
> $timer->start_time();
>
> $this->submited_bulk_stats =
> $request->getParameterHolder()->get('bulk_stats');
>
> $this->form = new BulkAddStatsForm();
>
> $this->count_updated_records = 0;
> $this->count_inserted_records = 0;
>
> if ($request->isMethod('post'))
> {
> $this->form->bind($request->getParameter('bulk_stats'),
> $request->getFiles('bulk_stats'));
> if ($this->form->isValid())
> {
> $values = $this->form->getValues();
>
> $file = $this->form->getValue('file');
> $file_name =
> str_replace($file->getOriginalExtension(), '',
> $file->getOriginalName());
> $save_to =
> sfConfig::get('sf_upload_dir').DIRECTORY_SEPARATOR.'uploaded_'.$file_name.'_'.date("dmYHis").$file->getOriginalExtension();
> $file->save($save_to);
>
> echo "<br>File saved: $save_to";
> echo "<br>File size: ".filesize($save_to);
>
> // get accounts
> $accounts = AccountsPeer::doSelect(new
> Criteria());
> $re_indexed_accounts =
> $re_indexed_accounts_by_id = array();
> foreach ($accounts as $tmp_account) {
>
> $re_indexed_accounts[$tmp_account->getAffName().$tmp_account->getAffUser()]
> = $tmp_account;
>
> $re_indexed_accounts_by_id[$tmp_account->getId()] = $tmp_account;
> }
>
> // get parking programs
> $parking_programs =
> ParkingProgramPeer::doSelect(new Criteria());
> $re_indexed_parking_programs = array();
> foreach ($parking_programs as $pp) {
>
> $re_indexed_parking_programs[$pp->getId()] = $pp;
> }
>
> // Process the uploaded file
> $objReader = new PHPExcel_Reader_Excel2007;
> $objPHPExcel = $objReader->load($save_to);
>
> // Define allowed columns
> $allowed_columns =
> array('domain','views','clicks','revenue','currency','date','park','account');
> $re_indexed_data = array();
> $data =
> $objPHPExcel->getActiveSheet()->toArray();
>
> // Get db connections since we are using two
> different db's here
> $connection_cdb = Propel::getConnection();
> $connection_stats =
> Propel::getConnection('cdb_stats');
>
> // get domains
> $domains_re_indexed = array();
> $query = 'SELECT * FROM domains';
> $statement = $connection_cdb->prepare($query);
> $statement->execute();
> while ($dObj = $statement->fetch(PDO::FETCH_OBJ))
> {
> $domains_re_indexed[$dObj->dn] =
> array('domain_id'=>$dObj->domain_ID, 'deal_id'=>$dObj->deal_id);
> }
>
> try {
> $validated_data = array();
> $sfLogger =
> sfContext::getInstance()->getLogger();
>
> foreach($data as $row_number => $row) {
> $batch_options = array();
>
> // Check its not an empty row
> if (!strlen(implode("",
> $row))) {
> continue;
> }
>
> foreach($row as $col_number =>
> $col) {
> echo "<br>MEMORY:
> ".memory_get_usage();
> if ($row_number == 1) {
> $col =
> strtolower($col);
> $key =
> array_search($col, $allowed_columns);
> if ($key !==
> false) {
>
> $data_indexes[$col_number] = $allowed_columns[$key];
> }
> } else {
> if
> (isset($data_indexes[$col_number]) &&
> strlen($data_indexes[$col_number]) && strlen($col)) {
> if
> ($data_indexes[$col_number] == "date") {
>
> $col = date("Y-m-d", strtotime(str_replace("/", "-", $col)));
> }
>
> $batch_options[$data_indexes[$col_number]] = $col;
> }
> }
> }
>
> if ($row_number > 1) {
>
> // Park info now comes
> from the form
> if
> (!isset($values['parking_program']) ||
> !strlen($values['parking_program'])) {
> throw new
> Exception('Missing or invalid parking program');
> } else {
>
> $batch_options['aff_name'] =
> strtolower($re_indexed_parking_programs[$values['parking_program']]->getName());
> }
>
> // Account info now
> comes from the form
> if
> (!isset($values['accounts']) || !strlen($values['accounts'])) {
> throw new
> Exception('Missing or invalid parking account');
> } else {
>
> $batch_options['aff_user'] =
> strtolower($re_indexed_accounts_by_id[$values['accounts']]->getAffUser());
> }
>
> // Specify for batch
> results to be returned as a serialized array
> $batch_results =
> array();
>
> $batch_options['return_results'] = true;
> $batch_options['row']
> = $row_number;
>
> $batch_options['skip_merge'] = 1;
>
> $output = array();
> foreach($batch_options
> as $k=>$v) {
> $output[] =
> "$k:$v";
> }
> echo "<br>".implode(",
> ", $output);
> flush();
>
> // Validate and set
> data
> $c = new Criteria();
>
>
> $c->add(DailyStatsTempPeer::DN, $batch_options['domain']);
>
> $c->add(DailyStatsTempPeer::AFF_NAME, $batch_options['aff_name']);
>
> $c->add(DailyStatsTempPeer::AFF_USER, $batch_options['aff_user']);
>
> $c->add(DailyStatsTempPeer::VIEWS, $batch_options['views']);
>
> $c->add(DailyStatsTempPeer::CLICKS, $batch_options['clicks']);
>
> $c->add(DailyStatsTempPeer::REVENUE, $batch_options['revenue']);
>
> $c->add(DailyStatsTempPeer::STAT_DATE, $batch_options['date']);
>
> $c->add(DailyStatsTempPeer::STAT_CURRENCY, $batch_options['currency']);
>
> // Set other data that
> needs to be added with the stats
> // ~ Set click rate
>
> $c->add(DailyStatsTempPeer::CLICK_RATE,
> ($c->getValue(DailyStatsTempPeer::VIEWS) ?
> $c->getValue(DailyStatsTempPeer::CLICKS)/$c->getValue(DailyStatsTempPeer::VIEWS)
> : 0));
> // ~ Set revenue per
> click
>
> $c->add(DailyStatsTempPeer::REVENUE_PER_CLICK,
> ($c->getValue(DailyStatsTempPeer::CLICKS) ?
> $c->getValue(DailyStatsTempPeer::REVENUE)/$c->getValue(DailyStatsTempPeer::CLICKS)
> : 0));
> // ~ Set data from
> accounts table
>
> if
> (isset($re_indexed_accounts[$c->getValue(DailyStatsTempPeer::AFF_NAME).$c->getValue(DailyStatsTempPeer::AFF_USER)]))
> {
> $account =
> $re_indexed_accounts[$c->getValue(DailyStatsTempPeer::AFF_NAME).$c->getValue(DailyStatsTempPeer::AFF_USER)];
>
> $c->add(DailyStatsTempPeer::BONUS, $account->getBonus());
>
> $c->add(DailyStatsTempPeer::REV_SPLIT, $account->getRevSplit());
>
> $c->add(DailyStatsTempPeer::COMPANY, $account->getCompany());
>
> $c->add(DailyStatsTempPeer::ACCOUNT_ID, $account->getId());
> } else {
>
> $sfLogger->err("{".basename(__file__)."} Account not found
> for Park: {$c->getValue(DailyStatsTempPeer::AFF_NAME)}
>
> Account:{$c->getValue(DailyStatsTempPeer::AFF_USER)} on row
> $row_number");
> throw new
> Exception('Account not found for
> Park:'.$c->getValue(DailyStatsTempPeer::AFF_NAME).',
>
> Account:'.$c->getValue(DailyStatsTempPeer::AFF_USER).' on row
> '.$row_number);
> }
> // ~ Set additional
> colums data recently added
> if
> (isset($domains_re_indexed[$batch_options['domain']])) {
> $domain =
> $domains_re_indexed[$batch_options['domain']];
>
> $c->add(DailyStatsTempPeer::DOMAIN_ID, $domain['domain_id']);
>
> $c->add(DailyStatsTempPeer::DEAL_ID, $domain['deal_id']);
> }
>
> if
> ($batch_options['date'] == '1970-01-01' ||
> $batch_options['date'] == '1969-12-31') {
>
> $sfLogger->err("{".basename(__file__)."} Invalid date format:
> {$batch_options['date']} on row $row_number");
> throw new
> Exception('Please change date format in excel file
> to dd/mm/yyyy');
> }
>
>
> $checkIfRecordExistsCriteria = new Criteria();
>
> $checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::STAT_DATE));
>
> $checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::DN));
>
> $checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::AFF_NAME));
>
> $checkIfRecordExistsCriteria->add($c->getCriterion(DailyStatsTempPeer::AFF_USER));
>
> // Set the daily stat
> entree information to be used for logging
>
> $daily_stat_record_info =
> DailyStatsTempPeer::DN.":".$c->getValue(DailyStatsTempPeer::DN).", ".
>
> DailyStatsTempPeer::AFF_NAME.":".$c->getValue(DailyStatsTempPeer::AFF_NAME).",
> ".
>
> DailyStatsTempPeer::AFF_USER.":".$c->getValue(DailyStatsTempPeer::AFF_USER).",
> ".
>
> DailyStatsTempPeer::STAT_DATE.":".$c->getValue(DailyStatsTempPeer::STAT_DATE)."
> Row: $row_number";
>
> $affected_rows =
> DailyStatsTempPeer::doDelete($checkIfRecordExistsCriteria,
> $connection_stats);
>
> DailyStatsTempPeer::doInsert($c, $connection_stats);
>
> if ($affected_rows) {
>
> $this->count_updated_records++;
> } else {
>
> $this->count_inserted_records++;
> }
>
>
> DealsPeer::clearInstancePool();
>
> DailyStatsTempPeer::clearInstancePool();
>
> AccountsPeer::clearInstancePool();
>
> DomainsPeer::clearInstancePool();
>
> ParkingProgramPeer::clearInstancePool();
> }
> }
>
> sfBatchHelper::executeBatch('mergeDailyStatsTempToDailyStats', array());
>
> } catch (Exception $e) {
> $this->getUser()->setFlash('error',
> var_export($e, true), false);
> }
> }
> }
> // Remove the uploaded file
> if (isset($save_to) && strlen($save_to) &&
> file_exists($save_to)) {
> unlink($save_to);
> }
> $this->submited_bulk_stats = $submited_bulk_stats;
> }
>
>
>
> Works fine with a couple of thousand rows... but its clear to see the
> memory going up
> Don't know what this will tell you that i haven't already
>
>
>
> On Mon, Apr 12, 2010 at 5:28 PM, Dennis Benkert
> <spinecras...@googlemail.com> wrote:
>> So, how big are these CSV files? How many lines are you processing and
>> can you tell us how many entries you're inserting to your DB using
>> Propel?
>>
>> It would be best if you could show us the code that implements what
>> you described in your last mail. Investing maybe 30 minutes to write
>> an email that desribes the problem you have, what part of your code
>> suffers from it and the data you're using will most likely keep you
>> away from writing 5 or more emails desribing everything. Plus it will
>> stop me asking stupid questions because I have to digg into your
>> problem this way. :)
>>
>> - Dennis
>>
>> 2010/4/12 Richard U <richard....@gmail.com>:
>>> I am uploading a csv file......
>>> i require looping over each row... the data in each column has to be
>>> processed individually and allocated to the right table...
>>>
>>>
>>>
>>>
>>> On Mon, Apr 12, 2010 at 3:56 PM, Dennis Benkert
>>> <spinecras...@googlemail.com> wrote:
>>>> Although I am pretty sure you are frustrated there is no reason to show
>>>> disrespect to other people.
>>>>
>>>> Christian's point is that we cannot help you without getting some
>>>> information about your specific problem. Maybe you can provide some code
>>>> sample that show us your memory problems. Also it's interesting to know
>>>> what you want to do with the code (are you inserting thousands of entries,
>>>> looping over many entries and saving them again, etc.).
>>>>
>>>> - Dennis
>>>>
>>>> Am 12.04.2010 um 07:41 schrieb Slavka <richard....@gmail.com>:
>>>>
>>>>> Christian have you just started using symfony? if not then i am sure
>>>>> you are aware of the memory leaks i am talking about...
>>>>>
>>>>> Cheers
>>>>>
>>>>> On Apr 12, 3:30 pm, Christian Schaefer <cae...@gmail.com> wrote:
>>>>>>
>>>>>> Hi Slavka,
>>>>>>
>>>>>> from your tiny description it is not clear whether the leak you
>>>>>> experience is a result of Propel or symfony or your usage of them.
>>>>>>
>>>>>> You should know that symfony 1.2 is no longer supported but replaced
>>>>>> by symfony 1.4. The favoured ORM nowadays is Doctrine although Propel
>>>>>> is still supported.
>>>>>>
>>>>>> But most importantly: you posted a complaint rather than a question.
>>>>>> This might work with a licenced software from some company but with an
>>>>>> open source community a certain netiquette should apply.
>>>>>>
>>>>>> Cheers
>>>>>> /Christian
>>>>>>
>>>>>> On 12 Apr., 05:51, Slavka <richard....@gmail.com> wrote:
>>>>>>
>>>>>>
>>>>>>
>>>>>>> It seems to be pretty clear the symfony 1.2 and propel are still full
>>>>>>> of memory leaks... I have a simple for loop selecting the same object
>>>>>>> into the same variable... in theory this should not lead to increase
>>>>>>> memory use for every loop however that is what is happening...
>>>>>>
>>>>>>> Has anyone every looked into this... if so what are options to handle
>>>>>>> the leaks... , if there is no solution we are going to have to
>>>>>>> consider just using mysql functions...
>>>>>>
>>>>>>> Regards
>>>>>
>>>>> --
>>>>> If you want to report a vulnerability issue on symfony, please send it to
>>>>> security at symfony-project.com
>>>>>
>>>>> You received this message because you are subscribed to the Google
>>>>> Groups "symfony users" group.
>>>>> To post to this group, send email to symfony-users@googlegroups.com
>>>>> To unsubscribe from this group, send email to
>>>>> symfony-users+unsubscr...@googlegroups.com
>>>>> For more options, visit this group at
>>>>> http://groups.google.com/group/symfony-users?hl=en
>>>>>
>>>>> To unsubscribe, reply using "remove me" as the subject.
>>>>
>>>> --
>>>> If you want to report a vulnerability issue on symfony, please send it to
>>>> security at symfony-project.com
>>>>
>>>> You received this message because you are subscribed to the Google
>>>> Groups "symfony users" group.
>>>> To post to this group, send email to symfony-users@googlegroups.com
>>>> To unsubscribe from this group, send email to
>>>> symfony-users+unsubscr...@googlegroups.com
>>>> For more options, visit this group at
>>>> http://groups.google.com/group/symfony-users?hl=en
>>>>
>>>
>>> --
>>> If you want to report a vulnerability issue on symfony, please send it to
>>> security at symfony-project.com
>>>
>>> You received this message because you are subscribed to the Google
>>> Groups "symfony users" group.
>>> To post to this group, send email to symfony-users@googlegroups.com
>>> To unsubscribe from this group, send email to
>>> symfony-users+unsubscr...@googlegroups.com
>>> For more options, visit this group at
>>> http://groups.google.com/group/symfony-users?hl=en
>>>
>>
>> --
>> If you want to report a vulnerability issue on symfony, please send it to
>> security at symfony-project.com
>>
>> You received this message because you are subscribed to the Google
>> Groups "symfony users" group.
>> To post to this group, send email to symfony-users@googlegroups.com
>> To unsubscribe from this group, send email to
>> symfony-users+unsubscr...@googlegroups.com
>> For more options, visit this group at
>> http://groups.google.com/group/symfony-users?hl=en
>>
>
--
If you want to report a vulnerability issue on symfony, please send it to
security at symfony-project.com
You received this message because you are subscribed to the Google
Groups "symfony users" group.
To post to this group, send email to symfony-users@googlegroups.com
To unsubscribe from this group, send email to
symfony-users+unsubscr...@googlegroups.com
For more options, visit this group at
http://groups.google.com/group/symfony-users?hl=en
