Eileen has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/379174 )
Change subject: Update Omnimail jobs to support storing settings about multiple
jobs.
......................................................................
Update Omnimail jobs to support storing settings about multiple jobs.
Omnimail jobs store information about where the job is up to. This change
permits an input parameter 'job_suffix' to denote a job,
allowing multiple jobs of the same type to be tracked.
This might be used if a slow back-fill job is running off-peak
(ahem) due to possibly a mistake cause some data to have been missed
.....
without interfering with the up-to-date job.
Bug: T176255
Change-Id: If0a709b9e9cec7e3276d231e75cca49d2909d4cb
---
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
M
sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
5 files changed, 131 insertions(+), 64 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm
refs/changes/74/379174/1
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
index b40f880..c700aa8 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/CRM/Omnimail/Omnimail.php
@@ -42,6 +42,15 @@
protected $retrievalParameters;
/**
+ * @var string
+ */
+ protected $job_suffix;
+
+ /**
+ * @var string
+ */
+ protected $mail_provider;
+ /**
* CRM_Omnimail_Omnimail constructor.
*
* @param array $params
@@ -49,6 +58,8 @@
* @throws \API_Exception
*/
public function __construct($params) {
+ $this->job_suffix = $params['job_suffix'];
+ $this->mail_provider = $params['mail_provider'];
$this->setJobSettings($params);
$this->setOffset($params);
$this->setRetrievalParameters(CRM_Utils_Array::value('retrieval_parameters',
$this->jobSettings));
@@ -150,7 +161,20 @@
*/
protected function setJobSettings($params) {
$settings = CRM_Omnimail_Helper::getSettings();
- $this->jobSettings = CRM_Utils_Array::value($params['mail_provider'],
$settings['omnimail_' . $this->job . '_load'], array());
+ $this->jobSettings = CRM_Utils_Array::value($params['mail_provider'] .
$this->job_suffix, $settings['omnimail_' . $this->job . '_load'], array());
+ }
+
+ /**
+ * Save the job settings.
+ *
+ * @param array $setting
+ */
+ function saveJobSetting($setting) {
+ civicrm_api3('Setting', 'create', array(
+ 'omnimail_' . $this->job . '_load' => array(
+ $this->mail_provider . $this->job_suffix => $setting,
+ ),
+ ));
}
}
\ No newline at end of file
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
index eba0d6c..e0e8b7f 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnigroupmember/Load.php
@@ -27,15 +27,11 @@
$contacts = $job->getResult($params);
}
catch (CRM_Omnimail_IncompleteDownloadException $e) {
- civicrm_api3('Setting', 'create', array(
- 'omnimail_omnigroupmembers_load' => array(
- $params['mail_provider'] => array(
- 'last_timestamp' => $jobSettings['last_timestamp'],
- 'retrieval_parameters' => $e->getRetrievalParameters(),
- 'progress_end_date' => $e->getEndTimestamp(),
- 'offset' => 0,
- ),
- ),
+ $job->saveJobSetting(array(
+ 'last_timestamp' => $jobSettings['last_timestamp'],
+ 'retrieval_parameters' => $e->getRetrievalParameters(),
+ 'progress_end_date' => $e->getEndTimestamp(),
+ 'offset' => 0,
));
return civicrm_api3_create_success(1);
}
@@ -43,26 +39,17 @@
$defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
$locationTypeID = $defaultLocationType->id;
- if (isset($params['options']['offset'])) {
- $offset = $params['options']['offset'];
- }
- else {
- $offset = CRM_Utils_Array::value('offset', $jobSettings, 0);
- }
+ $offset = $job->getOffset();
$limit = (isset($params['options']['limit'])) ? $params['options']['limit']
: NULL;
$count = 0;
foreach ($contacts as $contact) {
if ($count === $limit) {
- civicrm_api3('Setting', 'create', array(
- 'omnimail_omnigroupmembers_load' => array(
- $params['mail_provider'] => array(
- 'last_timestamp' => $jobSettings['last_timestamp'],
- 'retrieval_parameters' => $job->getRetrievalParameters(),
- 'progress_end_date' => $job->endTimeStamp,
- 'offset' => $offset,
- ),
- ),
+ $job->saveJobSetting(array(
+ 'last_timestamp' => $jobSettings['last_timestamp'],
+ 'retrieval_parameters' => $job->getRetrievalParameters(),
+ 'progress_end_date' => $job->endTimeStamp,
+ 'offset' => $offset + $count,
));
// Do this here - ie. before processing a new row rather than at the end
of the last row
// to avoid thinking a job is incomplete if the limit co-incides with
available rows.
@@ -101,16 +88,10 @@
}
$values[$contact['id']] = reset($contact['values']);
}
- $offset++;
+
$count++;
// Every row seems extreme but perhaps not in this performance monitoring
phase.
- civicrm_api3('Setting', 'create', array(
- 'omnimail_omnigroupmembers_load' => array(
- $params['mail_provider'] => array_merge(
- $jobSettings, array('offset' => $offset)
- )
- ),
- ));
+ $job->saveJobSetting(array_merge($jobSettings, array('offset' => $offset +
$count)));
$rowsLeftBeforeThrottle--;
if ($throttleStagePoint && (strtotime('now') > $throttleStagePoint)) {
@@ -122,11 +103,8 @@
sleep(ceil($throttleStagePoint - strtotime('now')));
}
}
- civicrm_api3('Setting', 'create', array(
- 'omnimail_omnigroupmembers_load' => array(
- $params['mail_provider'] => array('last_timestamp' =>
$job->endTimeStamp),
- ),
- ));
+
+ $job->saveJobSetting(array('last_timestamp' => $job->endTimeStamp));
return civicrm_api3_create_success($values);
}
@@ -245,5 +223,11 @@
'type' => CRM_Utils_Type::T_INT,
'api.default' => 60,
);
+ $params['job_suffix'] = array(
+ 'title' => ts('A suffix string to add to job-specific settings.'),
+ 'description' => ts('The suffix allows for multiple settings to be stored
for one job. For example if wishing to run an up-top-date job and a catch-up
job'),
+ 'type' => CRM_Utils_Type::T_STRING,
+ 'api.default' => '',
+ );
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
index 056f045..5a328cc 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/api/v3/Omnirecipient/Load.php
@@ -44,15 +44,11 @@
foreach ($recipients as $recipient) {
if ($count === $limit) {
- civicrm_api3('Setting', 'create', array(
- 'omnimail_omnirecipient_load' => array(
- $params['mail_provider'] => array(
- 'last_timestamp' => $jobSettings['last_timestamp'],
- 'retrieval_parameters' => $omnimail->getRetrievalParameters(),
- 'progress_end_date' => $omnimail->endTimeStamp,
- 'offset' => $omnimail->getOffset() + $count,
- ),
- ),
+ $omnimail->saveJobSetting(array(
+ 'last_timestamp' => $jobSettings['last_timestamp'],
+ 'retrieval_parameters' => $omnimail->getRetrievalParameters(),
+ 'progress_end_date' => $omnimail->endTimeStamp,
+ 'offset' => $omnimail->getOffset() + $count,
));
// Do this here - ie. before processing a new row rather than at the
end of the last row
// to avoid thinking a job is incomplete if the limit co-incides with
available rows.
@@ -88,22 +84,14 @@
sleep(ceil($throttleStagePoint - strtotime('now')));
}
}
- civicrm_api3('Setting', 'create', array(
- 'omnimail_omnirecipient_load' => array(
- $params['mail_provider'] => array('last_timestamp' =>
$omnimail->endTimeStamp),
- ),
- ));
+ $omnimail->saveJobSetting(array('last_timestamp' =>
$omnimail->endTimeStamp));
return civicrm_api3_create_success(1);
}
catch (CRM_Omnimail_IncompleteDownloadException $e) {
- civicrm_api3('Setting', 'create', array(
- 'omnimail_omnirecipient_load' => array(
- $params['mail_provider'] => array(
- 'last_timestamp' => $omnimail->getStartTimestamp($params),
- 'retrieval_parameters' => $e->getRetrievalParameters(),
- 'progress_end_date' => $e->getEndTimestamp(),
- ),
- ),
+ $omnimail->saveJobSetting(array(
+ 'last_timestamp' => $omnimail->getStartTimestamp($params),
+ 'retrieval_parameters' => $e->getRetrievalParameters(),
+ 'progress_end_date' => $e->getEndTimestamp(),
));
return civicrm_api3_create_success(1);
}
@@ -156,5 +144,11 @@
'type' => CRM_Utils_Type::T_INT,
'api.default' => 300,
);
+ $params['job_suffix'] = array(
+ 'title' => ts('A suffix string to add to job-specific settings.'),
+ 'description' => ts('The suffix allows for multiple settings to be stored
for one job. For example if wishing to run an up-top-date job and a catch-up
job'),
+ 'type' => CRM_Utils_Type::T_STRING,
+ 'api.default' => '',
+ );
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
index 46832dd..0a7b36f 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnigroupmemberLoadTest.php
@@ -144,6 +144,41 @@
}
/**
+ * Test when download does not complete in time.
+ */
+ public function testOmnigroupmemberLoadIncompleteUseSuffix() {
+ civicrm_api3('Setting', 'create', array(
+ 'omnimail_omnigroupmembers_load' => array(
+ 'Silverpop' . '_woot' => array('last_timestamp' => '1487890800'),
+ ),
+ ));
+ $responses = array(
+ file_get_contents(__DIR__ . '/Responses/ExportListResponse.txt'),
+ );
+ for ($i = 0; $i < 15; $i++) {
+ $responses[] = file_get_contents(__DIR__ .
'/Responses/JobStatusWaitingResponse.txt');
+ }
+ civicrm_api3('setting', 'create', array('omnimail_job_retry_interval' =>
0.01));
+ $group = civicrm_api3('Group', 'create', array('name' => 'Omnimailers2',
'title' => 'Omni2'));
+
+ civicrm_api3('Omnigroupmember', 'load', array('mail_provider' =>
'Silverpop', 'username' => 'Donald', 'password' => 'Duck', 'client' =>
$this->getMockRequest($responses), 'group_identifier' => 123, 'group_id' =>
$group['id']));
+
+ $groupMembers = civicrm_api3('GroupContact', 'get', array('group_id' =>
$group['id']));
+ $this->assertEquals(0, $groupMembers['count']);
+
+ $this->assertEquals(array(
+ 'last_timestamp' => '1487890800',
+ 'retrieval_parameters' => array(
+ 'jobId' => '101719657',
+ 'filePath' => '/download/20170509_noCID - All - Jul 5 2017 06-27-45
AM.csv',
+ ),
+ 'progress_end_date' => '1488495600',
+ 'offset' => 0,
+ ), $this->getJobSettings(array('mail_provider' => 'Silverpop',
'job_suffix' => '_woot')));
+ $this->cleanupGroup($group);
+ }
+
+ /**
* After completing an incomplete download the end date should be the
progress end date.
*/
public function testCompleteIncomplete() {
@@ -205,10 +240,12 @@
/**
* Get job settings.
*
+ * @param array $params
+ *
* @return array
*/
- public function getJobSettings() {
- $omnimail = new CRM_Omnimail_Omnigroupmembers(array('mail_provider' =>
'Silverpop'));
+ public function getJobSettings($params = array('mail_provider' =>
'Silverpop')) {
+ $omnimail = new CRM_Omnimail_Omnigroupmembers($params);
return $omnimail->getJobSettings();
}
diff --git
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
index d637c67..7d94d41 100644
---
a/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
+++
b/sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit/OmnirecipientLoadTest.php
@@ -188,6 +188,32 @@
}
/**
+ * Test the suffix works for multiple jobs..
+ */
+ public function testCompleteIncompleteUseSuffix() {
+ $client = $this->setupSuccessfulDownloadClient();
+ civicrm_api3('setting', 'create', array(
+ 'omnimail_omnirecipient_load' => array(
+ 'Silverpop_woot' => array(
+ 'last_timestamp' => '1487890800',
+ 'retrieval_parameters' => array(
+ 'jobId' => '101569750',
+ 'filePath' => 'Raw Recipient Data Export Jul 03 2017 00-47-42 AM
1295.zip',
+ ),
+ 'progress_end_date' => '1488495600',
+ ),
+ ),
+ ));
+
+ civicrm_api3('Omnirecipient', 'load', array('mail_provider' =>
'Silverpop', 'username' => 'Donald', 'password' => 'Duck', 'client' => $client,
'job_suffix' => '_woot'));
+ $this->assertEquals(4, CRM_Core_DAO::singleValueQuery('SELECT COUNT(*)
FROM civicrm_mailing_provider_data'));
+ $this->assertEquals(array(
+ 'last_timestamp' => '1488495600',
+ ), $this->getJobSettings(array('mail_provider' => 'Silverpop',
'job_suffix' => '_woot')));
+ }
+
+
+ /**
* An exception should be thrown if the download is incomplete & we pass in
a timestamp.
*
* This is because the incomplete download will continue, and we will
incorrectly
@@ -226,10 +252,12 @@
/**
* Get job settings.
*
+ * @param array $params
+ *
* @return array
*/
- public function getJobSettings() {
- $omnimail = new CRM_Omnimail_Omnirecipients(array('mail_provider' =>
'Silverpop'));
+ public function getJobSettings($params = array('mail_provider' =>
'Silverpop')) {
+ $omnimail = new CRM_Omnimail_Omnirecipients($params);
return $omnimail->getJobSettings();
}
}
--
To view, visit https://gerrit.wikimedia.org/r/379174
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If0a709b9e9cec7e3276d231e75cca49d2909d4cb
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/crm
Gerrit-Branch: master
Gerrit-Owner: Eileen <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
