Adamw has submitted this change and it was merged.
Change subject: JP Morgan upload and more generalization
......................................................................
JP Morgan upload and more generalization
Wrote tests for normalization and import. All checks upload parsers rewritten.
Improved error reporting to log the number of records imported before dirty
exit, and the offending row number.
Deployment: drush updatedb
TODO: rollback all imports on error
Change-Id: Iab81ddb4e699f253c82c72eafd389794c97daec3
---
M sites/all/modules/offline2civicrm/AzlChecksFile.php
M sites/all/modules/offline2civicrm/ChecksFile.php
A sites/all/modules/offline2civicrm/EmptyRowException.php
A sites/all/modules/offline2civicrm/JpMorganFile.php
M sites/all/modules/offline2civicrm/PayPalChecksFile.php
M sites/all/modules/offline2civicrm/offline2civicrm.info
M sites/all/modules/offline2civicrm/offline2civicrm.module
A sites/all/modules/offline2civicrm/tests/AzlChecksFileTest.php
A sites/all/modules/offline2civicrm/tests/JpMorganFileTest.php
A sites/all/modules/offline2civicrm/tests/PayPalChecksFileTest.php
A sites/all/modules/offline2civicrm/tests/data/jpmorgan.csv
A sites/all/modules/offline2civicrm/tests/includes/AzlChecksFileProbe.php
A sites/all/modules/offline2civicrm/tests/includes/JpMorganFileProbe.php
A sites/all/modules/offline2civicrm/tests/includes/PayPalChecksFileProbe.php
M sites/all/modules/wmf_civicrm/wmf_civicrm.install
15 files changed, 642 insertions(+), 182 deletions(-)
Approvals:
Adamw: Verified; Looks good to me, approved
Mwalker: Looks good to me, approved
diff --git a/sites/all/modules/offline2civicrm/AzlChecksFile.php
b/sites/all/modules/offline2civicrm/AzlChecksFile.php
index 4fbad6e..ae93026 100644
--- a/sites/all/modules/offline2civicrm/AzlChecksFile.php
+++ b/sites/all/modules/offline2civicrm/AzlChecksFile.php
@@ -1,42 +1,38 @@
<?php
class AzlChecksFile extends ChecksFile {
- protected $required_fields = array(
- 'check_number',
- 'date',
- 'gift_source',
- 'gross',
- 'import_batch_number',
- 'payment_method',
- 'restrictions',
- );
-
- protected $required_columns = array(
- 'Batch',
- 'Check Number',
- 'City',
- 'Contribution Type',
- 'Country',
- 'Direct Mail Appeal',
- 'Email',
- 'Gift Source',
- 'Payment Instrument',
- 'Postal Code',
- 'Postmark Date',
- 'Received Date',
- 'Restrictions',
- 'Source',
- 'State',
- 'Street Address',
- 'Thank You Letter Date',
- 'Total Amount',
- );
-
function getRequiredColumns() {
- return $this->required_columns;
+ return array(
+ 'Batch',
+ 'Check Number',
+ 'City',
+ 'Contribution Type',
+ 'Country',
+ 'Direct Mail Appeal',
+ 'Email',
+ 'Gift Source',
+ 'Payment Instrument',
+ 'Postal Code',
+ 'Postmark Date',
+ 'Received Date',
+ 'Restrictions',
+ 'Source',
+ 'State',
+ 'Street Address',
+ 'Thank You Letter Date',
+ 'Total Amount',
+ );
}
function getRequiredFields() {
- return $this->required_fields;
+ return array(
+ 'check_number',
+ 'date',
+ 'gift_source',
+ 'gross',
+ 'import_batch_number',
+ 'payment_method',
+ 'restrictions',
+ );
}
}
diff --git a/sites/all/modules/offline2civicrm/ChecksFile.php
b/sites/all/modules/offline2civicrm/ChecksFile.php
index 7798a32..8920c85 100644
--- a/sites/all/modules/offline2civicrm/ChecksFile.php
+++ b/sites/all/modules/offline2civicrm/ChecksFile.php
@@ -37,126 +37,28 @@
$num_successful = 0;
$num_duplicates = 0;
- $row_index = 0;
+ $this->row_index = -1;
while( ( $row = fgetcsv( $file, 0, ',', '"', '\\')) !== FALSE) {
- list($currency, $source_amount) = explode( " ", _get_value(
"Source", $row, $headers ) );
- $total_amount = floatval( trim( _get_value( "Total Amount", $row,
$headers ), '$' ) );
+ $this->row_index++;
- if ( abs( $source_amount - $total_amount ) > .01 ) {
- $pretty_msg = json_encode( array_combine( array_keys( $headers
), $row ) );
- watchdog( 'offline2civicrm', "Amount mismatch in row: " .
$pretty_msg, NULL, WATCHDOG_ERROR );
- throw new WmfException( 'INVALID_MESSAGE', "Amount mismatch
during checks import" );
+ // Zip headers and row into a dict
+ $data = array_combine( array_keys( $headers ), array_slice( $row,
0, count( $headers ) ) );
+
+ // Strip whitespaces
+ foreach ( $data as $key => &$value ) {
+ $value = trim( $value );
}
- $msg = array(
- "optout" => "1",
- "anonymous" => "0",
- "letter_code" => _get_value( "Letter Code", $row, $headers ),
- "contact_source" => "check",
- "language" => "en",
- "street_address" => _get_value( "Street Address", $row,
$headers ),
- "supplemental_address_1" => _get_value( "Additional Address
1", $row, $headers ),
- "city" => _get_value( "City", $row, $headers ),
- "state_province" => _get_value( "State", $row, $headers ),
- "postal_code" => _get_value( "Postal Code", $row, $headers ),
- "payment_method" => _get_value( "Payment Instrument", $row,
$headers ),
- "payment_submethod" => "",
- "check_number" => _get_value( "Check Number", $row, $headers ),
- "currency" => $currency,
- "original_currency" => $currency,
- "original_gross" => $total_amount,
- "fee" => "0",
- "gross" => $total_amount,
- "net" => $total_amount,
- "date" => strtotime( _get_value( "Received Date", $row,
$headers ) ),
- "no_thank_you" => _get_value( "No Thank You", $row, $headers ),
- "thankyou_date" => strtotime( _get_value( "Thank You Letter
Date", $row, $headers ) ),
- "postmark_date" => strtotime( _get_value( "Postmark Date",
$row, $headers ) ),
- "restrictions" => _get_value( "Restrictions", $row, $headers ),
- "gift_source" => _get_value( "Gift Source", $row, $headers ),
- "direct_mail_appeal" => _get_value( "Direct Mail Appeal",
$row, $headers ),
- "import_batch_number" => _get_value( "Batch", $row, $headers ),
- );
-
- $contype = _get_value( 'Contribution Type', $row, $headers );
- switch ( $contype ) {
- case "Merkle":
- $msg['gateway'] = "merkle";
- break;
-
- case "Arizona Lockbox":
- $msg['gateway'] = "arizonalockbox";
- break;
-
- case "Cash":
- $msg['contribution_type'] = "cash";
- break;
-
- default:
- throw new WmfException( 'INVALID_MESSAGE', "Contribution
Type '$contype' is unknown whilst importing checks!" );
+ try {
+ $msg = $this->parseRow( $data );
+ } catch ( EmptyRowException $ex ) {
+ continue;
+ } catch ( WmfException $ex ) {
+ $rowNum = $this->row_index + 2;
+ $errorMsg = "Import aborted due to error at row {$rowNum}:
{$ex->getMessage()}, after {$num_successful} records were stored successfully
and {$num_duplicates} duplicates encountered.";
+ throw new Exception($errorMsg);
}
-
- // Attempt to get the organization name if it exists...
- // Merkle used the "Organization Name" column header where AZL
uses "Company"
- $orgname = _get_value( 'Organization Name', $row, $headers, FALSE
);
- if ( $orgname === FALSE ) {
- $orgname = _get_value( 'Company', $row, $headers, FALSE );
- }
-
- if( $orgname === FALSE ) {
- // If it's still false let's just assume it's an individual
- $msg['contact_type'] = "Individual";
- $msg["first_name"] = _get_value( "First Name", $row, $headers
);
- $msg["middle_name"] = _get_value( "Middle Name", $row,
$headers );
- $msg["last_name"] = _get_value( "Last Name", $row, $headers );
- } else {
- $msg['contact_type'] = "Organization";
- $msg['organization_name'] = $orgname;
- }
-
- // check for additional address information
- if( _get_value( 'Additional Address 2', $row, $headers ) != ''){
- $msg['supplemental_address_2'] .= ' ' . _get_value(
'Additional Address 2', $row, $headers );
- }
-
- // An email address is one of the crucial fields we need
- if( _get_value( 'Email', $row, $headers ) == ''){
- // set to the default, no TY will be sent
- $msg['email'] = "nob...@wikimedia.org";
- } else {
- $msg['email'] = _get_value( 'Email', $row, $headers );
- }
-
- // CiviCRM gets all weird when there is no country set
- // Making the assumption that none = US
- if( _get_value( 'Country', $row, $headers ) == ''){
- $msg['country'] = "US";
- } else {
- $msg['country'] = _get_value( 'Country', $row, $headers );
- }
-
- if ( $msg['country'] === "US" ) {
- // left-pad the zipcode
- if ( preg_match( '/^(\d{1,4})(-\d+)?$/', $msg['postal_code'],
$matches ) ) {
- $msg['postal_code'] = str_pad( $matches[1], 5, "0",
STR_PAD_LEFT );
- if ( !empty( $matches[2] ) ) {
- $msg['postal_code'] .= $matches[2];
- }
- }
- }
-
- // Generating a transaction id so that we don't import the same
rows multiple times
- $name_salt = $msg['contact_type'] == "Individual" ?
$msg["first_name"] . $msg["last_name"] : $msg["organization_name"];
- if ( !empty( $msg['check_number'] ) ) {
- $msg['gateway_txn_id'] = md5( $msg['check_number'] .
$name_salt );
- } else {
- $msg['gateway_txn_id'] = md5( $msg['date'] . $name_salt .
$row_index );
- }
-
- $row_index++;
-
- $this->mungeMessage( $msg );
// check to see if we have already processed this check
if ( $existing = wmf_civicrm_get_contributions_from_gateway_id(
$msg['gateway'], $msg['gateway_txn_id'] ) ){
@@ -166,16 +68,7 @@
continue;
}
- $failed = array();
- foreach ( $this->getRequiredFields() as $key ) {
- if ( !array_key_exists( $key, $msg ) or empty( $msg[$key] ) ) {
- $failed[] = $key;
- }
- }
- if ( $failed ) {
- throw new WmfException( 'CIVI_REQ_FIELD', t( "Missing required
fields @keys during check import", array( "@keys" => implode( ", ", $failed ) )
) );
- }
-
+ // tha business.
$contribution = wmf_civicrm_contribution_message_import( $msg );
watchdog( 'offline2civicrm',
@@ -192,8 +85,195 @@
watchdog( 'offline2civicrm', $message, array(), WATCHDOG_INFO );
}
- protected function mungeMessage( &$msg ) { }
+ /**
+ * Read a row and transform into normalized queue message form
+ *
+ * @param array $row native format for this upload file, usually a dict
+ *
+ * @return array queue message format
+ */
+ protected function parseRow( $data ) {
+ $msg = array();
- abstract protected function getRequiredFields();
+ foreach ( $this->getFieldMapping() as $normal => $header ) {
+ if ( !empty( $data[$header] ) ) {
+ $msg[$normal] = $data[$header];
+ }
+ }
+
+ foreach ( $this->getDatetimeFields() as $field ) {
+ if ( !empty( $msg[$field] ) ) {
+ $msg[$field] = strtotime( $msg[$field] );
+ }
+ }
+
+ $this->setDefaults( $msg );
+
+ $this->mungeMessage( $msg );
+
+ $failed = array();
+ foreach ( $this->getRequiredFields() as $key ) {
+ if ( !array_key_exists( $key, $msg ) or empty( $msg[$key] ) ) {
+ $failed[] = $key;
+ }
+ }
+ if ( $failed ) {
+ throw new WmfException( 'CIVI_REQ_FIELD', t( "Missing required
fields @keys during check import", array( "@keys" => implode( ", ", $failed ) )
) );
+ }
+
+ return $msg;
+ }
+
+ protected function setDefaults( &$msg ) {
+ foreach ( $this->getDefaultValues() as $key => $defaultValue ) {
+ if ( empty( $msg[$key] ) ) {
+ $msg[$key] = $defaultValue;
+ }
+ }
+ }
+
+ /**
+ * Do any final transformation on a normalized and default-laden queue
+ * message. This is very specific to each upload source.
+ */
+ protected function mungeMessage( &$msg ) {
+ $contype = $msg['raw_contribution_type'];
+ switch ( $contype ) {
+ case "Merkle":
+ $msg['gateway'] = "merkle";
+ break;
+
+ case "Arizona Lockbox":
+ $msg['gateway'] = "arizonalockbox";
+ break;
+
+ case "Cash":
+ $msg['contribution_type'] = "cash";
+ break;
+
+ default:
+ throw new WmfException( 'INVALID_MESSAGE', "Contribution Type
'$contype' is unknown whilst importing checks!" );
+ }
+
+ if ( !empty( $msg['organization_name'] ) ) {
+ $msg['contact_type'] = "Organization";
+ }
+
+ // Check that the message amounts match
+ list($currency, $source_amount) = explode( ' ',
$msg['contribution_source'] );
+ $msg['gross'] = floatval( trim( $msg['gross'], '$' ) );
+
+ if ( abs( $source_amount - $msg['gross'] ) > .01 ) {
+ $pretty_msg = json_encode( $msg );
+ watchdog( 'offline2civicrm', "Amount mismatch in row: " .
$pretty_msg, NULL, WATCHDOG_ERROR );
+ throw new WmfException( 'INVALID_MESSAGE', "Amount mismatch during
checks import" );
+ }
+
+ $msg = array_merge( $msg, array(
+ 'currency' => $currency,
+ 'original_currency' => $currency,
+ 'original_gross' => $msg['gross'],
+ ) );
+
+ // left-pad the zipcode
+ if ( $msg['country'] === 'US' ) {
+ if ( preg_match( '/^(\d{1,4})(-\d+)?$/', $msg['postal_code'],
$matches ) ) {
+ $msg['postal_code'] = str_pad( $matches[1], 5, "0",
STR_PAD_LEFT );
+ if ( !empty( $matches[2] ) ) {
+ $msg['postal_code'] .= $matches[2];
+ }
+ }
+ }
+
+ // Generate a transaction ID so that we don't import the same rows
multiple times
+ if ( empty( $msg['gateway_txn_id'] ) ) {
+ if ( $msg['contact_type'] === 'Individual' ) {
+ $name_salt = $msg['first_name'] . $msg['last_name'];
+ } else {
+ $name_salt = $msg['organization_name'];
+ }
+
+ if ( !empty( $msg['check_number'] ) ) {
+ $msg['gateway_txn_id'] = md5( $msg['check_number'] .
$name_salt );
+ } else {
+ $msg['gateway_txn_id'] = md5( $msg['date'] . $name_salt .
$this->row_index );
+ }
+ }
+ }
+
+ protected function getDefaultValues() {
+ return array(
+ 'contact_source' => 'check',
+ 'contact_type' => 'Individual',
+ 'country' => 'US',
+ 'email' => 'nob...@wikimedia.org',
+ 'gift_source' => 'Community Gift',
+ 'restrictions' => 'Unrestricted - General',
+ );
+ }
+
+ /**
+ * Return column mappings
+ *
+ * @return array of {normalized field name} => {spreadsheet column title}
+ */
+ protected function getFieldMapping() {
+ return array(
+ 'check_number' => 'Check Number',
+ 'city' => 'City',
+ 'contribution_source' => 'Source',
+ 'country' => 'Country',
+ 'date' => 'Received Date',
+ 'direct_mail_appeal' => 'Direct Mail Appeal',
+ 'email' => 'Email',
+ 'first_name' => 'First Name',
+ 'gift_source' => 'Gift Source',
+ 'gross' => 'Total Amount',
+ 'import_batch_number' => 'Batch',
+ 'last_name' => 'Last Name',
+ 'letter_code' => 'Letter Code',
+ 'middle_name' => 'Middle Name',
+ 'no_thank_you' => 'No Thank You',
+ 'organization_name' => 'Organization Name',
+ 'payment_method' => 'Payment Instrument',
+ 'postal_code' => 'Postal Code',
+ 'postmark_date' => 'Postmark Date',
+ 'raw_contribution_type' => 'Contribution Type',
+ 'restrictions' => 'Restrictions',
+ 'state_province' => 'State',
+ 'street_address' => 'Street Address',
+ 'supplemental_address_1' => 'Additional Address 1',
+ 'supplemental_address_2' => 'Additional Address 2',
+ 'thankyou_date' => 'Thank You Letter Date',
+ );
+ }
+
+ /**
+ * Date fields which must be converted to unix timestamps
+ *
+ * @return array of field names
+ */
+ protected function getDatetimeFields() {
+ return array(
+ 'date',
+ 'thankyou_date',
+ 'postmark_date',
+ );
+ }
+
+ /**
+ * Columns which must exist in the spreadsheet
+ *
+ * This is just a "schema" check. We don't require that the fields
contain data.
+ *
+ * @return array of column header titles
+ */
abstract protected function getRequiredColumns();
+
+ /**
+ * Fields that must not be empty in the normalized message
+ *
+ * @return array of normalized message field names
+ */
+ abstract protected function getRequiredFields();
}
diff --git a/sites/all/modules/offline2civicrm/EmptyRowException.php
b/sites/all/modules/offline2civicrm/EmptyRowException.php
new file mode 100644
index 0000000..420eff7
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/EmptyRowException.php
@@ -0,0 +1,4 @@
+<?php
+
+class EmptyRowException extends Exception {
+}
diff --git a/sites/all/modules/offline2civicrm/JpMorganFile.php
b/sites/all/modules/offline2civicrm/JpMorganFile.php
new file mode 100644
index 0000000..338ff09
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/JpMorganFile.php
@@ -0,0 +1,78 @@
+<?php
+
+class JpMorganFile extends ChecksFile {
+ protected function getRequiredColumns() {
+ return array(
+ 'ACCOUNT NAME',
+ 'CURRENCY',
+ 'REFERENCE',
+ 'Bank Ref Number',
+ 'TRANSACTION DATE',
+ 'TRANSACTION TYPE',
+ 'VALUE DATE',
+ 'CREDITS',
+ );
+ }
+
+ protected function getRequiredFields() {
+ return array(
+ 'date',
+ 'gateway_txn_id',
+ 'gross',
+ 'original_currency',
+ 'original_gross',
+ );
+ }
+
+ protected function getFieldMapping() {
+ return array(
+ 'gateway_account' => 'ACCOUNT NAME',
+ 'original_currency' => 'CURRENCY',
+ 'gateway_txn_id' => 'Bank Ref Number',
+ 'date' => 'TRANSACTION DATE',
+ 'settlement_date' => 'VALUE DATE',
+ 'original_gross' => 'CREDITS',
+ );
+ }
+
+ protected function getDatetimeFields() {
+ return array(
+ 'date',
+ 'settlement_date',
+ );
+ }
+
+ protected function getDefaultValues() {
+ return array(
+ 'contact_type' => 'Individual',
+ 'direct_mail_appeal' => 'White Mail',
+ 'email' => 'nob...@wikimedia.org',
+ 'gateway' => 'jpmorgan',
+ 'gift_source' => 'Community Gift',
+ 'no_thank_you' => 'No Contact Details',
+ 'payment_instrument' => 'JP Morgan EUR',
+ 'restrictions' => 'Unrestricted - General',
+ );
+ }
+
+ protected function parseRow( $data ) {
+ // Empty rows are acceptable for this file
+ if ( empty( $data['ACCOUNT NAME'] ) and empty( $data['REFERENCE'] ) ) {
+ throw new EmptyRowException();
+ }
+
+ return parent::parseRow( $data );
+ }
+
+ protected function mungeMessage( &$msg ) {
+ // Approximate value in USD
+ $msg['gross'] = exchange_rate_convert(
+ $msg['original_currency'], $msg['original_gross'],
$msg['settlement_date']
+ );
+
+ // Flag as big-time if over $1000
+ if ( $msg['gross'] > 1000 ) {
+ $msg['gift_source'] = 'Benefactor Gift';
+ }
+ }
+}
diff --git a/sites/all/modules/offline2civicrm/PayPalChecksFile.php
b/sites/all/modules/offline2civicrm/PayPalChecksFile.php
index 3eb69cd..45fbfc2 100644
--- a/sites/all/modules/offline2civicrm/PayPalChecksFile.php
+++ b/sites/all/modules/offline2civicrm/PayPalChecksFile.php
@@ -1,34 +1,31 @@
<?php
class PayPalChecksFile extends ChecksFile {
- protected $required_fields = array(
- 'date',
- 'gift_source',
- 'gross',
- 'payment_method',
- 'restrictions',
- );
-
- protected $required_columns = array(
- 'Contribution Type',
- 'Received Date',
- 'Direct Mail Appeal',
- 'First Name',
- 'Gift Source',
- 'Last Name',
- 'No Thank You',
- 'Payment Instrument',
- 'Restrictions',
- 'Source',
- 'Total Amount',
- );
protected function getRequiredColumns() {
- return $this->required_columns;
+ return array(
+ 'Contribution Type',
+ 'Received Date',
+ 'Direct Mail Appeal',
+ 'First Name',
+ 'Gift Source',
+ 'Last Name',
+ 'No Thank You',
+ 'Payment Instrument',
+ 'Restrictions',
+ 'Source',
+ 'Total Amount',
+ );
}
protected function getRequiredFields() {
- return $this->required_fields;
+ return array(
+ 'date',
+ 'gift_source',
+ 'gross',
+ 'payment_method',
+ 'restrictions',
+ );
}
protected function mungeMessage( &$msg ) {
diff --git a/sites/all/modules/offline2civicrm/offline2civicrm.info
b/sites/all/modules/offline2civicrm/offline2civicrm.info
index 88b67dc..95753a7 100644
--- a/sites/all/modules/offline2civicrm/offline2civicrm.info
+++ b/sites/all/modules/offline2civicrm/offline2civicrm.info
@@ -2,6 +2,7 @@
description = Imports offline contributions from a CSV
package = offline2civicrm
core = 7.x
+dependencies[] = exchange_rates
dependencies[] = queue2civicrm
dependencies[] = wmf_civicrm
dependencies[] = wmf_communication
@@ -12,5 +13,7 @@
files[] = ChecksImportLog.php
files[] = ContributionConversion.php
files[] = CsvBatchFile.php
+files[] = EmptyRowException.php
+files[] = JpMorganFile.php
files[] = PayPalChecksFile.php
files[] = tests/ContributionConversion.test
diff --git a/sites/all/modules/offline2civicrm/offline2civicrm.module
b/sites/all/modules/offline2civicrm/offline2civicrm.module
index 60f9798..b86992c 100644
--- a/sites/all/modules/offline2civicrm/offline2civicrm.module
+++ b/sites/all/modules/offline2civicrm/offline2civicrm.module
@@ -98,6 +98,7 @@
'#options' => array(
'azl' => t( 'AZ Lockbox' ),
'paypal' => t( 'EFT donations' ),
+ 'jpmorgan' => t( 'JP Morgan' ),
),
);
$form['log'] = array(
@@ -131,6 +132,9 @@
case 'paypal':
$importer = new PayPalChecksFile( $file->uri );
break;
+ case 'jpmorgan':
+ $importer = new JpMorganFile( $file->uri );
+ break;
default:
throw new Exception( 'Bad file format selection' );
}
diff --git a/sites/all/modules/offline2civicrm/tests/AzlChecksFileTest.php
b/sites/all/modules/offline2civicrm/tests/AzlChecksFileTest.php
new file mode 100644
index 0000000..fb6a650
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/tests/AzlChecksFileTest.php
@@ -0,0 +1,133 @@
+<?php
+
+class AzlChecksFileTest extends BaseWmfDrupalPhpUnitTestCase {
+ function setUp() {
+ parent::setUp();
+
+ require_once __DIR__ . "/includes/AzlChecksFileProbe.php";
+ }
+
+ function testParseRow_Individual() {
+ $data = array(
+ 'Batch' => '1234',
+ 'Contribution Type' => 'Arizona Lockbox',
+ 'Total Amount' => '50',
+ 'Source' => 'USD 50.00',
+ 'Postmark Date' => '',
+ 'Received Date' => '4/1/14',
+ 'Payment Instrument' => 'Check',
+ 'Check Number' => '2020',
+ 'Restrictions' => 'Unrestricted - General',
+ 'Gift Source' => 'Community Gift',
+ 'Direct Mail Appeal' => 'White Mail',
+ 'Prefix' => 'Mrs.',
+ 'First Name' => 'Sub',
+ 'Last Name' => 'Tell',
+ 'Suffix' => '',
+ 'Street Address' => '1000 Markdown Markov',
+ 'Additional Address 1' => '',
+ 'Additional Address 2' => '',
+ 'City' => 'Best St. Louis',
+ 'State' => 'MA',
+ 'Postal Code' => '2468',
+ 'Country' => '',
+ 'Phone' => '(123) 456-0000',
+ 'Email' => '',
+ 'Thank You Letter Date' => '5/1/14',
+ 'AC Flag' => 'Y',
+ );
+ $expected_normal = array(
+ 'check_number' => '2020',
+ 'city' => 'Best St. Louis',
+ 'contact_source' => 'check',
+ 'contact_type' => 'Individual',
+ 'contribution_source' => 'USD 50.00',
+ 'country' => 'US',
+ 'currency' => 'USD',
+ 'date' => 1396335600,
+ 'direct_mail_appeal' => 'White Mail',
+ 'email' => 'nob...@wikimedia.org',
+ 'first_name' => 'Sub',
+ 'gateway' => 'arizonalockbox',
+ 'gateway_txn_id' => 'e59ed825ea04516fb2abf1c130d47525',
+ 'gift_source' => 'Community Gift',
+ 'gross' => '50.00',
+ 'import_batch_number' => '1234',
+ 'last_name' => 'Tell',
+ 'original_currency' => 'USD',
+ 'original_gross' => '50.00',
+ 'payment_method' => 'Check',
+ 'postal_code' => '02468',
+ 'raw_contribution_type' => 'Arizona Lockbox',
+ 'restrictions' => 'Unrestricted - General',
+ 'state_province' => 'MA',
+ 'street_address' => '1000 Markdown Markov',
+ 'thankyou_date' => 1398927600,
+ );
+
+ $importer = new AzlChecksFileProbe( "null URI" );
+ $output = $importer->_parseRow( $data );
+
+ $this->assertEquals( $expected_normal, $output );
+ }
+
+ function testParseRow_Organization() {
+ $data = array(
+ 'Batch' => '1235',
+ 'Contribution Type' => 'Arizona Lockbox',
+ 'Total Amount' => '51',
+ 'Source' => 'USD 51.00',
+ 'Postmark Date' => '',
+ 'Received Date' => '4/1/14',
+ 'Payment Instrument' => 'Check',
+ 'Check Number' => '202000001',
+ 'Restrictions' => 'Restricted-Foundation',
+ 'Gift Source' => 'Foundation Gift',
+ 'Direct Mail Appeal' => 'White Mail',
+ 'Organization Name' => 'One Pacific Entitlement',
+ 'Street Address' => '1000 Markdown Markov',
+ 'Additional Address 1' => '',
+ 'Additional Address 2' => '',
+ 'City' => 'Best St. Louis',
+ 'State' => 'MA',
+ 'Postal Code' => '123-LAX',
+ 'Country' => 'FR',
+ 'Phone' => '+357 (123) 456-0000',
+ 'Email' => '',
+ 'Thank You Letter Date' => '5/1/14',
+ 'AC Flag' => '',
+ );
+ $expected_normal = array(
+ 'check_number' => '202000001',
+ 'city' => 'Best St. Louis',
+ 'contact_source' => 'check',
+ 'contact_type' => 'Organization',
+ 'contribution_source' => 'USD 51.00',
+ 'country' => 'FR',
+ 'currency' => 'USD',
+ 'date' => 1396335600,
+ 'direct_mail_appeal' => 'White Mail',
+ 'email' => 'nob...@wikimedia.org',
+ 'gateway' => 'arizonalockbox',
+ 'gateway_txn_id' => '6dbb8d844c7509076e2a275fb76d0130',
+ 'gift_source' => 'Foundation Gift',
+ 'gross' => 51.00,
+ 'import_batch_number' => '1235',
+ 'organization_name' => 'One Pacific Entitlement',
+ 'original_currency' => 'USD',
+ 'original_gross' => 51.00,
+ 'payment_method' => 'Check',
+ 'postal_code' => '123-LAX',
+ 'raw_contribution_type' => 'Arizona Lockbox',
+ 'restrictions' => 'Restricted-Foundation',
+ 'state_province' => 'MA',
+ 'street_address' => '1000 Markdown Markov',
+ 'thankyou_date' => 1398927600,
+ );
+
+ $importer = new AzlChecksFileProbe( "null URI" );
+ $output = $importer->_parseRow( $data );
+
+ $this->assertEquals( $expected_normal, $output );
+ }
+}
diff --git a/sites/all/modules/offline2civicrm/tests/JpMorganFileTest.php
b/sites/all/modules/offline2civicrm/tests/JpMorganFileTest.php
new file mode 100644
index 0000000..23c032f
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/tests/JpMorganFileTest.php
@@ -0,0 +1,69 @@
+<?php
+
+class JpMorganFileTest extends BaseWmfDrupalPhpUnitTestCase {
+ function setUp() {
+ parent::setUp();
+
+ require_once __DIR__ . "/includes/JpMorganFileProbe.php";
+ }
+
+ function testParseRow() {
+ $data = array(
+ 'ACCOUNT NAME' => 'Testes EUR_Public',
+ 'CURRENCY' => 'EUR',
+ 'REFERENCE' => 'UNAVAILABLE',
+ 'Bank Ref Number' => '1234TEST',
+ 'TRANSACTION DATE' => '04/01/2000',
+ 'TRANSACTION TYPE' => 'FOO CREDIT RECEIVED',
+ 'VALUE DATE' => '04/02/2000',
+ 'CREDITS' => '5.50',
+ );
+ $expected_normal = array(
+ 'contact_type' => 'Individual',
+ 'date' => 954576000,
+ 'direct_mail_appeal' => 'White Mail',
+ 'email' => 'nob...@wikimedia.org',
+ 'gateway_account' => 'Testes EUR_Public',
+ 'gateway' => 'jpmorgan',
+ 'gateway_txn_id' => '1234TEST',
+ 'gift_source' => 'Community Gift',
+ //'gross' => 7.1874
+ 'no_thank_you' => 'No Contact Details',
+ 'original_currency' => 'EUR',
+ 'original_gross' => '5.50',
+ 'payment_instrument' => 'JP Morgan EUR',
+ 'restrictions' => 'Unrestricted - General',
+ 'settlement_date' => 954662400,
+ );
+
+ $importer = new JpMorganFileProbe( "no URI" );
+ $output = $importer->_parseRow( $data );
+
+ // FIXME: exchange rate conversion cannot be mocked yet, so just make
sure it is present
+ $this->assertTrue( $output['gross'] > 0 );
+ unset( $output['gross'] );
+
+ $this->assertEquals( $expected_normal, $output );
+ }
+
+ function testImport() {
+ global $user;
+ //FIXME: move to BaseWmfDrupalPhpUnitTestCase
+ $user = new stdClass();
+ $user->name = "foo_who";
+ $user->uid = "321";
+ $user->roles = array( DRUPAL_AUTHENTICATED_RID => 'authenticated user'
);
+
+ //FIXME
+ $_GET['q'] = '';
+ //FIXME
+ civicrm_initialize();
+
+ $importer = new JpMorganFileProbe( __DIR__ . "/data/jpmorgan.csv" );
+ $importer->import();
+
+ $contribution = wmf_civicrm_get_contributions_from_gateway_id(
'jpmorgan', '1234TEST' );
+ $this->assertEquals( 1, count( $contribution ) );
+ $this->assertEquals( $contribution[0]['trxn_id'], 'JPMORGAN 1234TEST
1399363947' );
+ }
+}
diff --git a/sites/all/modules/offline2civicrm/tests/PayPalChecksFileTest.php
b/sites/all/modules/offline2civicrm/tests/PayPalChecksFileTest.php
new file mode 100644
index 0000000..9459017
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/tests/PayPalChecksFileTest.php
@@ -0,0 +1,59 @@
+<?php
+
+class PayPalChecksFileTest extends BaseWmfDrupalPhpUnitTestCase {
+ function setUp() {
+ parent::setUp();
+
+ require_once __DIR__ . "/includes/PayPalChecksFileProbe.php";
+ }
+
+ function testParseRow() {
+ $data = array(
+ 'Contribution Type' => 'Cash',
+ 'Total Amount' => '$10.00',
+ 'Source' => 'USD 10.00',
+ 'Received Date' => '1/27/13',
+ 'Payment Instrument' => 'EFT',
+ 'Restrictions' => 'Unrestricted - General',
+ 'Gift Source' => 'Community Gift',
+ 'Direct Mail Appeal' => 'MissionFish (PayPal)',
+ 'Prefix' => '',
+ 'First Name' => 'Diz and',
+ 'Last Name' => 'Bird',
+ 'Suffix' => '',
+ 'Street Address' => '',
+ 'Additional Address 1' => '',
+ 'Additional Address 2' => '',
+ 'City' => '',
+ 'State' => '',
+ 'Postal Code' => '',
+ 'Country' => '',
+ 'Phone' => '',
+ 'Email' => '',
+ 'No Thank You' => 'no reas',
+ );
+ $expected_normal = array(
+ 'contact_source' => 'check',
+ 'contact_type' => 'Individual',
+ 'contribution_source' => 'USD 10.00',
+ 'country' => 'US',
+ 'date' => 1359273600,
+ 'direct_mail_appeal' => 'MissionFish (PayPal)',
+ 'email' => 'nob...@wikimedia.org',
+ 'first_name' => 'Diz and',
+ 'gateway' => 'paypal',
+ 'gift_source' => 'Community Gift',
+ 'gross' => '$10.00',
+ 'last_name' => 'Bird',
+ 'no_thank_you' => 'no reas',
+ 'payment_method' => 'EFT',
+ 'raw_contribution_type' => 'Cash',
+ 'restrictions' => 'Unrestricted - General',
+ );
+
+ $importer = new PayPalChecksFileProbe( "no URI" );
+ $output = $importer->_parseRow( $data );
+
+ $this->assertEquals( $expected_normal, $output );
+ }
+}
diff --git a/sites/all/modules/offline2civicrm/tests/data/jpmorgan.csv
b/sites/all/modules/offline2civicrm/tests/data/jpmorgan.csv
new file mode 100644
index 0000000..266cfb2
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/tests/data/jpmorgan.csv
@@ -0,0 +1,2 @@
+ACCOUNT NAME,CURRENCY,REFERENCE,Bank Ref Number,TRANSACTION DATE,TRANSACTION
TYPE,VALUE DATE,CREDITS
+Testes EUR_Public,EUR,UNAVAILABLE,1234TEST,04/01/2000,FOO CREDIT
RECEIVED,04/02/2000,5.50
diff --git
a/sites/all/modules/offline2civicrm/tests/includes/AzlChecksFileProbe.php
b/sites/all/modules/offline2civicrm/tests/includes/AzlChecksFileProbe.php
new file mode 100644
index 0000000..c6b18c2
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/tests/includes/AzlChecksFileProbe.php
@@ -0,0 +1,7 @@
+<?php
+
+class AzlChecksFileProbe extends AzlChecksFile {
+ function _parseRow( $data ) {
+ return $this->parseRow( $data );
+ }
+}
diff --git
a/sites/all/modules/offline2civicrm/tests/includes/JpMorganFileProbe.php
b/sites/all/modules/offline2civicrm/tests/includes/JpMorganFileProbe.php
new file mode 100644
index 0000000..7dac60d
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/tests/includes/JpMorganFileProbe.php
@@ -0,0 +1,7 @@
+<?php
+
+class JpMorganFileProbe extends JpMorganFile {
+ function _parseRow( $data ) {
+ return $this->parseRow( $data );
+ }
+}
diff --git
a/sites/all/modules/offline2civicrm/tests/includes/PayPalChecksFileProbe.php
b/sites/all/modules/offline2civicrm/tests/includes/PayPalChecksFileProbe.php
new file mode 100644
index 0000000..c35352f
--- /dev/null
+++ b/sites/all/modules/offline2civicrm/tests/includes/PayPalChecksFileProbe.php
@@ -0,0 +1,7 @@
+<?php
+
+class PayPalChecksFileProbe extends PayPalChecksFile {
+ function _parseRow( $data ) {
+ return $this->parseRow( $data );
+ }
+}
diff --git a/sites/all/modules/wmf_civicrm/wmf_civicrm.install
b/sites/all/modules/wmf_civicrm/wmf_civicrm.install
index c60b123..c86472e 100644
--- a/sites/all/modules/wmf_civicrm/wmf_civicrm.install
+++ b/sites/all/modules/wmf_civicrm/wmf_civicrm.install
@@ -23,6 +23,7 @@
wmf_civicrm_update_7016();
wmf_civicrm_update_7017();
wmf_civicrm_update_7018();
+ wmf_civicrm_update_7019();
}
/**
@@ -554,6 +555,7 @@
*/
function wmf_civicrm_update_6012()
{
+ $ret = array();
$api = wmf_civicrm_bootstrap_civi();
$success = $api->CustomGroup->get(array(
@@ -802,3 +804,15 @@
return array();
}
+
+/**
+ * Add JP Morgan payment instrument
+ */
+function wmf_civicrm_update_7019()
+{
+ $payment_instruments = array(
+ 'JP Morgan EUR',
+ );
+
+ wmf_civicrm_create_option_values( 'payment_instrument',
$payment_instruments );
+}
--
To view, visit https://gerrit.wikimedia.org/r/131642
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iab81ddb4e699f253c82c72eafd389794c97daec3
Gerrit-PatchSet: 4
Gerrit-Project: wikimedia/fundraising/crm
Gerrit-Branch: master
Gerrit-Owner: Adamw <awi...@wikimedia.org>
Gerrit-Reviewer: Adamw <awi...@wikimedia.org>
Gerrit-Reviewer: Katie Horn <kh...@wikimedia.org>
Gerrit-Reviewer: Mwalker <mwal...@wikimedia.org>
Gerrit-Reviewer: Ssmith <ssm...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
