Jgleeson has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/391022 )
Change subject: Separated out Donation Stats to new DonationsStats.php helper class ...................................................................... Separated out Donation Stats to new DonationsStats.php helper class Change-Id: If7fdbbff797efe240d3af20017b99340b6b0f135 --- M composer.lock M sites/all/modules/queue2civicrm/DonationQueueConsumer.php A sites/all/modules/queue2civicrm/DonationStats.php M sites/all/modules/queue2civicrm/queue2civicrm.info M sites/all/modules/queue2civicrm/queue2civicrm.module 5 files changed, 173 insertions(+), 108 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm refs/changes/22/391022/1 diff --git a/composer.lock b/composer.lock index 5cfa165..815a644 100644 --- a/composer.lock +++ b/composer.lock @@ -693,12 +693,12 @@ "source": { "type": "git", "url": "https://github.com/jackgleeson/stats-collector.git", - "reference": "06e2ef5b2563e1a0f58b2f05fa48ebe874238431" + "reference": "2555f001961328a6e0da1b64bbfac97d47559d13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/jackgleeson/stats-collector/zipball/06e2ef5b2563e1a0f58b2f05fa48ebe874238431", - "reference": "06e2ef5b2563e1a0f58b2f05fa48ebe874238431", + "url": "https://api.github.com/repos/jackgleeson/stats-collector/zipball/2555f001961328a6e0da1b64bbfac97d47559d13", + "reference": "2555f001961328a6e0da1b64bbfac97d47559d13", "shasum": "" }, "require": { @@ -725,7 +725,7 @@ } ], "description": "Utility to help record, analyse and export statistics across any PHP process e.g. http request, batch jobs or cli-scrpts", - "time": "2017-11-10T18:27:43+00:00" + "time": "2017-11-13T15:06:33+00:00" }, { "name": "league/csv", diff --git a/sites/all/modules/queue2civicrm/DonationQueueConsumer.php b/sites/all/modules/queue2civicrm/DonationQueueConsumer.php index c4be75f..5a04edf 100644 --- a/sites/all/modules/queue2civicrm/DonationQueueConsumer.php +++ b/sites/all/modules/queue2civicrm/DonationQueueConsumer.php @@ -1,11 +1,9 @@ <?php namespace queue2civicrm; -use Queue2civicrmTrxnCounter; use SmashPig\Core\DataStores\PendingDatabase; -use SmashPig\Core\UtcDate; -use Statistics\Collector\Collector as StatsCollector; use wmf_common\TransactionalWmfQueueConsumer; use WmfException; +use DonationStats; class DonationQueueConsumer extends TransactionalWmfQueueConsumer { @@ -72,103 +70,10 @@ } // record donations stats - $this->recordDonationStats($message, $contribution); + $DonationStats = new DonationStats(); + $DonationStats->recordDonationStats($message, $contribution); // delete any pending db entries with matching gateway and order_id PendingDatabase::get()->deleteMessage($message); } - - /** - * Record donation stats including: - * - Number of donations by gateway - * - Average time between gateway transaction datetime and imported to civi (now) - * - Average time between donation message enqueued datetime and imported to civi (now) - * - * @param array $message - * @param array $contribution - */ - protected function recordDonationStats($message, $contribution) { - $paymentGateway = $message['gateway']; - - // set the root namespace for the stats to follow. - StatsCollector::getInstance()->setNamespace("donations"); - $this->recordGatewayDonation($paymentGateway); - $this->recordOverallDonations(); - $this->recordGatewayTransactionAge($paymentGateway, $contribution['receive_date']); - $this->recordOverallGatewayTransactionAge(); - if (isset($message['source_enqueued_time'])) { - $this->recordMessageEnqueuedAge($paymentGateway, $message['source_enqueued_time']); - $this->recordOverallMessageEnqueuedAge(); - } - } - - protected function recordGatewayDonation($paymentGateway) { - $statsCollector = StatsCollector::getInstance(); - // record a stat to record the number of gateway specific donations - $statsCollector->incrementStat("gateway.{$paymentGateway}", 1); - } - - protected function recordOverallDonations() { - $statsCollector = StatsCollector::getInstance(); - // set/update the current total sum of gateway donations - $statOptions = ['clobber' => TRUE]; - $statsCollector->addStat( - "overall.donations", - $statsCollector->getStatSum("gateway.*"), - $statOptions - ); - } - - protected function recordGatewayTransactionAge($paymentGateway, $transactionDate) { - $statsCollector = StatsCollector::getInstance(); - // work out time between gateway's official transaction time and now - $gatewayReceivedAge = UtcDate::getUtcTimestamp() - UtcDate::getUtcTimestamp($transactionDate); - - // record a stat for the gateway received age - $statsCollector->addStat("received_age.{$paymentGateway}", - $gatewayReceivedAge); - } - - protected function recordOverallGatewayTransactionAge() { - $statsCollector = StatsCollector::getInstance(); - - // set/update the overall current moving average for all payment gateway received ages - $statOptions = ['clobber' => TRUE]; - $statsCollector->addStat( - "overall.average.received_age", - $statsCollector->getStatAverage("received_age.*"), - $statOptions - ); - } - - protected function recordMessageEnqueuedAge($paymentGateway, $enqueuedTime) { - $statsCollector = StatsCollector::getInstance(); - // work out time between the message enqueued time and now if 'source_enqueued_time' is set - $enqueuedAge = UtcDate::getUtcTimestamp() - $enqueuedTime; - - // record a stat for the message enqueued age - $statsCollector->addStat("enqueued_age.{$paymentGateway}", $enqueuedAge); - - // set/update the current moving average of enqueued message ages - $statOptions = ['clobber' => TRUE]; - $statsCollector->addStat( - "average.enqueued_age.{$paymentGateway}", - $statsCollector->getStatAverage("enqueued_age.{$paymentGateway}"), - $statOptions - ); - - } - - protected function recordOverallMessageEnqueuedAge() { - $statsCollector = StatsCollector::getInstance(); - - // set/update the current moving average of enqueued message ages - $statOptions = ['clobber' => TRUE]; - $statsCollector->addStat( - "overall.average.enqueued_age", - $statsCollector->getStatAverage("enqueued_age.*"), - $statOptions - ); - } - } diff --git a/sites/all/modules/queue2civicrm/DonationStats.php b/sites/all/modules/queue2civicrm/DonationStats.php new file mode 100644 index 0000000..366ebbb --- /dev/null +++ b/sites/all/modules/queue2civicrm/DonationStats.php @@ -0,0 +1,159 @@ +<?php + +use Statistics\Collector\Collector as StatsCollector; +use Statistics\Exporter\Prometheus as PrometheusStatsExporter; +use SmashPig\Core\UtcDate; + +/** + * Class DonationStats + * + * Helper class to encapsulate donation stats recording & exporting. + * + */ +class DonationStats { + + /** + * @var \Statistics\Collector\iCollector + */ + protected $statsCollector; + + public function __construct() { + $this->statsCollector = StatsCollector::getInstance(); + + // set the root namespace for all donation related stats + $this->statsCollector->setNamespace("donations"); + } + + /** + * Record donation stats: + * 1) Number of donations by gateway + * 2) Number of overall donations + * 3) Time between gateway transaction time and civiCRM import time (now) + * 4) Gateway specific moving average of (3) + * 5) Overall moving average of (3) + * 6) Time between donation message enqueued time and civiCRM import time (now) + * 7) Gateway specific moving average of (5) + * 8) Overall moving average of (5) + * + * @param array $message + * @param array $contribution + */ + public function recordDonationStats($message, $contribution) { + $paymentGateway = $message['gateway']; + + // donation counter + $this->recordGatewayDonation($paymentGateway); + $this->recordOverallDonations(); + + // difference between gateway transaction age to civiCRM save time + $this->recordGatewayTransactionAge($paymentGateway, $contribution['receive_date']); + $this->recordGatewayAverageTransactionAge($paymentGateway); + $this->recordOverallGatewayTransactionAge(); + + // difference between message enqueued time to civiCRM save time + if (isset($message['source_enqueued_time'])) { + $this->recordMessageEnqueuedAge($paymentGateway, $message['source_enqueued_time']); + $this->recordGatewayAverageMessageEnqueuedAge($paymentGateway); + $this->recordOverallMessageEnqueuedAge(); + } + } + + public function getOverallAverageMessageAge() { + $overallAverageMessageAge = $this->statsCollector->getStat( + "overall.average.received_age" + ); + return $overallAverageMessageAge; + } + + /** + * Export overall & average stats data to Prometheus out files using the + * Statistics\Exporter\Prometheus exporter. + */ + public function exportDonationStatsToPrometheus() { + $prometheusExporter = new PrometheusStatsExporter('donation_stats', '/var/spool/prometheus'); + + // target the stats we want export. + // NOTE: we could pass the entire statsCollector object here to be export() but individual records which are + // averaged as part of the queue consumer process have no individual value so they are omitted + $statsToExport = [ + 'donations.average.*', + 'donations.overall.*', + ]; + $stats = $this->statsCollector->getStats($statsToExport, TRUE); + $prometheusExporter->export($stats); + } + + protected function recordGatewayDonation($paymentGateway) { + // record a stat to record the number of gateway donations + $this->statsCollector->incrementStat("gateway.{$paymentGateway}", 1); + } + + protected function recordOverallDonations() { + // set/update the current total sum of all gateway donations during this queue consumer run + $statOptions = ['clobber' => TRUE]; + $this->statsCollector->addStat( + "overall.donations", + $this->statsCollector->getStatSum("gateway.*"), + $statOptions + ); + } + + protected function recordGatewayTransactionAge($paymentGateway, $transactionDate) { + // work out time between gateway's official transaction time and now + $gatewayReceivedAge = UtcDate::getUtcTimestamp() - UtcDate::getUtcTimestamp($transactionDate); + + // record a stat for the gateway received age + $this->statsCollector->addStat("received_age.{$paymentGateway}", + $gatewayReceivedAge); + } + + protected function recordGatewayAverageTransactionAge($paymentGateway) { + // set/update the current moving average of transaction age + $statOptions = ['clobber' => TRUE]; + $this->statsCollector->addStat( + "average.received_age.{$paymentGateway}", + $this->statsCollector->getStatAverage("received_age.{$paymentGateway}"), + $statOptions + ); + } + + protected function recordOverallGatewayTransactionAge() { + // set/update the overall current moving average for all payment gateway received ages + $statOptions = ['clobber' => TRUE]; + $this->statsCollector->addStat( + "overall.average.received_age", + $this->statsCollector->getStatAverage("received_age.*"), + $statOptions + ); + } + + protected function recordMessageEnqueuedAge($paymentGateway, $enqueuedTime) { + // work out time between the message enqueued time and now if 'source_enqueued_time' is set + $enqueuedAge = UtcDate::getUtcTimestamp() - $enqueuedTime; + + // record a stat for the message enqueued age + $this->statsCollector->addStat("enqueued_age.{$paymentGateway}", $enqueuedAge); + } + + protected function recordGatewayAverageMessageEnqueuedAge($paymentGateway) { + // set/update the current moving average of enqueued message ages + $statOptions = ['clobber' => TRUE]; + $this->statsCollector->addStat( + "average.enqueued_age.{$paymentGateway}", + $this->statsCollector->getStatAverage("enqueued_age.{$paymentGateway}"), + $statOptions + ); + } + + protected function recordOverallMessageEnqueuedAge() { + // set/update the current moving average of enqueued message ages + $statOptions = ['clobber' => TRUE]; + $this->statsCollector->addStat( + "overall.average.enqueued_age", + $this->statsCollector->getStatAverage("enqueued_age.*"), + $statOptions + ); + } + + +} \ No newline at end of file diff --git a/sites/all/modules/queue2civicrm/queue2civicrm.info b/sites/all/modules/queue2civicrm/queue2civicrm.info index 7363dd8..807c3e9 100644 --- a/sites/all/modules/queue2civicrm/queue2civicrm.info +++ b/sites/all/modules/queue2civicrm/queue2civicrm.info @@ -6,6 +6,8 @@ dependencies[] = wmf_civicrm package = queue2civicrm files[] = Queue2civicrmTrxnCounter.php +files[] = DonationStats.php files[] = DonationQueueConsumer.php files[] = tests/includes/Message.php files[] = tests/includes/TestingSmashPigDbQueueConfiguration.php +`` \ No newline at end of file diff --git a/sites/all/modules/queue2civicrm/queue2civicrm.module b/sites/all/modules/queue2civicrm/queue2civicrm.module index e57d281..6d9c93e 100644 --- a/sites/all/modules/queue2civicrm/queue2civicrm.module +++ b/sites/all/modules/queue2civicrm/queue2civicrm.module @@ -1,7 +1,6 @@ <?php use queue2civicrm\DonationQueueConsumer; -use Statistics\Collector\Collector as StatsCollector; -use Statistics\Exporter\Prometheus as PrometheusStatsExporter; +use DonationStats; // include common functions @@ -149,13 +148,13 @@ * * TODO: metrics stuff should be a hook */ - $statsCollector = StatsCollector::getInstance(); - $prometheusExporter = new PrometheusStatsExporter('donation_stats','/var/spool/prometheus'); - $prometheusExporter->export($statsCollector->getStats(['donations.average.*', 'donations.overall.*'],true)); + + $DonationStats = new DonationStats(); + $DonationStats->exportDonationStatsToPrometheus(); if ($processed > 0) { watchdog('queue2civicrm', 'Successfully processed ' . $processed . ' contribution(s).'); -// watchdog('queue2civicrm', 'Average message age: ' . $ageMetrics['overall'] . ' seconds.'); + watchdog('queue2civicrm', 'Average message age: ' . $DonationStats->getOverallAverageMessageAge() . ' seconds.'); } else { watchdog('queue2civicrm', 'No contributions processed.'); -- To view, visit https://gerrit.wikimedia.org/r/391022 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If7fdbbff797efe240d3af20017b99340b6b0f135 Gerrit-PatchSet: 1 Gerrit-Project: wikimedia/fundraising/crm Gerrit-Branch: master Gerrit-Owner: Jgleeson <jglee...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits