jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/404682 )
Change subject: Merge branch 'master' of ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm into deployment ...................................................................... Merge branch 'master' of ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/crm into deployment a47c16f Restored `composer test` script 6f03194 DonationStats now tracking average processing times abfae19 CiviCRM submodule update 356b429 Add Citibank International to payment instruments on new installs. 97f582f Re-save any contacts without a display name. 69fdda3 Add extension to change contact types a8d515d Update CiviCRM submodule f4821dc Added DonationStatsCollector class to record stats across Donation Queue Consumer. Change-Id: Idd51fbd22966b9f79bff0bcfb25ff5acaeb1189c --- D phpunit.xml D sites/all/modules/queue2civicrm/tests/phpunit/DonationStatsCollectorTest.php 2 files changed, 0 insertions(+), 450 deletions(-) Approvals: Jgleeson: Looks good to me, approved jenkins-bot: Verified diff --git a/phpunit.xml b/phpunit.xml deleted file mode 100644 index 3204b23..0000000 --- a/phpunit.xml +++ /dev/null @@ -1,59 +0,0 @@ -<<<<<<< HEAD (354f32 Update CiviCRM submodule) -======= -<phpunit - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:noNamespaceSchemaLocation="http://schema.phpunit.de/3.7/phpunit.xsd"; - bootstrap="sites/default/bootstrap-phpunit.php"> - - - <testsuites> - <testsuite name="large_donation tests"> - <directory>sites/all/modules/large_donation/tests</directory> - </testsuite> - <testsuite name="offline2civicrm tests"> - <directory>sites/all/modules/offline2civicrm/tests</directory> - <exclude>sites/all/modules/offline2civicrm/tests/includes/BaseChecksFileTest.php</exclude> - </testsuite> - <testsuite name="exchange_rates tests"> - <directory>sites/all/modules/exchange_rates/tests/phpunit</directory> - </testsuite> - <testsuite name="metrics_reporting tests"> - <directory>sites/all/modules/metrics_reporting/tests/phpunit</directory> - </testsuite> - <testsuite name="orphan_slayer tests"> - <directory>sites/all/modules/orphan_slayer/tests/phpunit</directory> - </testsuite> - <testsuite name="queue2civicrm tests"> - <directory>sites/all/modules/queue2civicrm/tests/phpunit</directory> - </testsuite> - <testsuite name="recurring_globalcollect tests"> - <directory>sites/all/modules/recurring_globalcollect/tests</directory> - </testsuite> - <testsuite name="thank_you tests"> - <directory>sites/all/modules/thank_you/tests/phpunit</directory> - </testsuite> - <testsuite name="wmf_audit tests"> - <directory>sites/all/modules/wmf_audit/tests</directory> - </testsuite> - <testsuite name="wmf_campaigns tests"> - <directory>sites/all/modules/wmf_campaigns/tests</directory> - </testsuite> - <testsuite name="wmf_civicrm tests"> - <directory>sites/all/modules/wmf_civicrm/tests/phpunit</directory> - </testsuite> - <testsuite name="wmf_common tests"> - <directory>sites/all/modules/wmf_common/tests/phpunit</directory> - </testsuite> - <testsuite name="wmf_communication tests"> - <directory>sites/all/modules/wmf_communication/tests/phpunit</directory> - <exclude>sites/all/modules/wmf_communication/tests/phpunit/CiviMailTestBase.php</exclude> - </testsuite> - <testsuite name="omnimail_tests"> - <directory>sites/default/civicrm/extensions/org.wikimedia.omnimail/tests/phpunit</directory> - </testsuite> - <testsuite name="contacteditor_tests"> - <directory>sites/default/civicrm/extensions/org.wikimedia.contacteditor/tests/phpunit</directory> - </testsuite> - </testsuites> -</phpunit> ->>>>>>> BRANCH (f4821d Added DonationStatsCollector class to record stats across Do) diff --git a/sites/all/modules/queue2civicrm/tests/phpunit/DonationStatsCollectorTest.php b/sites/all/modules/queue2civicrm/tests/phpunit/DonationStatsCollectorTest.php deleted file mode 100644 index 9cdd2c4..0000000 --- a/sites/all/modules/queue2civicrm/tests/phpunit/DonationStatsCollectorTest.php +++ /dev/null @@ -1,391 +0,0 @@ -<?php - -/** - * Tests for DonationStatsCollector. - * - * Where visibility is an obstacle, we use reflection to trigger behaviour to ensure internal results are correct - * - * @group Queue2Civicrm - * @group DonationStats - */ -class DonationStatsCollectorTest extends \BaseWmfDrupalPhpUnitTestCase { - - protected $statsFilename; - - protected $statsFilePath; - - protected $statsFileExtension; - - /** - * @var \DonationStatsCollector - */ - protected $donationStatsCollector; - - public function setUp() { - parent::setUp(); - $this->donationStatsCollector = $this->getTestDonationStatsCollectorInstance(); - } - - /** - * @dataProvider singleDummyDonationDataProvider - */ - public function testCanRecordSingleDonationStat($message, $contribution) { - $contribution['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution['receive_date']); - - $this->donationStatsCollector->recordDonationStats($message, $contribution); - - $recordedStats = $this->donationStatsCollector->getAllStats(); - - $expected = [ - 'test_namespace.count' => ['ACME_PAYMENTS' => 1], - 'test_namespace.transaction_age' => ['ACME_PAYMENTS' => 3600], // 1 hour - ]; - - $this->assertEquals($expected, $recordedStats); - } - - /** - * @dataProvider singleDummyDonationDataProvider - */ - public function testCanRecordMultipleDonationStats($message, $contribution) { - $contribution['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution['receive_date']); - - // count stat doesn't exist yet - $this->assertFalse($this->donationStatsCollector->exists("count")); - - //record stats - for ($i = 0; $i < 3; $i++) { - $this->donationStatsCollector->recordDonationStats($message, $contribution); - } - - $recordedStats = $this->donationStatsCollector->getAllStats(); - - $expected = [ - 'test_namespace.count' => ['ACME_PAYMENTS' => 3], - 'test_namespace.transaction_age' => [ - 'ACME_PAYMENTS' => [3600, 3600, 3600], - ], - ]; - - $this->assertEquals($expected, $recordedStats); - } - - /** - * @dataProvider singleDummyDonationDataProvider - */ - public function testCanGenerateDonationProcessingRateStats($message, $contribution) { - $contribution['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution['receive_date']); - // open up protected methods - $reflectionMethodAggregates = new ReflectionMethod('DonationStatsCollector', 'generateAggregateStats'); - $reflectionMethodPurgeSuperfluousStats = new ReflectionMethod('DonationStatsCollector', 'purgeSuperfluousStats'); - $reflectionMethodAggregates->setAccessible(TRUE); - $reflectionMethodPurgeSuperfluousStats->setAccessible(TRUE); - - $start = microtime(TRUE); - $this->donationStatsCollector->recordDonationStats($message, $contribution); - $end = microtime(TRUE); - - //simulate the tracking with supplied timestamps - $this->donationStatsCollector->timerNamespace = "test"; - $this->donationStatsCollector->startTimer(NULL, $start); - $this->donationStatsCollector->endTimer(NULL, $end); - - // call generateAggregateStats() && purgeSuperfluousStats() - $reflectionMethodAggregates->invoke($this->donationStatsCollector); - $reflectionMethodPurgeSuperfluousStats->invoke($this->donationStatsCollector); - - $recordedStats = $this->donationStatsCollector->getAllStats(); - - // we expect the internal times to be generated exactly as worked out below - $batchProcessingTime = $end - $start; - $totalDonations = 1; // we only recorded one (line 80) - $donationsProcessedPerSecond = $totalDonations / $batchProcessingTime; - - $expected = [ - 'test_namespace.count' => ['ACME_PAYMENTS' => 1, 'all' => 1], - 'test_namespace.average_transaction_age' => ['ACME_PAYMENTS' => 3600, 'all' => 3600], - 'test_namespace.processing_rate' => [ - 'period=batch' => $batchProcessingTime, - 'period=1s' => $donationsProcessedPerSecond, - 'period=5s' => $donationsProcessedPerSecond * 5, - 'period=10s' => $donationsProcessedPerSecond * 10, - 'period=30s' => $donationsProcessedPerSecond * 30, - ], - ]; - - $this->assertEquals($expected, $recordedStats); - } - - /** - * @dataProvider doubleDummyDonationDataProvider - */ - public function testCanGenerateAverageDataFromRecordedStats($message, $contribution) { - // open up protected method - $reflectionMethod = new ReflectionMethod('DonationStatsCollector', 'generateAverageStats'); - $reflectionMethod->setAccessible(TRUE); - - //record stats - for ($i = 0; $i < 2; $i++) { - $message[$i]['source_enqueued_time'] = \SmashPig\Core\UtcDate::getUtcTimestamp($message[$i]['source_enqueued_time']); - $contribution[$i]['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution[$i]['receive_date']); - $this->donationStatsCollector->recordDonationStats($message[$i], $contribution[$i]); - } - - // call $DonationStatsCollector->generateAverageStats() - $reflectionMethod->invoke($this->donationStatsCollector); - - $recordedStats = $this->donationStatsCollector->getAllStats(); - - $expected = [ - 'test_namespace.count' => [ - 'ACME_PAYMENTS' => 1, - 'NICE_GATEWAY' => 1, - ], - 'test_namespace.transaction_age' => ['ACME_PAYMENTS' => 3600, 'NICE_GATEWAY' => 3600], - 'test_namespace.enqueued_age' => ['ACME_PAYMENTS' => 3600, 'NICE_GATEWAY' => 3600], - 'test_namespace.average_enqueued_age' => [ - 'ACME_PAYMENTS' => 3600, //average data - 'NICE_GATEWAY' => 3600, //average data - ], - 'test_namespace.average_transaction_age' => [ - 'ACME_PAYMENTS' => 3600, //average data - 'NICE_GATEWAY' => 3600, //average data - ], - ]; - - $this->assertEquals($expected, $recordedStats); - } - - /** - * @dataProvider doubleDummyDonationDataProvider - */ - public function testCanGenerateOverallDataFromRecordedStats($message, $contribution) { - // open up protected methods - $reflectionMethodAverages = new ReflectionMethod('DonationStatsCollector', 'generateAverageStats'); - $reflectionMethodOverall = new ReflectionMethod('DonationStatsCollector', 'generateOverallStats'); - $reflectionMethodAverages->setAccessible(TRUE); - $reflectionMethodOverall->setAccessible(TRUE); - - //record stats - for ($i = 0; $i < 2; $i++) { - $message[$i]['source_enqueued_time'] = \SmashPig\Core\UtcDate::getUtcTimestamp($message[$i]['source_enqueued_time']); - $contribution[$i]['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution[$i]['receive_date']); - $this->donationStatsCollector->recordDonationStats($message[$i], $contribution[$i]); - } - - // call generateAverageStats() && generateOverallStats() - $reflectionMethodAverages->invoke($this->donationStatsCollector); - $reflectionMethodOverall->invoke($this->donationStatsCollector); - - $recordedStats = $this->donationStatsCollector->getAllStats(); - - $expected = [ - 'test_namespace.count' => [ - 'ACME_PAYMENTS' => 1, - 'NICE_GATEWAY' => 1, - 'all' => 2, // overall data - ], - 'test_namespace.transaction_age' => ['ACME_PAYMENTS' => 3600, 'NICE_GATEWAY' => 3600], - 'test_namespace.enqueued_age' => ['ACME_PAYMENTS' => 3600, 'NICE_GATEWAY' => 3600], - 'test_namespace.average_enqueued_age' => [ - 'ACME_PAYMENTS' => 3600, - 'NICE_GATEWAY' => 3600, - 'all' => 3600 // overall data - ], - 'test_namespace.average_transaction_age' => [ - 'ACME_PAYMENTS' => 3600, - 'NICE_GATEWAY' => 3600, - 'all' => 3600 // overall data - ], - ]; - - $this->assertEquals($expected, $recordedStats); - } - - /** - * @dataProvider singleDummyDonationDataProvider - */ - public function testGetOverallAverageGatewayTransactionAge($message, $contribution) { - $contribution['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution['receive_date']); - // open up protected methods - $reflectionMethodAverages = new ReflectionMethod('DonationStatsCollector', 'generateAverageStats'); - $reflectionMethodOverall = new ReflectionMethod('DonationStatsCollector', 'generateOverallStats'); - $reflectionMethodAverages->setAccessible(TRUE); - $reflectionMethodOverall->setAccessible(TRUE); - - $this->donationStatsCollector->recordDonationStats($message, $contribution); - - // call generateAverageStats() && generateOverallStats() - $reflectionMethodAverages->invoke($this->donationStatsCollector); - $reflectionMethodOverall->invoke($this->donationStatsCollector); - - $OverallAverageGatewayTransactionAge = $this->donationStatsCollector->getOverallAverageGatewayTransactionAge(); - - $this->assertEquals(3600, $OverallAverageGatewayTransactionAge); - } - - /** - * @dataProvider singleDummyDonationDataProvider - */ - public function testExportingStatsCreatesOutputFile($message, $contribution) { - $this->setUpStatsOutFileProperties(); - $contribution['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution['receive_date']); - - // test output file does not currently exist - $this->assertFileNotExists($this->statsFilePath . $this->statsFilename . $this->statsFileExtension); - - $this->donationStatsCollector->recordDonationStats($message, $contribution); - $this->donationStatsCollector->outputFileName = $this->statsFilename; - $this->donationStatsCollector->outputFilePath = rtrim($this->statsFilePath, '/'); - $this->donationStatsCollector->export(); - - //test output file has been created - $this->assertFileExists($this->statsFilePath . $this->statsFilename . $this->statsFileExtension); - - // clean up - $this->removeStatsOutFile(); - } - - /** - * We also check for the mapping and presence of Prometheus labels {key=value} here. - * - * @dataProvider singleDummyDonationDataProvider - */ - public function testExportedPrometheusOutputIsCorrect($message, $contribution) { - $this->setUpStatsOutFileProperties(); - $contribution['receive_date'] = \SmashPig\Core\UtcDate::getUtcDatabaseString($contribution['receive_date']); - - $this->donationStatsCollector->recordDonationStats($message, $contribution); - $this->donationStatsCollector->outputFileName = $this->statsFilename; - $this->donationStatsCollector->outputFilePath = rtrim($this->statsFilePath, '/'); - $this->donationStatsCollector->export(); - - $expectedStatsOutput = [ - 'test_namespace_average_transaction_age{gateway="ACME_PAYMENTS"}' => '3600', - 'test_namespace_average_transaction_age{gateway="all"}' => '3600', - 'test_namespace_count{gateway="ACME_PAYMENTS"}' => '1', - 'test_namespace_count{gateway="all"}' => '1', - ]; - - $statsWrittenAssocArray = $this->buildArrayFromPrometheusOutputFile( - $this->statsFilePath . $this->statsFilename . $this->statsFileExtension - ); - //compare written stats data with expected - $this->assertEquals($expectedStatsOutput, $statsWrittenAssocArray); - - // clean up - $this->removeStatsOutFile(); - } - - /** - * 'receive_date' is stored as the relative difference to our actual-usage overwriting timestamp - * - * Data providers are called before all setUp methods which causes a lag between timestamps generated within - * Data providers and actual tests. This causes problems when you are asserting time values based on now() - * between test and provider (they don't match). Numerous workarounds exist, including this approach. - * - * @return array - */ - public function singleDummyDonationDataProvider() { - return [ - [ - ['gateway' => "ACME_PAYMENTS"], - ['receive_date' => '-1 hour'], - ], - ]; - } - - /** - * 'receive_date' is stored as the relative difference to our actual-usage overwriting timestamp - * 'source_enqueued_time' is stored as the relative difference to our actual-usage overwriting timestamp - * - * Data providers are called before all setUp methods which causes a lag between timestamps generated within - * Data providers and actual tests. This causes problems when you are asserting time values based on now() - * between test and provider (they don't match). Numerous workarounds exist, including this approach. - * - * @return array - */ - public function doubleDummyDonationDataProvider() { - return [ - [ - [ - ['gateway' => "ACME_PAYMENTS", 'source_enqueued_time' => '-1 hour'], - ['gateway' => "NICE_GATEWAY", 'source_enqueued_time' => '-1 hour'], - ], - [ - ['receive_date' => '-1 hour'], - ['receive_date' => '-1 hour'], - ], - ], - ]; - } - - public function tearDown() { - parent::tearDown(); - } - - private function setUpStatsOutFileProperties() { - $this->statsFilename = "test_stats"; - $this->statsFilePath = CRM_Utils_File::tempdir(); - $this->statsFileExtension = '.prom'; - } - - private function removeStatsOutFile() { - unlink($this->statsFilePath . $this->statsFilename . $this->statsFileExtension); - rmdir($this->statsFilePath); - } - - private function getTestDonationStatsCollectorInstance() { - $this->resetDonationStatsCollector(); - $DonationStatsCollector = DonationStatsCollector::getInstance(); - $DonationStatsCollector->setNamespace(("test_namespace")); - return $DonationStatsCollector; - } - - /** - * Test helper method to convert Prometheus outfile files to associative arrays - * - * @param $prometheusFileLocation - * - * @return array|string - */ - private function buildArrayFromPrometheusOutputFile($prometheusFileLocation) { - $statsWrittenAssocArray = []; - if (file_exists($prometheusFileLocation)) { - $statsFileFullPath = $prometheusFileLocation; - $statsWritten = rtrim(file_get_contents($statsFileFullPath)); // remove trailing \n - $statsWrittenLinesArray = explode("\n", $statsWritten); - foreach ($statsWrittenLinesArray as $statsLine) { - list($name, $value) = explode(" ", $statsLine); - if (array_key_exists($name, $statsWrittenAssocArray)) { - if (is_array($statsWrittenAssocArray[$name])) { - $statsWrittenAssocArray[$name][] = $value; - } - else { - $statsWrittenAssocArray[$name] = [$statsWrittenAssocArray[$name], $value]; - } - } - else { - $statsWrittenAssocArray[$name] = $value; - } - } - } - else { - return "Prometheus file does not exist"; - } - - return $statsWrittenAssocArray; - } - - /** - * DonationStatsCollector singleton needs clearing before each test (and on first run due to DonationQueueTest). - * - * DonationQueueTest indirectly records stats when calling DonationQueueConsumer::processMessage() so we clear all - * DonationStatsCollector before running DonationStatsCollectorTest to work form a known starting point. - * - * @see DonationQueueTest - */ - private function resetDonationStatsCollector() { - DonationStatsCollector::tearDown(TRUE); - } -} -- To view, visit https://gerrit.wikimedia.org/r/404682 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Idd51fbd22966b9f79bff0bcfb25ff5acaeb1189c Gerrit-PatchSet: 2 Gerrit-Project: wikimedia/fundraising/crm Gerrit-Branch: deployment Gerrit-Owner: Jgleeson <jglee...@wikimedia.org> Gerrit-Reviewer: Ejegg <ej...@ejegg.com> Gerrit-Reviewer: Jgleeson <jglee...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
