[MediaWiki-commits] [Gerrit] Script to calculate MLEB download statistics - change (translatewiki)
Nikerabbit has uploaded a new change for review.
https://gerrit.wikimedia.org/r/116733
Change subject: Script to calculate MLEB download statistics
..
Script to calculate MLEB download statistics
Change-Id: Iebb2cc23edad35fb5de9ed5e862499a7124be091
---
A bin/mlebstats.php
M puppet/modules/awstats/files/awstats.cron
2 files changed, 189 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/translatewiki
refs/changes/33/116733/1
diff --git a/bin/mlebstats.php b/bin/mlebstats.php
new file mode 100755
index 000..a6c017b
--- /dev/null
+++ b/bin/mlebstats.php
@@ -0,0 +1,188 @@
+ downloads
+ );
+
+ $file = new SplFileObject( $path );
+ $file->setFlags(
+ SplFileObject::READ_AHEAD
+ | SplFileObject::DROP_NEW_LINE
+ | SplFileObject::SKIP_EMPTY
+ );
+
+ while ( !$file->eof() ) {
+ $line = $file->current();
+
+ if ( startsWith( $line, 'POS_DOWNLOADS ' ) ) {
+ $spaceInTheMidle = stripos( $line, ' ' );
+ $offsetToDownloads = (int)substr( $line,
$spaceInTheMidle + 1 );
+ break;
+ }
+
+ if ( $line === 'END_MAP' ) {
+ throw new Exception( "Unable to parse file $path" );
+ }
+
+ $file->next();
+ }
+
+ $file->fseek( $offsetToDownloads );
+
+ while ( !$file->eof() ) {
+ $line = $file->current();
+ if ( startsWith( $line, '/mleb/' ) ) {
+ list( $filename, $downloads, $hits, $size ) = explode(
' ', $line );
+ if ( endsWith( $filename, '.tar.bz2' ) ) {
+ $stats[$filename] = $downloads;
+ }
+ }
+
+ if ( $line === 'END_DOWNLOADS' ) {
+ break;
+ }
+
+ $file->next();
+ }
+
+ return $stats;
+}
+
+function getFiles( $dir ) {
+ return glob( "$dir/awstats*.txt" );
+}
+
+function getCuteName( $filename ) {
+ $matches = null;
+ preg_match( '/MediaWikiLanguageExtensionBundle-(\d{4}.\d\d)/',
$filename, $matches );
+ return $matches[1];
+}
+
+function arrayToList( $array ) {
+ array_walk( $array, function( &$value, $key ) {
+ $value = array( $key, $value );
+ } );
+ return array_values( $array );
+}
+
+function printAllTime( $stats ) {
+ $combined = array();
+ foreach ( $stats as $month ) {
+ foreach ( $month as $file => $downloads ) {
+ if ( !isset( $combined[$file] ) ) {
+ $combined[$file] = 0;
+ }
+ $combined[$file] += $downloads;
+ }
+ }
+
+ // Awful hack to able to iterate both arrays at once
+ arsort( $combined );
+ $popular = arrayToList( $combined );
+ ksort( $combined );
+ $sorted = arrayToList( $combined );
+
+
+ printf( "%-13s | %-13s\n", 'Most popular', 'In order' );
+ echo "-\n";
+ $len = count( $combined );
+ for ( $i = 0; $i < $len; $i++ ) {
+ list( $pk, $pv ) = $popular[$i];
+ list( $sk, $sv ) = $sorted[$i];
+
+ printf(
+ "%7s %5d | %7s %5d \n",
+ getCuteName( $pk ),
+ $pv,
+ getCuteName( $sk ),
+ $sv
+ );
+ }
+}
+
+function printMonth( $stats ) {
+ $total = 0;
+ foreach ( $stats as $filename => $downloads ) {
+ $total += $downloads;
+ printf( "%7s %5d\n", getCuteName( $filename ), $downloads );
+ }
+}
+
+ob_start();
+
+$skipped = array();
+$stats = array();
+
+foreach ( getFiles( $awstatsDir ) as $path ) {
+ $file = basename( $path );
+ $year = (int)substr( $file, 9, 4 );
+ $month = (int)substr( $file, 7, 2 );
+
+ if ( $year < 2013 || ( $year === 2013 && $month < 10 ) ) {
+ // No download stats in expected format available
+ continue;
+ }
+
+ try {
+ $stats["$year-$month"] = getStatsFromFile( $path );
+ } catch ( Exception $e ) {
+ $skipped[] = $file;
+ }
+}
+
+
+$lastmonth = mktime( 0, 0, 0, date( 'm' ) - 1, 1, date( 'Y' ) );
+$index = date( 'Y-n', $lastmonth );
+$pretty = date( 'Y-m', $lastmonth );
+
+if ( isset( $stats[$index] ) ) {
+ echo "Stats for last month ($pretty)\n\n";
+ printMonth( $stats[$index] );
+ echo "\n\n";
+} else {
+ echo "Unable to print stats for $pretty\n\n";
+}
+
+
+echo "All time statistics since 2013-10\n\n";
+
+printAllTime( $stats );
+
+if ( $skipped ) {
+ $skipped = implode( ', ', $skipped )
[MediaWiki-commits] [Gerrit] Script to calculate MLEB download statistics - change (translatewiki)
jenkins-bot has submitted this change and it was merged.
Change subject: Script to calculate MLEB download statistics
..
Script to calculate MLEB download statistics
Change-Id: Iebb2cc23edad35fb5de9ed5e862499a7124be091
---
A bin/scripts/mlebstats.php
M puppet/modules/awstats/files/awstats.cron
2 files changed, 191 insertions(+), 0 deletions(-)
Approvals:
KartikMistry: Looks good to me, but someone else must approve
Siebrand: Looks good to me, approved
jenkins-bot: Verified
diff --git a/bin/scripts/mlebstats.php b/bin/scripts/mlebstats.php
new file mode 100644
index 000..e391464
--- /dev/null
+++ b/bin/scripts/mlebstats.php
@@ -0,0 +1,190 @@
+ downloads
+ );
+
+ $file = new SplFileObject( $path );
+ $file->setFlags(
+ SplFileObject::READ_AHEAD
+ | SplFileObject::DROP_NEW_LINE
+ | SplFileObject::SKIP_EMPTY
+ );
+
+ while ( !$file->eof() ) {
+ $line = $file->current();
+
+ if ( startsWith( $line, 'POS_DOWNLOADS ' ) ) {
+ $spaceInTheMidle = stripos( $line, ' ' );
+ $offsetToDownloads = (int)substr( $line,
$spaceInTheMidle + 1 );
+ break;
+ }
+
+ if ( $line === 'END_MAP' ) {
+ throw new Exception( "Unable to parse file $path" );
+ }
+
+ $file->next();
+ }
+
+ $file->fseek( $offsetToDownloads );
+
+ while ( !$file->eof() ) {
+ $line = $file->current();
+ if ( startsWith( $line, '/mleb/' ) ) {
+ list( $filename, $downloads, $hits, $size ) = explode(
' ', $line );
+ if ( endsWith( $filename, '.tar.bz2' ) ) {
+ $stats[$filename] = $downloads;
+ }
+ }
+
+ if ( $line === 'END_DOWNLOADS' ) {
+ break;
+ }
+
+ $file->next();
+ }
+
+ return $stats;
+}
+
+function getFiles( $dir ) {
+ return glob( "$dir/awstats*.txt" );
+}
+
+function getCuteName( $filename ) {
+ $matches = null;
+ preg_match( '/MediaWikiLanguageExtensionBundle-(\d{4}.\d\d)/',
$filename, $matches );
+ return $matches[1];
+}
+
+function arrayToList( $array ) {
+ array_walk( $array, function( &$value, $key ) {
+ $value = array( $key, $value );
+ } );
+ return array_values( $array );
+}
+
+function printAllTime( $stats ) {
+ $combined = array();
+ foreach ( $stats as $month ) {
+ foreach ( $month as $file => $downloads ) {
+ if ( !isset( $combined[$file] ) ) {
+ $combined[$file] = 0;
+ }
+ $combined[$file] += $downloads;
+ }
+ }
+
+ // Awful hack to able to iterate both arrays at once
+ arsort( $combined );
+ $popular = arrayToList( $combined );
+ ksort( $combined );
+ $sorted = arrayToList( $combined );
+
+
+ printf( "%-13s | %-13s\n", 'Most popular', 'In order' );
+ echo "-\n";
+ $len = count( $combined );
+ for ( $i = 0; $i < $len; $i++ ) {
+ list( $pk, $pv ) = $popular[$i];
+ list( $sk, $sv ) = $sorted[$i];
+
+ printf(
+ "%7s %5d | %7s %5d \n",
+ getCuteName( $pk ),
+ $pv,
+ getCuteName( $sk ),
+ $sv
+ );
+ }
+}
+
+function printMonth( $stats ) {
+ $total = 0;
+ foreach ( $stats as $filename => $downloads ) {
+ $total += $downloads;
+ printf( "%7s %5d\n", getCuteName( $filename ), $downloads );
+ }
+
+ printf( "Total: %d downloads\n", $total );
+}
+
+ob_start();
+
+$skipped = array();
+$stats = array();
+
+foreach ( getFiles( $awstatsDir ) as $path ) {
+ $file = basename( $path );
+ $year = (int)substr( $file, 9, 4 );
+ $month = (int)substr( $file, 7, 2 );
+
+ if ( $year < 2013 || ( $year === 2013 && $month < 10 ) ) {
+ // No download stats in expected format available
+ continue;
+ }
+
+ try {
+ $stats["$year-$month"] = getStatsFromFile( $path );
+ } catch ( Exception $e ) {
+ $skipped[] = $file;
+ }
+}
+
+
+$lastmonth = mktime( 0, 0, 0, date( 'm' ) - 1, 1, date( 'Y' ) );
+$index = date( 'Y-n', $lastmonth );
+$pretty = date( 'Y-m', $lastmonth );
+
+if ( isset( $stats[$index] ) ) {
+ echo "Stats for last month ($pretty)\n\n";
+ printMonth( $stats[$index] );
+ echo "\n\n";
+} else {
+ echo "Unable to print stats for $pretty\n\n";
+}
+
+
+echo "All time statistics sinc
