Addshore has submitted this change and it was merged.
Change subject: Move cron files from puppet to this repo
......................................................................
Move cron files from puppet to this repo
Relates to the change at Ib9c541e67cfe7580e2432a2cb813ea1de36863e6
Bug: T140095
Depends-On: Ib9c541e67cfe7580e2432a2cb813ea1de36863e6
Change-Id: I7d05126bc62b7606b3f16d2d2e38b64f8ae5c03d
(cherry picked from commit 50c85976e3780bd63354b48bd823e6257b939025)
---
M README.md
A cron/daily.03.sh
A cron/daily.12.sh
A cron/minutely.sh
A cron/weekly.sh
M src/wikidata/dumpScanProcessing.php
6 files changed, 103 insertions(+), 5 deletions(-)
Approvals:
Addshore: Looks good to me, approved
jenkins-bot: Verified
diff --git a/README.md b/README.md
index 84a3593..8b1272b 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,7 @@
mm-wikidata-pass password1
mm-wikidatatech-pass password2
mm-user [email protected]
+ dump-dir /tmp/dumps
## Graphite
diff --git a/cron/daily.03.sh b/cron/daily.03.sh
new file mode 100755
index 0000000..788c345
--- /dev/null
+++ b/cron/daily.03.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# @author Addshore
+#
+# This script should be run through cron at 03 hours every day.
+# The first parameter should be the directory this repo is checked out into.
+
+if [ -z "$1" ]
+ then
+ echo "No argument supplied"
+ exit 1
+fi
+
+# Data model
+eval "$1/src/wikidata/datamodel/properties_by_datatype.php"
+eval "$1/src/wikidata/datamodel/terms_by_language.php"
+eval "$1/src/wikidata/datamodel/sitelinks_per_site.php"
+eval "$1/src/wikidata/datamodel/sitelinks_per_item.php"
+eval "$1/src/wikidata/datamodel/statements_per_entity.php"
+eval "$1/src/wikidata/sparql/ranks.php"
+eval "$1/src/wikidata/sparql/instanceof.php"
+eval "$1/src/wikidata/wikidata-analysis/metrics.php"
+
+# Entity usage
+eval "$1/src/wikidata/entityUsage.php"
+
+# Social
+eval "$1/src/wikidata/social/facebook.php"
+eval "$1/src/wikidata/social/googleplus.php"
+eval "$1/src/wikidata/social/identica.php"
+eval "$1/src/wikidata/social/irc.php"
+eval "$1/src/wikidata/social/mail.php"
+eval "$1/src/wikidata/social/newsletter.php"
+eval "$1/src/wikidata/social/techmail.php"
+eval "$1/src/wikidata/social/twitter.php"
+
+# Misc
+eval "$1/src/wikidata/phabricatorTasks.php"
+eval "$1/src/wikidata/showcaseItems.php"
+eval "$1/src/wikidata/specialEntityData.php"
+eval "$1/src/catwatch/userprops.php"
+
+# Site Stats
+eval "$1/src/wikidata/site_stats/good_articles.sh"
+eval "$1/src/wikidata/site_stats/total_edits.sh"
+eval "$1/src/wikidata/site_stats/total_pages.sh"
+eval "$1/src/wikidata/site_stats/active_users.php"
+eval "$1/src/wikidata/site_stats/users.sh"
+eval "$1/src/wikidata/site_stats/admins.sh"
+eval "$1/src/wikidata/site_stats/bureaucrats.sh"
+eval "$1/src/wikidata/site_stats/bots.sh"
+eval "$1/src/wikidata/site_stats/propertycreators.sh"
+eval "$1/src/wikidata/site_stats/rolling_rc.php"
+eval "$1/src/wikidata/site_stats/pages_by_namespace.php"
+eval "$1/src/wikidata/site_stats/user_languages.php"
+eval "$1/src/wikidata/site_stats/page_size.php"
+
diff --git a/cron/daily.12.sh b/cron/daily.12.sh
new file mode 100755
index 0000000..2b108f9
--- /dev/null
+++ b/cron/daily.12.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# @author Addshore
+#
+# This script should be run through cron at 12 hours every day.
+# The first parameter should be the directory this repo is checked out into.
+
+if [ -z "$1" ]
+ then
+ echo "No argument supplied"
+ exit 1
+fi
+
+# Logrotate is at 6:25, + time for rsync (hourly?), 12 gives us roughly 6 hours
+eval "$1/src/wikidata/apiLogScanner.sh"
\ No newline at end of file
diff --git a/cron/minutely.sh b/cron/minutely.sh
new file mode 100755
index 0000000..e8543fb
--- /dev/null
+++ b/cron/minutely.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# @author Addshore
+#
+# This script should be run through cron every minute.
+# The first parameter should be the directory this repo is checked out into.
+
+if [ -z "$1" ]
+ then
+ echo "No argument supplied"
+ exit 1
+fi
+
+eval "$1/src/wikidata/dispatch.php"
+eval "$1/src/wikidata/recentChanges.php"
+eval "$1/src/wikidata/sparql/minutely.php"
diff --git a/cron/weekly.sh b/cron/weekly.sh
new file mode 100755
index 0000000..8ca7645
--- /dev/null
+++ b/cron/weekly.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# @author Addshore
+#
+# This script should be run every sunday at 01 hours
+# The first parameter should be the directory this repo is checked out into.
+
+if [ -z "$1" ]
+ then
+ echo "No argument supplied"
+ exit 1
+fi
+
+#noop
\ No newline at end of file
diff --git a/src/wikidata/dumpScanProcessing.php
b/src/wikidata/dumpScanProcessing.php
index 3eee346..cfa4499 100755
--- a/src/wikidata/dumpScanProcessing.php
+++ b/src/wikidata/dumpScanProcessing.php
@@ -3,13 +3,12 @@
/**
* @author Addshore
- * Data directory must be passed into the script as the first parameter
*/
-if( !array_key_exists(1, $argv) ) {
- throw new Exception( "Data directory not passed into script" );
-}
-$dataDir = $argv[1];
+require_once( __DIR__ . '/../../lib/load.php' );
+
+$config = Config::getConfig();
+$dataDir = $config['dump-dir'];
//Make sure the output dir exists
if ( !file_exists( $dataDir ) ) {
throw new Exception( "Data directory does not exist: " . $dataDir );
--
To view, visit https://gerrit.wikimedia.org/r/298498
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7d05126bc62b7606b3f16d2d2e38b64f8ae5c03d
Gerrit-PatchSet: 3
Gerrit-Project: analytics/wmde/scripts
Gerrit-Branch: production
Gerrit-Owner: Addshore <[email protected]>
Gerrit-Reviewer: Addshore <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
