Mwalker has uploaded a new change for review.
https://gerrit.wikimedia.org/r/150097
Change subject: Use HKEYS instead of HSCAN
......................................................................
Use HKEYS instead of HSCAN
HSCAN was only introduced in Redis 1.8.0 but we run Ubuntu 12.04 which
is behind that version. Therefore, instead of being nice and batching
things; do it all at once.
Change-Id: If65084b6cd0e80afe89fc30022cbde28e2d86a80
---
M lib/RedisWrapper.js
M lib/threads/gc.js
2 files changed, 23 insertions(+), 36 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator
refs/changes/97/150097/1
diff --git a/lib/RedisWrapper.js b/lib/RedisWrapper.js
index 0504dae..ce1f7b1 100644
--- a/lib/RedisWrapper.js
+++ b/lib/RedisWrapper.js
@@ -131,8 +131,8 @@
RedisWrapper.prototype.hlen = function( key ) {
return Promise.promisify( this.client.hlen, false, this.client )( key );
};
-RedisWrapper.prototype.hscan = function( key, iterator ) {
- return Promise.promisify( this.client.hscan, false, this.client )( key,
iterator );
+RedisWrapper.prototype.hkeys = function( key ) {
+ return Promise.promisify( this.client.hkeys, false, this.client )( key
);
};
module.exports = RedisWrapper;
diff --git a/lib/threads/gc.js b/lib/threads/gc.js
index 86d0f0a..0a23225 100644
--- a/lib/threads/gc.js
+++ b/lib/threads/gc.js
@@ -45,7 +45,6 @@
var running = false;
var intervalTimer = null;
var redisClient = null;
-var redisIterator = 0;
/* === Public Exported Functions =========================================== */
/**
@@ -132,48 +131,36 @@
*/
function clearJobStatusObjects() {
var clearedFailedJobs = 0,
- clearedNonFailedJobs = 0;
+ clearedNonFailedJobs = 0,
+ fjl = Date.now() -
config.garbage_collection.failed_job_lifetime,
+ ajl = Date.now() - config.garbage_collection.job_lifetime;
console.info( "Starting run to clear job status objects", { channel:
'gc' } );
- var iterateRedis = function( iterationResult ) {
- var multi = redisClient.multi();
- var fjl = Date.now() -
config.garbage_collection.failed_job_lifetime,
- ajl = Date.now() -
config.garbage_collection.job_lifetime;
-
- redisIterator = iterationResult[0];
- for ( var i = 0; i < iterationResult[1].length; i += 2 ) {
- var job = jd.fromJson( iterationResult[1][ i+1 ] );
+ var scrubKey = function( key ) {
+ return redisClient.hget( config.redis.status_set_name, key
).then( function( jdjson ) {
+ var job = jd.fromJson( jdjson );
if ( job.status === 'failed' && ( job.timestamp < fjl )
) {
clearedFailedJobs += 1;
- multi.hdel( config.redis.status_set_name,
job.collectionId );
+ return redisClient.hdel(
config.redis.status_set_name, job.collectionId );
} else if ( job.timestamp < ajl ) {
clearedNonFailedJobs += 1;
- multi.hdel( config.redis.status_set_name,
job.collectionId );
+ return redisClient.hdel(
config.redis.status_set_name, job.collectionId );
}
- }
-
- if ( redisIterator === "0" ) {
- console.info(
- "Cleared %s non-failed jobs and %s failed jobs",
- clearedNonFailedJobs,
- clearedFailedJobs,
- { channel: 'gc' }
- );
- } else {
- return Promise.promisify( multi.exec, false, multi )()
- .then( function() {
- return multi.hscan(
config.redis.status_set_name, redisIterator );
- } )
- .then( iterateRedis );
- }
+ } );
};
+ var scrubKeyGuarded = Promise.guard( Promise.guard.n( 5 ), scrubKey );
- return redisClient
- .hlen( config.redis.status_set_name )
- .then( function() {
- return redisClient.hscan( config.redis.status_set_name,
redisIterator );
- } )
- .then( iterateRedis );
+ return redisClient.hkeys( config.redis.status_set_name ).then(
function( keys ) {
+ console.info( "Got %s status keys to iterate through",
keys.length, { channel: 'gc' } );
+ return keys;
+ } ).map( scrubKeyGuarded ).then( function() {
+ console.info(
+ "Cleared %s non-failed jobs and %s failed jobs",
+ clearedNonFailedJobs,
+ clearedFailedJobs,
+ { channel: 'gc' }
+ );
+ } );
}
function cleanDir( dir, lifetime ) {
--
To view, visit https://gerrit.wikimedia.org/r/150097
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If65084b6cd0e80afe89fc30022cbde28e2d86a80
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator
Gerrit-Branch: master
Gerrit-Owner: Mwalker <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
