https://www.mediawiki.org/wiki/Special:Code/MediaWiki/106605
Revision: 106605
Author: aaron
Date: 2011-12-18 21:50:41 +0000 (Sun, 18 Dec 2011)
Log Message:
-----------
* Added support for process caching and made it used for SHA-1. Non-FS backends
will be more inclined to use this (PHP alreay caches FS stats).
* Refactored LockServerDaemon a bit to avoid "connect on construct" antipattern.
Modified Paths:
--------------
branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
branches/FileBackend/phase3/maintenance/locking/LockServerDaemon.php
Modified:
branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
===================================================================
--- branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
2011-12-18 21:43:31 UTC (rev 106604)
+++ branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
2011-12-18 21:50:41 UTC (rev 106605)
@@ -38,7 +38,7 @@
return null;
}
- function store( array $params ) {
+ protected function doStore( array $params ) {
$status = Status::newGood();
list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
@@ -79,7 +79,7 @@
return $status;
}
- function copy( array $params ) {
+ protected function doCopy( array $params ) {
$status = Status::newGood();
list( $c, $source ) = $this->resolveStoragePath( $params['src']
);
@@ -127,7 +127,7 @@
return $status;
}
- function move( array $params ) {
+ protected function doMove( array $params ) {
$status = Status::newGood();
list( $c, $source ) = $this->resolveStoragePath( $params['src']
);
@@ -176,7 +176,7 @@
return $status;
}
- function delete( array $params ) {
+ protected function doDelete( array $params ) {
$status = Status::newGood();
list( $c, $source ) = $this->resolveStoragePath( $params['src']
);
@@ -203,7 +203,7 @@
return $status;
}
- function concatenate( array $params ) {
+ protected function doConcatenate( array $params ) {
$status = Status::newGood();
list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
@@ -301,7 +301,7 @@
return $status;
}
- function create( array $params ) {
+ protected function doCreate( array $params ) {
$status = Status::newGood();
list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
Modified: branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
===================================================================
--- branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
2011-12-18 21:43:31 UTC (rev 106604)
+++ branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
2011-12-18 21:50:41 UTC (rev 106605)
@@ -356,6 +356,10 @@
* @since 1.19
*/
abstract class FileBackend extends FileBackendBase {
+ /** @var Array */
+ protected $cache = array(); // (storage path => key => value)
+ protected $maxCacheSize = 50; // integer; max paths with entries
+
/**
* Store a file into the backend from a file on disk.
* Do not call this function from places outside FileBackend and FileOp.
@@ -367,9 +371,18 @@
* @param $params Array
* @return Status
*/
- abstract public function store( array $params );
+ final public function store( array $params ) {
+ $status = $this->doStore( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
/**
+ * @see FileBackend::store()
+ */
+ abstract protected function doStore( array $params );
+
+ /**
* Copy a file from one storage path to another in the backend.
* Do not call this function from places outside FileBackend and FileOp.
* $params include:
@@ -380,9 +393,18 @@
* @param $params Array
* @return Status
*/
- abstract public function copy( array $params );
+ final public function copy( array $params ) {
+ $status = $this->doCopy( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
/**
+ * @see FileBackend::copy()
+ */
+ abstract protected function doCopy( array $params );
+
+ /**
* Delete a file at the storage path.
* Do not call this function from places outside FileBackend and FileOp.
* $params include:
@@ -391,9 +413,18 @@
* @param $params Array
* @return Status
*/
- abstract public function delete( array $params );
+ final public function delete( array $params ) {
+ $status = $this->doDelete( $params );
+ $this->clearCache( array( $params['src'] ) );
+ return $status;
+ }
/**
+ * @see FileBackend::delete()
+ */
+ abstract protected function doDelete( array $params );
+
+ /**
* Move a file from one storage path to another in the backend.
* Do not call this function from places outside FileBackend and FileOp.
* $params include:
@@ -404,7 +435,16 @@
* @param $params Array
* @return Status
*/
- public function move( array $params ) {
+ final public function move( array $params ) {
+ $status = $this->doMove( $params );
+ $this->clearCache( array( $params['src'], $params['dst'] ) );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::move()
+ */
+ protected function doMove( array $params ) {
// Copy source to dest
$status = $this->backend->copy( $params );
if ( !$status->isOK() ) {
@@ -426,9 +466,18 @@
* @param $params Array
* @return Status
*/
- abstract public function concatenate( array $params );
+ final public function concatenate( array $params ) {
+ $status = $this->doConcatenate( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
/**
+ * @see FileBackend::concatenate()
+ */
+ abstract protected function doConcatenate( array $params );
+
+ /**
* Create a file in the backend with the given contents.
* Do not call this function from places outside FileBackend and FileOp.
* $params include:
@@ -439,8 +488,17 @@
* @param $params Array
* @return Status
*/
- abstract public function create( array $params );
+ final public function create( array $params ) {
+ $status = $this->doCreate( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
+ /**
+ * @see FileBackend::create()
+ */
+ abstract protected function doCreate( array $params );
+
public function prepare( array $params ) {
return Status::newGood();
}
@@ -454,11 +512,20 @@
}
public function getFileSha1Base36( array $params ) {
- $fsFile = $this->getLocalReference( array( 'src' =>
$params['src'] ) );
+ $path = $params['src'];
+ if ( isset( $this->cache[$path]['sha1'] ) ) {
+ return $this->cache[$path]['sha1'];
+ }
+ $fsFile = $this->getLocalReference( array( 'src' => $path ) );
if ( !$fsFile ) {
return false;
} else {
- return $fsFile->getSha1Base36();
+ $sha1 = $fsFile->getSha1Base36();
+ if ( $sha1 !== false ) { // don't cache negatives
+ $this->trimCache(); // limit memory
+ $this->cache[$path]['sha1'] = $sha1;
+ }
+ return $sha1;
}
}
@@ -565,6 +632,8 @@
}
}
+ // Clear any cache entries (after locks acquired)
+ $this->clearCache();
// Actually attempt the operation batch...
$status->merge( FileOp::attemptBatch( $performOps, $opts ) );
@@ -572,6 +641,35 @@
}
/**
+ * Invalidate the file existence and property cache
+ *
+ * @param $paths Array Clear cache for specific files
+ * @return void
+ */
+ final public function clearCache( array $paths = null ) {
+ if ( $paths === null ) {
+ $this->cache = array();
+ } else {
+ foreach ( $paths as $path ) {
+ unset( $this->cache[$path] );
+ }
+ }
+ }
+
+ /**
+ * Prune the cache if it is too big to add an item
+ *
+ * @return void
+ */
+ protected function trimCache() {
+ if ( count( $this->cache ) >= $this->maxCacheSize ) {
+ reset( $this->cache );
+ $key = key( $this->cache );
+ unset( $this->cache[$key] );
+ }
+ }
+
+ /**
* Check if a given path is a mwstore:// path.
* This does not do any actual validation or existence checks.
*
Modified:
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
===================================================================
---
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
2011-12-18 21:43:31 UTC (rev 106604)
+++
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
2011-12-18 21:50:41 UTC (rev 106605)
@@ -93,6 +93,10 @@
return $status; // abort
}
+ // Clear any cache entries (after locks acquired)
+ foreach ( $this->fileBackends as $backend ) {
+ $backend->clearCache();
+ }
// Actually attempt the operation batch...
$status->merge( FileOp::attemptBatch( $performOps, $opts ) );
@@ -116,7 +120,7 @@
$newOp[$par] = preg_replace(
'!^mwstore://' . preg_quote(
$this->name ) . '/!',
'mwstore://' .
$backend->getName() . '/',
- $newOp[$par]
+ $newOp[$par] // string or array
);
}
}
Modified: branches/FileBackend/phase3/maintenance/locking/LockServerDaemon.php
===================================================================
--- branches/FileBackend/phase3/maintenance/locking/LockServerDaemon.php
2011-12-18 21:43:31 UTC (rev 106604)
+++ branches/FileBackend/phase3/maintenance/locking/LockServerDaemon.php
2011-12-18 21:50:41 UTC (rev 106605)
@@ -35,11 +35,14 @@
/** @var Array */
protected $sessionIndexEx = array(); // (session => key => 1)
+ protected $address; // string (IP/hostname)
+ protected $port; // integer
protected $authKey; // string key
protected $connTimeout; // array ( 'sec' => integer, 'usec' => integer )
protected $lockTimeout; // integer number of seconds
protected $maxLocks; // integer
protected $maxClients; // integer
+ protected $maxBacklog; // integer
protected $startTime; // integer UNIX timestamp
protected $lockCount = 0; // integer
@@ -70,7 +73,10 @@
}
}
+ $this->address = $config['address'];
+ $this->port = $config['port'];
$this->authKey = $config['authKey'];
+
$connTimeout = isset( $config['connTimeout'] )
? $config['connTimeout']
: 1.5;
@@ -87,10 +93,15 @@
$this->maxClients = isset( $config['maxClients'] )
? $config['maxClients']
: 100;
- $backlog = isset( $config['maxBacklog'] )
+ $this->maxBacklog = isset( $config['maxBacklog'] )
? $config['maxBacklog']
: 10;
+ }
+ /**
+ * @return void
+ */
+ protected function setupSocket() {
if ( !function_exists( 'socket_create' ) ) {
throw new Exception( "PHP sockets extension missing
from PHP CLI mode." );
}
@@ -99,10 +110,10 @@
throw new Exception( "socket_create(): " .
socket_strerror( socket_last_error() ) );
}
socket_set_option( $sock, SOL_SOCKET, SO_REUSEADDR, 1 ); //
bypass 2MLS
- if ( socket_bind( $sock, $config['address'], $config['port'] )
=== false ) {
+ if ( socket_bind( $sock, $this->address, $this->port ) ===
false ) {
throw new Exception( "socket_bind(): " .
socket_strerror( socket_last_error( $sock ) ) );
- } elseif ( socket_listen( $sock, $backlog ) === false ) {
+ } elseif ( socket_listen( $sock, $this->maxBacklog ) === false
) {
throw new Exception( "socket_listen(): " .
socket_strerror( socket_last_error( $sock ) ) );
}
@@ -115,6 +126,8 @@
* @return void
*/
public function main() {
+ // Setup socket and start listing
+ $this->setupSocket();
// Create a list of all the clients that will be connected to
us.
$clients = array( $this->sock ); // start off with listening
socket
do {
_______________________________________________
MediaWiki-CVS mailing list
MediaWiki-CVS@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
