https://www.mediawiki.org/wiki/Special:Code/MediaWiki/105366
Revision: 105366
Author: aaron
Date: 2011-12-06 22:10:43 +0000 (Tue, 06 Dec 2011)
Log Message:
-----------
* Added type-hinting to doOperation()
* Setup a null lock manager by default
* Added FileOp::getParam() function; replaced uses of empty() with it
* In FileBackendMultiWrite:
** Removed 'isCache' code as that won't play nice with doOperations()
** Fixed $predicates handling in doOperations()
** Avoid undefined index use on $ops in doOperations()
** Cleaned up constructor input format
Modified Paths:
--------------
branches/FileBackend/phase3/includes/Setup.php
branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
branches/FileBackend/phase3/includes/filerepo/backend/FileOp.php
Modified: branches/FileBackend/phase3/includes/Setup.php
===================================================================
--- branches/FileBackend/phase3/includes/Setup.php 2011-12-06 22:05:43 UTC
(rev 105365)
+++ branches/FileBackend/phase3/includes/Setup.php 2011-12-06 22:10:43 UTC
(rev 105366)
@@ -120,8 +120,12 @@
$wgLockManagers[] = array(
'name' => 'fsLockManager',
'class' => 'FSLockManager',
- 'lockDirectory' => $wgUploadDirectory,
+ 'lockDirectory' => "{$wgUploadDirectory}/lockdir",
);
+$wgLockManagers[] = array(
+ 'name' => 'nullLockManager',
+ 'class' => 'NullLockManager',
+);
/**
* Initialise $wgLocalFileRepo from backwards-compatible settings
Modified: branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
===================================================================
--- branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
2011-12-06 22:05:43 UTC (rev 105365)
+++ branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
2011-12-06 22:10:43 UTC (rev 105366)
@@ -31,7 +31,7 @@
* $config includes:
* 'name' : The name of this backend
* 'wikiId' : Prefix to container names that is unique to this
wiki
- * 'lockManager' : The file lock manager to use
+ * 'lockManager' : Registered name of the file lock manager to use
*
* @param $config Array
*/
@@ -86,7 +86,7 @@
* @param $op Array
* @return Status
*/
- final public function doOperation( $op ) {
+ final public function doOperation( array $op ) {
return $this->doOperations( array( $op ) );
}
@@ -382,6 +382,7 @@
/**
* Return a list of FileOp objects from a list of operations.
+ * The result must have the same number of items as the input.
* An exception is thrown if an unsupported operation is requested.
*
* @param Array $ops Same format as doOperations()
@@ -400,7 +401,6 @@
// Get params for this operation
$params = $operation;
unset( $params['op'] ); // don't need this
- unset( $params['ignoreErrors'] ); // don't need
this
// Append the FileOp class
$performOps[] = new $class( $this, $params );
} else {
@@ -429,13 +429,13 @@
return $status; // abort
}
- $failedOps = array(); // failed ops with ignoreErrors
+ $failedOps = array(); // failed ops with 'ignoreErrors'
$predicates = FileOp::newPredicates(); // account for previous
op in prechecks
// Do pre-checks for each operation; abort on failure...
foreach ( $performOps as $index => $fileOp ) {
$status->merge( $fileOp->precheck( $predicates ) );
if ( !$status->isOK() ) { // operation failed?
- if ( !empty( $ops[$index]['ignoreErrors'] ) ) {
+ if ( $fileOp->getParam( 'ignoreErrors' ) ) {
$failedOps[$index] = 1; // remember not
to call attempt()/finish()
++$status->failCount;
$status->success[$index] = false;
@@ -453,7 +453,7 @@
}
$status->merge( $fileOp->attempt() );
if ( !$status->isOK() ) { // operation failed?
- if ( !empty( $ops[$index]['ignoreErrors'] ) ) {
+ if ( $fileOp->getParam( 'ignoreErrors' ) ) {
$failedOps[$index] = 1; // remember not
to call finish()
++$status->failCount;
$status->success[$index] = false;
Modified:
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
===================================================================
---
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
2011-12-06 22:05:43 UTC (rev 105365)
+++
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
2011-12-06 22:10:43 UTC (rev 105366)
@@ -14,51 +14,42 @@
* backend is read from or written to first. Functions like fileExists()
* and getFileProps() will return information based on the first backend
* that has the file. Special cases are listed below:
- * a) getFileList() will return results from the first backend that is
- * not declared as non-persistent cache. This is for correctness.
- * b) getFileTimestamp() will always check only the master backend to
+ * a) getFileTimestamp() will always check only the master backend to
* avoid confusing and inconsistent results.
- * c) getFileHash() will always check only the master backend to keep
+ * b) getFileHash() will always check only the master backend to keep
* the result format consistent.
*
* All write operations are performed on all backends.
* If an operation fails on one backend it will be rolled back from the others.
*
- * Non-persistent backends used for caching must be declared.
- *
* @ingroup FileBackend
*/
class FileBackendMultiWrite extends FileBackendBase {
/** @var Array Prioritized list of FileBackend objects */
protected $fileBackends = array(); // array of (backend index =>
backends)
- /** @var Array List of FileBackend object informations */
- protected $fileBackendsInfo = array(); // array (backend index => array
of settings)
+ protected $masterIndex = -1; // index of master backend
- protected $masterIndex; // index of master backend
-
/**
* Construct a proxy backend that consist of several internal backends.
* $config contains:
- * 'name' : The name of the proxy backend
- * 'lockManger' : LockManager instance
- * 'backends' : Array of (backend object, settings) pairs.
- * The settings per backend include:
- * 'isCache' : The backend is non-persistent
- * 'isMaster': This must be set for one
non-persistent backend.
+ * 'name' : The name of the proxy backend
+ * 'lockManager' : Registered name of the file lock manager to use
+ * 'backends' : Array of backend config and multi-backend
settings.
+ * Each value is the config used in the constructor
of a
+ * FileBackend class, but with these additional
settings:
+ * 'class' : The
name of the backend class
+ * 'isMultiMaster': This must be set for one
non-persistent backend.
* @param $config Array
*/
public function __construct( array $config ) {
parent::__construct( $config );
-
- $this->masterIndex = -1;
- foreach ( $config['backends'] as $index => $info ) {
- list( $backend, $settings ) = $info;
- $this->fileBackends[$index] = $backend;
- // Default backend settings
- $defaults = array( 'isCache' => false, 'isMaster' =>
false );
- // Apply custom backend settings to defaults
- $this->fileBackendsInfo[$index] = $info + $defaults;
- if ( $info['isMaster'] ) {
+ foreach ( $config['backends'] as $index => $config ) {
+ if ( !isset( $config['class'] ) ) {
+ throw new MWException( 'No class given for a
backend config.' );
+ }
+ $class = $config['class'];
+ $this->fileBackends[$index] = new $class( $config );
+ if ( !empty( $config['isMultiMaster'] ) ) {
if ( $this->masterIndex >= 0 ) {
throw new MWException( 'More than one
master backend defined.' );
}
@@ -73,16 +64,19 @@
final public function doOperations( array $ops ) {
$status = Status::newGood( array() );
+ $performOps = array(); // list of FileOp objects
+ $batchSize = count( $ops ); // each backend has this many ops
+ $filesToLock = array(); // storage paths to lock
// Build up a list of FileOps. The list will have all the ops
// for one backend, then all the ops for the next, and so on.
+ // These batches of ops are all part of a continuous array.
// Also build up a list of files to lock...
- $performOps = array();
- $filesToLock = array();
foreach ( $this->fileBackends as $index => $backend ) {
$performOps = array_merge( $performOps,
$backend->getOperations( $ops ) );
- // Set $filesToLock from the first backend so we don't
try to set all
- // locks two or three times (depending on the number of
backends).
if ( $index == 0 ) {
+ // Set $filesToLock from the first batch so we
don't try to set all
+ // locks two or three times over (depending on
the number of backends).
+ // A lock on one storage path is a lock on all
the backends.
foreach ( $performOps as $index => $fileOp ) {
$filesToLock = array_merge(
$filesToLock, $fileOp->storagePathsUsed() );
}
@@ -95,13 +89,18 @@
return $status; // abort
}
- $failedOps = array(); // failed ops with ignoreErrors
- $predicates = FileOp::newPredicates(); // account for previous
op in prechecks
+ $failedOps = array(); // failed ops with 'ignoreErrors'
+ $predicates = FileOp::newPredicates(); // effects of previous
ops
// Do pre-checks for each operation; abort on failure...
foreach ( $performOps as $index => $fileOp ) {
+ if ( $index > 0 && ( $index % $batchSize ) == 0 ) {
+ // We are starting the op batch for another
backend
+ // which is not effected by the other op
batches.
+ $predicates = FileOp::newPredicates();
+ }
$status->merge( $fileOp->precheck( $predicates ) );
if ( !$status->isOK() ) { // operation failed?
- if ( !empty( $ops[$index]['ignoreErrors'] ) ) {
+ if ( $fileOp->getParam( 'ignoreErrors' ) ) {
$failedOps[$index] = 1; // remember not
to call attempt()/finish()
++$status->failCount;
$status->success[$index] = false;
@@ -119,7 +118,7 @@
}
$status->merge( $fileOp->attempt() );
if ( !$status->isOK() ) { // operation failed?
- if ( !empty( $ops[$index]['ignoreErrors'] ) ) {
+ if ( $fileOp->getParam( 'ignoreErrors' ) ) {
$failedOps[$index] = 1; // remember not
to call finish()
++$status->failCount;
$status->success[$index] = false;
@@ -189,6 +188,7 @@
}
function fileExists( array $params ) {
+ # Hit all backends in case an operation failed to
copy/move/delete a file
foreach ( $this->backends as $backend ) {
if ( $backend->fileExists( $params ) ) {
return true;
@@ -246,11 +246,8 @@
function getFileList( array $params ) {
foreach ( $this->backends as $index => $backend ) {
- // Skip cache backends (like one using memcached)
- if ( !$this->fileBackendsInfo[$index]['isCache'] ) {
- return $backend->getFileList( $params );
- }
+ return $backend->getFileList( $params );
}
- return array();
+ return array(); // sanity
}
}
Modified: branches/FileBackend/phase3/includes/filerepo/backend/FileOp.php
===================================================================
--- branches/FileBackend/phase3/includes/filerepo/backend/FileOp.php
2011-12-06 22:05:43 UTC (rev 105365)
+++ branches/FileBackend/phase3/includes/filerepo/backend/FileOp.php
2011-12-06 22:10:43 UTC (rev 105366)
@@ -44,6 +44,20 @@
$this->initialize();
}
+ /*
+ * Get the value of the parameter with the given name.
+ * Returns null if the parameter is not set.
+ *
+ * @param $name string
+ * @return mixed
+ */
+ final public function getParam( $name ) {
+ if ( isset( $this->params[$name] ) ) {
+ return $this->params[$name];
+ }
+ return null;
+ }
+
/**
* Get a new empty predicates array for precheck()
*
@@ -168,7 +182,7 @@
protected function doFinish() {
return Status::newGood();
}
-
+
/**
* Backup any file at the source to a temporary file
*
@@ -201,7 +215,7 @@
protected function checkAndBackupDest() {
$status = Status::newGood();
- if ( !empty( $this->params['overwriteDest'] ) ) {
+ if ( $this->getParam( 'overwriteDest' ) ) {
// Create a temporary backup copy...
$params = array( 'src' => $this->params['dst'] );
$this->tmpDestFile = $this->backend->getLocalCopy(
$params );
@@ -209,7 +223,7 @@
$status->fatal( 'backend-fail-backup',
$this->params['dst'] );
return $status;
}
- } elseif ( !empty( $this->params['overwriteSame'] ) ) {
+ } elseif ( $this->getParam( 'overwriteSame' ) ) {
// Get the source content hash (if there is a single
source)
$shash = $this->getSourceMD5();
// If there is a single source, then we can do some
checks already.
@@ -343,7 +357,7 @@
$status = Status::newGood();
// Check if destination file exists
if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
- if ( empty( $this->params['overwriteDest'] ) ) {
+ if ( !$this->getParam( 'overwriteDest' ) ) {
$status->fatal( 'backend-fail-alreadyexists',
$this->params['dst'] );
return $status;
}
@@ -410,7 +424,7 @@
$status = Status::newGood();
// Check if destination file exists
if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
- if ( empty( $this->params['overwriteDest'] ) ) {
+ if ( !$this->getParam( 'overwriteDest' ) ) {
$status->fatal( 'backend-fail-alreadyexists',
$this->params['dst'] );
return $status;
}
@@ -472,7 +486,7 @@
$status = Status::newGood();
// Check if destination file exists
if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
- if ( empty( $this->params['overwriteDest'] ) ) {
+ if ( !$this->getParam( 'overwriteDest' ) ) {
$status->fatal( 'backend-fail-alreadyexists',
$this->params['dst'] );
return $status;
}
@@ -546,7 +560,7 @@
$status = Status::newGood();
// Check if destination file exists
if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
- if ( empty( $this->params['overwriteDest'] ) ) {
+ if ( !$this->getParam( 'overwriteDest' ) ) {
$status->fatal( 'backend-fail-alreadyexists',
$this->params['dst'] );
return $status;
}
@@ -648,7 +662,7 @@
$status = Status::newGood();
// Check if destination file exists
if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
- if ( empty( $this->params['overwriteDest'] ) ) {
+ if ( !$this->getParam( 'overwriteDest' ) ) {
$status->fatal( 'backend-fail-alreadyexists',
$this->params['dst'] );
return $status;
}
@@ -715,7 +729,7 @@
$status = Status::newGood();
// Check if the source file exists
if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
- if ( empty( $this->params['ignoreMissingSource'] ) ) {
+ if ( !$this->getParam( 'ignoreMissingSource' ) ) {
$status->fatal( 'backend-fail-notexists',
$this->params['src'] );
return $status;
}
_______________________________________________
MediaWiki-CVS mailing list
MediaWiki-CVS@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
