https://www.mediawiki.org/wiki/Special:Code/MediaWiki/105413
Revision: 105413
Author: aaron
Date: 2011-12-07 03:28:55 +0000 (Wed, 07 Dec 2011)
Log Message:
-----------
* Merge backend statuses in FileBackendMultiWrite::streamFile()
* Redid FSFileBackend::getFileList() using RecursiveDirectoryIterator from SPL
* Added more unit tests (including one for getFileList)
* Use null lock manger for parser tests to keep things simple
Modified Paths:
--------------
branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
branches/FileBackend/phase3/tests/phpunit/includes/filerepo/FileBackendTest.php
branches/FileBackend/phase3/tests/phpunit/includes/parser/NewParserTest.php
Modified:
branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
===================================================================
--- branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
2011-12-07 02:52:08 UTC (rev 105412)
+++ branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
2011-12-07 03:28:55 UTC (rev 105413)
@@ -453,7 +453,7 @@
if ( $dir === null ) { // invalid storage path
return array(); // empty result
}
- return new FSFileIterator( $dir );
+ return new RecursiveIteratorIterator( new
RecursiveDirectoryIterator( $dir ) );
}
function getLocalReference( array $params ) {
@@ -507,151 +507,3 @@
return $ok;
}
}
-
-/**
- * Semi-DFS based file browsing iterator. The highest number of file handles
- * open at any given time is proportional to the height of the directory tree.
- */
-class FSFileIterator implements Iterator {
- private $directory; // starting directory
- private $itemStart = 0;
-
- private $position = 0;
- private $currentFile = false;
- private $dirStack = array(); // array of (dir name, dir handle) tuples
-
- private $loaded = false;
-
- /**
- * Get an iterator to list the files under $directory and its
subdirectories
- * @param $directory string
- */
- public function __construct( $directory ) {
- // Removing any trailing slash
- if ( substr( $this->directory, -1 ) === '/' ) {
- $this->directory = substr( $this->directory, 0, -1 );
- }
- $this->directory = realpath( $directory );
- // Remove internal base directory and trailing slash from
results
- $this->itemStart = strlen( $this->directory ) + 1;
- }
-
- private function load() {
- if ( !$this->loaded ) {
- $this->loaded = true;
- // If we get an invalid directory then the result is an
empty list
- if ( is_dir( $this->directory ) ) {
- $handle = opendir( $this->directory );
- if ( $handle ) {
- $this->pushDirectory( $this->directory,
$handle );
- $this->currentFile = $this->nextFile();
- }
- }
- }
- }
-
- function rewind() {
- $this->closeDirectories();
- $this->position = 0;
- $this->currentFile = false;
- $this->loaded = false;
- }
-
- function current() {
- $this->load();
- // Remove internal base directory and trailing slash from
results
- return substr( $this->currentFile, $this->itemStart );
- }
-
- function key() {
- $this->load();
- return $this->position;
- }
-
- function next() {
- $this->load();
- ++$this->position;
- $this->currentFile = $this->nextFile();
- }
-
- function valid() {
- $this->load();
- return ( $this->currentFile !== false );
- }
-
- function nextFile() {
- if ( !$this->currentDirectory() ) {
- return false; // nothing else to scan
- }
- # Next file under the current directory (and subdirectories).
- # This may advance the current directory down to a descendent.
- # The current directory is set to the parent if nothing is
found.
- $nextFile = $this->nextFileBelowCurrent();
- if ( $nextFile !== false ) {
- return $nextFile;
- } else {
- # Scan the higher directories
- return $this->nextFile();
- }
- }
-
- function nextFileBelowCurrent() {
- list( $dir, $handle ) = $this->currentDirectory();
- while ( false !== ( $file = readdir( $handle ) ) ) {
- // Exclude '.' and '..' as well .svn or .lock type files
- if ( $file[0] !== '.' ) {
- $path = "{$dir}/{$file}";
- // If the first thing we find is a file, then
return it.
- // If the first thing we find is a directory,
then return
- // the first file that it contains (via
recursion).
- // We exclude symlink dirs in order to avoid
cycles.
- if ( is_dir( $path ) && !is_link( $path ) ) {
- $subHandle = opendir( $path );
- if ( $subHandle ) {
- $this->pushDirectory( $path,
$subHandle );
- $nextFile =
$this->nextFileBelowCurrent();
- if ( $nextFile !== false ) {
- return $nextFile; //
found the next one!
- }
- }
- } elseif ( is_file( $path ) ) {
- return $path; // found the next one!
- }
- }
- }
- # If we didn't find anything else in this directory,
- # then back out so we scan the other higher directories
- $this->popDirectory();
- return false;
- }
-
- private function currentDirectory() {
- if ( !count( $this->dirStack ) ) {
- return false;
- }
- return $this->dirStack[count( $this->dirStack ) - 1];
- }
-
- private function popDirectory() {
- if ( count( $this->dirStack ) ) {
- list( $dir, $handle ) = array_pop( $this->dirStack );
- closedir( $handle );
- }
- }
-
- private function pushDirectory( $dir, $handle ) {
- $this->dirStack[] = array( $dir, $handle );
- }
-
- private function closeDirectories() {
- foreach ( $this->dirStack as $set ) {
- list( $dir, $handle ) = $set;
- closedir( $handle );
- }
- $this->dirStack = array();
- }
-
- function __destruct() {
- $this->closeDirectories();
- }
-}
Modified:
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
===================================================================
---
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
2011-12-07 02:52:08 UTC (rev 105412)
+++
branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
2011-12-07 03:28:55 UTC (rev 105413)
@@ -188,7 +188,7 @@
}
function fileExists( array $params ) {
- # Hit all backends in case an operation failed to
copy/move/delete a file
+ # Hit all backends in case of failed operations (out of sync)
foreach ( $this->backends as $backend ) {
if ( $backend->fileExists( $params ) ) {
return true;
@@ -213,6 +213,7 @@
}
function getFileProps( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
foreach ( $this->backends as $backend ) {
$props = $backend->getFileProps( $params );
if ( $props !== null ) {
@@ -223,18 +224,23 @@
}
function streamFile( array $params ) {
+ $status = Status::newGood();
foreach ( $this->backends as $backend ) {
- $status = $backend->streamFile( $params );
- if ( $status->isOK() ) {
+ $subStatus = $backend->streamFile( $params );
+ $status->merge( $subStatus );
+ if ( $subStatus->isOK() ) {
+ // Pass isOK() despite fatals from other
backends
+ $status->setResult( true );
return $status;
} elseif ( headers_sent() ) {
return $status; // died mid-stream...so this is
already fubar
}
}
- return Status::newFatal( 'backend-fail-stream', $params['src']
);
+ return $status;
}
function getLocalReference( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
foreach ( $this->backends as $backend ) {
$fsFile = $backend->getLocalReference( $params );
if ( $fsFile ) {
@@ -245,6 +251,7 @@
}
function getLocalCopy( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
foreach ( $this->backends as $backend ) {
$tmpFile = $backend->getLocalCopy( $params );
if ( $tmpFile ) {
@@ -256,6 +263,7 @@
function getFileList( array $params ) {
foreach ( $this->backends as $index => $backend ) {
+ # Get results from the first backend
return $backend->getFileList( $params );
}
return array(); // sanity
Modified:
branches/FileBackend/phase3/tests/phpunit/includes/filerepo/FileBackendTest.php
===================================================================
---
branches/FileBackend/phase3/tests/phpunit/includes/filerepo/FileBackendTest.php
2011-12-07 02:52:08 UTC (rev 105412)
+++
branches/FileBackend/phase3/tests/phpunit/includes/filerepo/FileBackendTest.php
2011-12-07 03:28:55 UTC (rev 105413)
@@ -1,7 +1,8 @@
<?php
class FileBackendTest extends MediaWikiTestCase {
- private $backend, $filesToPrune, $pathsToPrune;
+ private $backend, $multiBackend;
+ private $filesToPrune, $pathsToPrune;
function setUp() {
parent::setUp();
@@ -12,11 +13,34 @@
'cont1' => wfTempDir() . '/localtesting/cont1',
'cont2' => wfTempDir() . '/localtesting/cont2' )
) );
- $this->filesToPrune = array();
- $this->pathsToPrune = array();
+ $this->multiBackend = new FileBackendMultiWrite( array(
+ 'name' => 'localtestingmulti',
+ 'lockManager' => 'fsLockManager',
+ 'backends' => array(
+ array(
+ 'name' => 'localmutlitesting1',
+ 'class' => 'FSFileBackend',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array(
+ 'cont1' => wfTempDir() .
'/localtestingmulti1/cont1',
+ 'cont2' => wfTempDir() .
'/localtestingmulti1/cont2' ),
+ 'isMultiMaster' => false
+ ),
+ array(
+ 'name' => 'localmutlitesting2',
+ 'class' => 'FSFileBackend',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array(
+ 'cont1' => wfTempDir() .
'/localtestingmulti2/cont1',
+ 'cont2' => wfTempDir() .
'/localtestingmulti2/cont2' ),
+ 'isMultiMaster' => true
+ )
+ )
+ ) );
+ $this->filesToPrune = $this->pathsToPrune = array();
}
- private function basePath() {
+ private function singleBasePath() {
return 'mwstore://localtesting';
}
@@ -49,7 +73,7 @@
$cases = array();
$tmpName = TempFSFile::factory( "unittests_", 'txt'
)->getPath();
- $toPath = $this->basePath() . '/cont1/fun/obj1.txt';
+ $toPath = $this->singleBasePath() . '/cont1/fun/obj1.txt';
$op = array( 'op' => 'store', 'src' => $tmpName, 'dst' =>
$toPath );
$cases[] = array(
$op, // operation
@@ -70,36 +94,241 @@
/**
* @dataProvider provider_testCopy
*/
- public function testCopy() {
-
+ public function testCopy( $op, $source, $dest ) {
+ $this->pathsToPrune[] = $source;
+ $this->pathsToPrune[] = $dest;
+
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => 'blahblah', 'dst'
=> $source ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file
at $source succeeded." );
+
+ $status = $this->backend->doOperation( $op );
+ $this->assertEquals( true, $status->isOK(),
+ "Copy from $source to $dest succeeded." );
+ $this->assertEquals( true, $status->isGood(),
+ "Copy from $source to $dest succeeded without
warnings." );
+ $this->assertEquals( true, $this->backend->fileExists( array(
'src' => $source ) ),
+ "Source file $source still exists." );
+ $this->assertEquals( true, $this->backend->fileExists( array(
'src' => $dest ) ),
+ "Destination file $dest exists after copy." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $source
) );
+ $props2 = $this->backend->getFileProps( array( 'src' => $dest )
);
+ $this->assertEquals( $props1, $props2,
+ "Source and destination have the same props." );
}
+ public function provider_testCopy() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont1/file.txt';
+ $dest = $this->singleBasePath() . '/cont2/fileMoved.txt';
+
+ $op = array( 'op' => 'copy', 'src' => $source, 'dst' => $dest );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ return $cases;
+ }
+
/**
* @dataProvider provider_testMove
*/
- public function testMove() {
-
+ public function testMove( $op, $source, $dest ) {
+ $this->pathsToPrune[] = $source;
+ $this->pathsToPrune[] = $dest;
+
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => 'blahblah', 'dst'
=> $source ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file
at $source succeeded." );
+
+ $status = $this->backend->doOperation( $op );
+ $this->assertEquals( true, $status->isOK(),
+ "Move from $source to $dest succeeded." );
+ $this->assertEquals( true, $status->isGood(),
+ "Move from $source to $dest succeeded without
warnings." );
+ $this->assertEquals( false, $this->backend->fileExists( array(
'src' => $source ) ),
+ "Source file $source does not still exists." );
+ $this->assertEquals( true, $this->backend->fileExists( array(
'src' => $dest ) ),
+ "Destination file $dest exists after move." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $source
) );
+ $props2 = $this->backend->getFileProps( array( 'src' => $dest )
);
+ $this->assertEquals( false, $props1['fileExists'],
+ "Source file does not exist accourding to props." );
+ $this->assertEquals( true, $props2['fileExists'],
+ "Destination file exists accourding to props." );
}
+ public function provider_testMove() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont1/file.txt';
+ $dest = $this->singleBasePath() . '/cont2/fileMoved.txt';
+
+ $op = array( 'op' => 'move', 'src' => $source, 'dst' => $dest );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ return $cases;
+ }
+
/**
* @dataProvider provider_testDelete
*/
- public function testDelete() {
-
+ public function testDelete( $op, $source, $withSource, $okStatus ) {
+ $this->pathsToPrune[] = $source;
+
+ if ( $withSource ) {
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' =>
'blahblah', 'dst' => $source ) );
+ $this->assertEquals( true, $status->isOK(), "Creation
of file at $source succeeded." );
+ }
+
+ $status = $this->backend->doOperation( $op );
+ if ( $okStatus ) {
+ $this->assertEquals( true, $status->isOK(), "Deletion
of file at $source succeeded." );
+ } else {
+ $this->assertEquals( false, $status->isOK(), "Deletion
of file at $source failed." );
+ }
+
+ $this->assertEquals( false, $this->backend->fileExists( array(
'src' => $source ) ),
+ "Source file $source does not exist after move." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $source
) );
+ $this->assertEquals( false, $props1['fileExists'],
+ "Source file $source does not exist according to
props." );
}
+ public function provider_testDelete() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont1/myfacefile.txt';
+
+ $op = array( 'op' => 'delete', 'src' => $source );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ true, // with source
+ true // succeeds
+ );
+
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ false, // without source
+ false // fails
+ );
+
+ $op['ignoreMissingSource'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ false, // without source
+ true // succeeds
+ );
+
+ return $cases;
+ }
+
/**
* @dataProvider provider_testCreate
*/
- public function testCreate() {
-
+ public function testCreate( $op, $source, $alreadyExists, $okStatus,
$newSize ) {
+ $this->pathsToPrune[] = $source;
+
+ $oldText = 'blah...blah...waahwaah';
+ if ( $alreadyExists ) {
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => $oldText,
'dst' => $source ) );
+ $this->assertEquals( true, $status->isOK(), "Creation
of file at $source succeeded." );
+ }
+
+ $status = $this->backend->doOperation( $op );
+ if ( $okStatus ) {
+ $this->assertEquals( true, $status->isOK(), "Creation
of file at $source succeeded." );
+ } else {
+ $this->assertEquals( false, $status->isOK(), "Creation
of file at $source failed." );
+ }
+
+ $this->assertEquals( true, $this->backend->fileExists( array(
'src' => $source ) ),
+ "Source file $source exists after creation." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $source
) );
+ $this->assertEquals( true, $props1['fileExists'],
+ "Source file $source exists according to props." );
+ if ( $okStatus ) { // file content is what we saved
+ $this->assertEquals( $newSize, $props1['size'],
+ "Source file $source has expected size
according to props." );
+ } else { // file content is some other previous text
+ $this->assertEquals( strlen( $oldText ),
$props1['size'],
+ "Source file $source has different size that
given text according to props." );
+ }
}
/**
* @dataProvider provider_testConcatenate
*/
+ public function provider_testCreate() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont2/myspacefile.txt';
+
+ $dummyText = 'hey hey';
+ $op = array( 'op' => 'create', 'content' => $dummyText, 'dst'
=> $source );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ false, // no dest already exists
+ true, // succeeds
+ strlen( $dummyText )
+ );
+
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ true, // dest already exists
+ false, // fails
+ strlen( $dummyText )
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ true, // dest already exists
+ true, // succeeds
+ strlen( $dummyText )
+ );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testConcatenate
+ */
public function testConcatenate() {
-
+
}
/**
@@ -130,16 +359,87 @@
}
+ /**
+ * @dataProvider provider_testDoOperations
+ */
+ public function testDoOperations() {
+
+ }
+
+ public function testGetFileList() {
+ $base = $this->singleBasePath();
+ $files = array(
+ "$base/cont1/test1.txt",
+ "$base/cont1/test2.txt",
+ "$base/cont1/test3.txt",
+ "$base/cont1/subdir1/test1.txt",
+ "$base/cont1/subdir1/test2.txt",
+ "$base/cont1/subdir2/test3.txt",
+ "$base/cont1/subdir2/test4.txt",
+ "$base/cont1/subdir2/subdir/test1.txt",
+ "$base/cont1/subdir2/subdir/test2.txt",
+ "$base/cont1/subdir2/subdir/test3.txt",
+ "$base/cont1/subdir2/subdir/test4.txt",
+ "$base/cont1/subdir2/subdir/test5.txt",
+ "$base/cont1/subdir2/subdir/sub/test0.txt",
+ );
+ $this->pathsToPrune = array_merge( $this->pathsToPrune, $files
);
+
+ // Add the files
+ $ops = array();
+ foreach ( $files as $file ) {
+ $ops[] = array( 'op' => 'create', 'content' => 'xxy',
'dst' => $file );
+ }
+ $status = $this->backend->doOperations( $ops );
+ $this->assertEquals( true, $status->isOK(), "Creation of files
succeeded." );
+
+ // Expected listing
+ $expected = array(
+ "test1.txt",
+ "test2.txt",
+ "test3.txt",
+ "subdir1/test1.txt",
+ "subdir1/test2.txt",
+ "subdir2/test3.txt",
+ "subdir2/test1.txt",
+ "subdir2/subdir/test1.txt",
+ "subdir2/subdir/test2.txt",
+ "subdir2/subdir/test3.txt",
+ "subdir2/subdir/test4.txt",
+ "subdir2/subdir/test5.txt",
+ "subdir2/subdir/sub/test0.txt",
+ );
+ $expected = sort( $expected );
+
+ // Actual listing (no trailing slash)
+ $list = array();
+ $iter = $this->backend->getFileList( array( 'dir' =>
"$base/cont1" ) );
+ foreach ( $iter as $file ) {
+ $list[] = $file;
+ }
+
+ $this->assertEquals( $expected, sort( $list ), "Correct file
listing." );
+
+ // Actual listing (with trailing slash)
+ $list = array();
+ $iter = $this->backend->getFileList( array( 'dir' =>
"$base/cont1/" ) );
+ foreach ( $iter as $file ) {
+ $list[] = $file;
+ }
+
+ $this->assertEquals( $expected, sort( $list ), "Correct file
listing." );
+ }
+
function tearDown() {
parent::tearDown();
foreach ( $this->filesToPrune as $file ) {
@unlink( $file );
}
foreach ( $this->pathsToPrune as $file ) {
- $this->backend->delete( array( 'src' => $file ) );
+ $this->backend->doOperation( array( 'op' => 'delete',
'src' => $file ) );
+ $this->multiBackend->doOperation( array( 'op' =>
'delete', 'src' => $file ) );
}
- $this->backend = null;
- $this->filesToPrune = array();
- $this->pathsToPrune = array();
+ $this->backend = $this->multiBackend = null;
+ $this->filesToPrune = $this->pathsToPrune = array();
}
}
Modified:
branches/FileBackend/phase3/tests/phpunit/includes/parser/NewParserTest.php
===================================================================
--- branches/FileBackend/phase3/tests/phpunit/includes/parser/NewParserTest.php
2011-12-07 02:52:08 UTC (rev 105412)
+++ branches/FileBackend/phase3/tests/phpunit/includes/parser/NewParserTest.php
2011-12-07 03:28:55 UTC (rev 105413)
@@ -317,7 +317,7 @@
$backend = array(
'name' => 'local-backend',
'class' => 'FSFileBackend',
- 'lockManager' => 'fsLockManager',
+ 'lockManager' => 'nullLockManager',
'containerPaths' => array(
'images-public' => $this->uploadDir,
'images-thumb' => $this->uploadDir . '/thumb' )
_______________________________________________
MediaWiki-CVS mailing list
MediaWiki-CVS@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
