Gilles has submitted this change and it was merged. Change subject: Support for storing files under SHA-1 names ......................................................................
Support for storing files under SHA-1 names * Added a "storageLayout" flag to LocalRepo config (supports "sha1") * Added a simple migration script to copy files the SHA1 paths * Currently works with img_auth.php + thumb_handler.php for URLs * Added visibility to some LocalFile methods * Simple tests for the wrapper class. Co-Authored-By: Gilles Dubuc <gdu...@wikimedia.org> Change-Id: Iad46ad669c8ae3c02d10da10c3f7a16fe161663f Bug: T1210 --- M autoload.php A includes/filerepo/FileBackendDBRepoWrapper.php M includes/filerepo/FileRepo.php M includes/filerepo/ForeignDBRepo.php M includes/filerepo/ForeignDBViaLBRepo.php M includes/filerepo/LocalRepo.php M includes/filerepo/file/LocalFile.php A maintenance/migrateFileRepoLayout.php A tests/phpunit/includes/filerepo/FileBackendDBRepoWrapperTest.php A tests/phpunit/includes/filerepo/MigrateFileRepoLayoutTest.php 10 files changed, 1,076 insertions(+), 93 deletions(-) Approvals: Gilles: Looks good to me, approved jenkins-bot: Verified diff --git a/autoload.php b/autoload.php index 734aa6a..a8940c4 100644 --- a/autoload.php +++ b/autoload.php @@ -418,6 +418,7 @@ 'Field' => __DIR__ . '/includes/db/DatabaseUtility.php', 'File' => __DIR__ . '/includes/filerepo/file/File.php', 'FileBackend' => __DIR__ . '/includes/filebackend/FileBackend.php', + 'FileBackendDBRepoWrapper' => __DIR__ . '/includes/filerepo/FileBackendDBRepoWrapper.php', 'FileBackendError' => __DIR__ . '/includes/filebackend/FileBackend.php', 'FileBackendException' => __DIR__ . '/includes/filebackend/FileBackend.php', 'FileBackendGroup' => __DIR__ . '/includes/filebackend/FileBackendGroup.php', @@ -772,6 +773,7 @@ 'MessageCache' => __DIR__ . '/includes/cache/MessageCache.php', 'MessageContent' => __DIR__ . '/includes/content/MessageContent.php', 'MessageSpecifier' => __DIR__ . '/includes/libs/MessageSpecifier.php', + 'MigrateFileRepoLayout' => __DIR__ . '/maintenance/migrateFileRepoLayout.php', 'MigrateUserGroup' => __DIR__ . '/maintenance/migrateUserGroup.php', 'MimeMagic' => __DIR__ . '/includes/MimeMagic.php', 'MinifyScript' => __DIR__ . '/maintenance/minify.php', diff --git a/includes/filerepo/FileBackendDBRepoWrapper.php b/includes/filerepo/FileBackendDBRepoWrapper.php new file mode 100644 index 0000000..0401d0c --- /dev/null +++ b/includes/filerepo/FileBackendDBRepoWrapper.php @@ -0,0 +1,357 @@ +<?php +/** + * Proxy backend that manages file layout rewriting for FileRepo. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup FileRepo + * @ingroup FileBackend + * @author Aaron Schulz + */ + +/** + * @brief Proxy backend that manages file layout rewriting for FileRepo. + * + * LocalRepo may be configured to store files under their title names or by SHA-1. + * This acts as a shim in the later case, providing backwards compatability for + * most callers. All "public"/"deleted" zone files actually go in an "original" + * container and are never changed. + * + * This requires something like thumb_handler.php and img_auth.php for client viewing of files. + * + * @ingroup FileRepo + * @ingroup FileBackend + * @since 1.25 + */ +class FileBackendDBRepoWrapper extends FileBackend { + /** @var FileBackend */ + protected $backend; + /** @var string */ + protected $repoName; + /** @var Closure */ + protected $dbHandleFunc; + /** @var ProcessCacheLRU */ + protected $resolvedPathCache; + /** @var Array Map of (index => DBConnRef) */ + protected $dbs; + + public function __construct( array $config ) { + $config['name'] = $config['backend']->getName(); + $config['wikiId'] = $config['backend']->getWikiId(); + parent::__construct( $config ); + $this->backend = $config['backend']; + $this->repoName = $config['repoName']; + $this->dbHandleFunc = $config['dbHandleFactory']; + $this->resolvedPathCache = new ProcessCacheLRU( 100 ); + } + + /** + * Get the underlying FileBackend that is being wrapped + * + * @return FileBackend + */ + public function getInternalBackend() { + return $this->backend; + } + + /** + * Translate a legacy "title" path to it's "sha1" counterpart + * + * E.g. mwstore://local-backend/local-public/a/ab/<name>.jpg + * => mwstore://local-backend/local-original/x/y/z/<sha1>.jpg + * + * @param string $path + * @param bool $latest + * @return string + */ + public function getBackendPath( $path, $latest = true ) { + $paths = $this->getBackendPaths( array( $path ), $latest ); + return current( $paths ); + } + + /** + * Translate legacy "title" paths to their "sha1" counterparts + * + * E.g. mwstore://local-backend/local-public/a/ab/<name>.jpg + * => mwstore://local-backend/local-original/x/y/z/<sha1>.jpg + * + * @param array $paths + * @param bool $latest + * @return array Translated paths in same order + */ + public function getBackendPaths( array $paths, $latest = true ) { + $db = $this->getDB( $latest ? DB_MASTER : DB_SLAVE ); + $origBasePath = $this->backend->getContainerStoragePath( "{$this->repoName}-original" ); + + // @TODO: batching + $resolved = array(); + foreach ( $paths as $i => $path ) { + if ( !$latest && $this->resolvedPathCache->has( $path, 'target', 10 ) ) { + $resolved[$i] = $this->resolvedPathCache->get( $path, 'target' ); + continue; + } + + list( , $container, $rel ) = FileBackend::splitStoragePath( $path ); + + if ( $container === "{$this->repoName}-public" ) { + $name = basename( $path ); + if ( strpos( $path, '!' ) !== false ) { + $sha1 = $db->selectField( 'oldimage', 'oi_sha1', + array( 'oi_archive_name' => $name ), + __METHOD__ + ); + } else { + $sha1 = $db->selectField( 'image', 'img_sha1', + array( 'img_name' => $name ), + __METHOD__ + ); + } + if ( !strlen( $sha1 ) ) { + $resolved[$i] = $path; // give up + continue; + } + $resolved[$i] = $this->getPathForSHA1( $sha1 ); + $this->resolvedPathCache->set( $path, 'target', $resolved[$i] ); + } elseif ( $container === "{$this->repoName}-deleted" ) { + $name = basename( $path ); // <hash>.<ext> + $sha1 = substr( $name, 0, strpos( $name, '.' ) ); // ignore extension + $resolved[$i] = $this->getPathForSHA1( $sha1 ); + $this->resolvedPathCache->set( $path, 'target', $resolved[$i] ); + } else { + $resolved[$i] = $path; + } + } + + $res = array(); + foreach ( $paths as $i => $path ) { + $res[$i] = $resolved[$i]; + } + + return $res; + } + + protected function doOperationsInternal( array $ops, array $opts ) { + return $this->backend->doOperationsInternal( $this->mungeOpPaths( $ops ), $opts ); + } + + protected function doQuickOperationsInternal( array $ops ) { + return $this->backend->doQuickOperationsInternal( $this->mungeOpPaths( $ops ) ); + } + + protected function doPrepare( array $params ) { + return $this->backend->doPrepare( $params ); + } + + protected function doSecure( array $params ) { + return $this->backend->doSecure( $params ); + } + + protected function doPublish( array $params ) { + return $this->backend->doPublish( $params ); + } + + protected function doClean( array $params ) { + return $this->backend->doClean( $params ); + } + + public function concatenate( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function fileExists( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getFileTimestamp( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getFileSize( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getFileStat( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getFileXAttributes( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getFileSha1Base36( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getFileProps( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function streamFile( array $params ) { + // The stream methods use the file extension to determine the + // Content-Type (as MediaWiki should already validate it on upload). + // The translated SHA1 path has no extension, so this needs to use + // the untranslated path extension. + $type = StreamFile::contentTypeFromPath( $params['src'] ); + if ( $type && $type != 'unknown/unknown' ) { + $params['headers'][] = "Content-type: $type"; + } + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getFileContentsMulti( array $params ) { + return $this->translateArrayResults( __FUNCTION__, $params ); + } + + public function getLocalReferenceMulti( array $params ) { + return $this->translateArrayResults( __FUNCTION__, $params ); + } + + public function getLocalCopyMulti( array $params ) { + return $this->translateArrayResults( __FUNCTION__, $params ); + } + + public function getFileHttpUrl( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function directoryExists( array $params ) { + return $this->backend->directoryExists( $params ); + } + + public function getDirectoryList( array $params ) { + return $this->backend->getDirectoryList( $params ); + } + + public function getFileList( array $params ) { + return $this->backend->getFileList( $params ); + } + + public function getFeatures() { + return $this->backend->getFeatures(); + } + + public function clearCache( array $paths = null ) { + $this->backend->clearCache( null ); // clear all + } + + public function preloadCache( array $paths ) { + $paths = $this->getBackendPaths( $paths ); + $this->backend->preloadCache( $paths ); + } + + public function preloadFileStat( array $params ) { + return $this->translateSrcParams( __FUNCTION__, $params ); + } + + public function getScopedLocksForOps( array $ops, Status $status ) { + return $this->backend->getScopedFileLocks( $ops, $status ); + } + + /** + * Get the ultimate original storage path for a file + * + * Use this when putting a new file into the system + * + * @param string $sha1 File SHA-1 base36 + * @return string + */ + public function getPathForSHA1( $sha1 ) { + if ( strlen( $sha1 ) < 3 ) { + throw new MWException( "Invalid file SHA-1." ); + } + return $this->backend->getContainerStoragePath( "{$this->repoName}-original" ) . + "/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } + + /** + * Get a connection to the repo file registry DB + * + * @param integer $index + * @return DBConnRef + */ + protected function getDB( $index ) { + if ( !isset( $this->db[$index] ) ) { + $func = $this->dbHandleFunc; + $this->db[$index] = $func( $index ); + } + return $this->db[$index]; + } + + /** + * Translates paths found in the "src" or "srcs" keys of a params array + * + * @param string $function + * @param array $params + */ + protected function translateSrcParams( $function, array $params ) { + $latest = !empty( $params['latest'] ); + + if ( isset( $params['src'] ) ) { + $params['src'] = $this->getBackendPath( $params['src'], $latest ); + } + + if ( isset( $params['srcs'] ) ) { + $params['srcs'] = $this->getBackendPaths( $params['srcs'], $latest ); + } + + return $this->backend->$function( $params ); + } + + /** + * Translates paths when the backend function returns results keyed by paths + * + * @param string $function + * @param array $params + * @return array + */ + protected function translateArrayResults( $function, array $params ) { + $origPaths = $params['srcs']; + $params['srcs'] = $this->getBackendPaths( $params['srcs'], !empty( $params['latest'] ) ); + $pathMap = array_combine( $params['srcs'], $origPaths ); + + $results = $this->backend->$function( $params ); + + $contents = array(); + foreach ( $results as $path => $result ) { + $contents[$pathMap[$path]] = $result; + } + + return $contents; + } + + /** + * Translate legacy "title" source paths to their "sha1" counterparts + * + * This leaves destination paths alone since we don't want those to mutate + * + * @param array $ops + * @return array + */ + protected function mungeOpPaths( array $ops ) { + // Ops that use 'src' and do not mutate core file data there + static $srcRefOps = array( 'store', 'copy', 'describe' ); + foreach ( $ops as &$op ) { + if ( isset( $op['src'] ) && in_array( $op['op'], $srcRefOps ) ) { + $op['src'] = $this->getBackendPath( $op['src'], true ); + } + if ( isset( $op['srcs'] ) ) { + $op['srcs'] = $this->getBackendPaths( $op['srcs'], true ); + } + } + return $ops; + } +} diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php index 82bbd76..7370c5c 100644 --- a/includes/filerepo/FileRepo.php +++ b/includes/filerepo/FileRepo.php @@ -49,6 +49,9 @@ /** @var int */ public $descriptionCacheExpiry; + /** @var bool */ + protected $hasSha1Storage = false; + /** @var FileBackend */ protected $backend; @@ -1885,6 +1888,14 @@ return $ret; } + + /** + * Returns whether or not storage is SHA-1 based + * @return boolean + */ + public function hasSha1Storage() { + return $this->hasSha1Storage; + } } /** diff --git a/includes/filerepo/ForeignDBRepo.php b/includes/filerepo/ForeignDBRepo.php index 6e9e6ad..dfdb375 100644 --- a/includes/filerepo/ForeignDBRepo.php +++ b/includes/filerepo/ForeignDBRepo.php @@ -76,17 +76,8 @@ */ function getMasterDB() { if ( !isset( $this->dbConn ) ) { - $this->dbConn = DatabaseBase::factory( $this->dbType, - array( - 'host' => $this->dbServer, - 'user' => $this->dbUser, - 'password' => $this->dbPassword, - 'dbname' => $this->dbName, - 'flags' => $this->dbFlags, - 'tablePrefix' => $this->tablePrefix, - 'foreign' => true, - ) - ); + $func = $this->getDBFactory(); + $this->dbConn = $func( DB_MASTER ); } return $this->dbConn; @@ -100,6 +91,25 @@ } /** + * @return Closure + */ + protected function getDBFactory() { + return function( $index ) { + return DatabaseBase::factory( $this->dbType, + array( + 'host' => $this->dbServer, + 'user' => $this->dbUser, + 'password' => $this->dbPassword, + 'dbname' => $this->dbName, + 'flags' => $this->dbFlags, + 'tablePrefix' => $this->tablePrefix, + 'foreign' => true, + ) + ); + }; + } + + /** * @return bool */ function hasSharedCache() { diff --git a/includes/filerepo/ForeignDBViaLBRepo.php b/includes/filerepo/ForeignDBViaLBRepo.php index 8153ffb..f49b716 100644 --- a/includes/filerepo/ForeignDBViaLBRepo.php +++ b/includes/filerepo/ForeignDBViaLBRepo.php @@ -66,6 +66,16 @@ return wfGetDB( DB_SLAVE, array(), $this->wiki ); } + /** + * @return Closure + */ + protected function getDBFactory() { + $wiki = $this->wiki; + return function( $index ) use ( $wiki ) { + return wfGetDB( $index, array(), $wiki ); + }; + } + function hasSharedCache() { return $this->hasSharedCache; } diff --git a/includes/filerepo/LocalRepo.php b/includes/filerepo/LocalRepo.php index 800a230..1852912 100644 --- a/includes/filerepo/LocalRepo.php +++ b/includes/filerepo/LocalRepo.php @@ -29,6 +29,9 @@ * @ingroup FileRepo */ class LocalRepo extends FileRepo { + /** @var bool */ + protected $hasSha1Storage = false; + /** @var array */ protected $fileFactory = array( 'LocalFile', 'newFromTitle' ); @@ -46,6 +49,20 @@ /** @var array */ protected $oldFileFactoryKey = array( 'OldLocalFile', 'newFromKey' ); + + function __construct( array $info = null ) { + parent::__construct( $info ); + + $this->hasSha1Storage = isset( $info['storageLayout'] ) && $info['storageLayout'] === 'sha1'; + + if ( $this->hasSha1Storage() ) { + $this->backend = new FileBackendDBRepoWrapper( array( + 'backend' => $this->backend, + 'repoName' => $this->name, + 'dbHandleFactory' => $this->getDBFactory() + ) ); + } + } /** * @throws MWException @@ -82,6 +99,11 @@ * @return FileRepoStatus */ function cleanupDeletedBatch( array $storageKeys ) { + if ( $this->hasSha1Storage() ) { + wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" ); + return Status::newGood(); + } + $backend = $this->backend; // convenience $root = $this->getZonePath( 'deleted' ); $dbw = $this->getMasterDB(); @@ -470,6 +492,16 @@ } /** + * Get a callback to get a DB handle given an index (DB_SLAVE/DB_MASTER) + * @return Closure + */ + protected function getDBFactory() { + return function( $index ) { + return wfGetDB( $index ); + }; + } + + /** * Get a key on the primary cache for this repository. * Returns false if the repository's cache is not accessible at this site. * The parameters are the parts of the key, as for wfMemcKey(). @@ -514,4 +546,56 @@ 'favicon' => wfExpandUrl( $wgFavicon ), ) ); } + + public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) { + return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() ); + } + + public function storeBatch( array $triplets, $flags = 0 ) { + return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() ); + } + + public function cleanupBatch( array $files, $flags = 0 ) { + return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() ); + } + + public function publish( + $srcPath, + $dstRel, + $archiveRel, + $flags = 0, + array $options = array() + ) { + return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() ); + } + + public function publishBatch( array $ntuples, $flags = 0 ) { + return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() ); + } + + public function delete( $srcRel, $archiveRel ) { + return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() ); + } + + public function deleteBatch( array $sourceDestPairs ) { + return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() ); + } + + /** + * Skips the write operation if storage is sha1-based, executes it normally otherwise + * + * @param string $function + * @param array $args + * @return FileRepoStatus + */ + protected function skipWriteOperationIfSha1( $function, array $args ) { + $this->assertWritableRepo(); // fail out if read-only + + if ( $this->hasSha1Storage() ) { + wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" ); + return Status::newGood(); + } else { + return call_user_func_array('parent::' . $function, $args ); + } + } } diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php index 6abe00c..4070553 100644 --- a/includes/filerepo/file/LocalFile.php +++ b/includes/filerepo/file/LocalFile.php @@ -1459,7 +1459,7 @@ * The archive name should be passed through to recordUpload for database * registration. * - * @param string $srcPath Local filesystem path to the source image + * @param string $srcPath Local filesystem path or virtual URL to the source image * @param int $flags A bitwise combination of: * File::DELETE_SOURCE Delete the source file, i.e. move rather than copy * @param array $options Optional additional parameters @@ -1477,7 +1477,7 @@ * The archive name should be passed through to recordUpload for database * registration. * - * @param string $srcPath Local filesystem path to the source image + * @param string $srcPath Local filesystem path or virtual URL to the source image * @param string $dstRel Target relative path * @param int $flags A bitwise combination of: * File::DELETE_SOURCE Delete the source file, i.e. move rather than copy @@ -1486,7 +1486,8 @@ * archive name, or an empty string if it was a new file. */ function publishTo( $srcPath, $dstRel, $flags = 0, array $options = array() ) { - if ( $this->getRepo()->getReadOnlyReason() !== false ) { + $repo = $this->getRepo(); + if ( $repo->getReadOnlyReason() !== false ) { return $this->readOnlyFatalStatus(); } @@ -1494,13 +1495,29 @@ $archiveName = wfTimestamp( TS_MW ) . '!' . $this->getName(); $archiveRel = 'archive/' . $this->getHashPath() . $archiveName; - $flags = $flags & File::DELETE_SOURCE ? LocalRepo::DELETE_SOURCE : 0; - $status = $this->repo->publish( $srcPath, $dstRel, $archiveRel, $flags, $options ); - if ( $status->value == 'new' ) { - $status->value = ''; + if ( $repo->hasSha1Storage() ) { + $sha1 = $repo->isVirtualUrl( $srcPath ) + ? $repo->getFileSha1( $srcPath ) + : File::sha1Base36( $srcPath ); + $dst = $repo->getBackend()->getPathForSHA1( $sha1 ); + $status = $repo->quickImport( $srcPath, $dst ); + if ( $flags & File::DELETE_SOURCE ) { + unlink( $srcPath ); + } + + if ( $this->exists() ) { + $status->value = $archiveName; + } } else { - $status->value = $archiveName; + $flags = $flags & File::DELETE_SOURCE ? LocalRepo::DELETE_SOURCE : 0; + $status = $repo->publish( $srcPath, $dstRel, $archiveRel, $flags, $options ); + + if ( $status->value == 'new' ) { + $status->value = ''; + } else { + $status->value = $archiveName; + } } $this->unlock(); // done @@ -1941,14 +1958,14 @@ $this->status = $file->repo->newGood(); } - function addCurrent() { + public function addCurrent() { $this->srcRels['.'] = $this->file->getRel(); } /** * @param string $oldName */ - function addOld( $oldName ) { + public function addOld( $oldName ) { $this->srcRels[$oldName] = $this->file->getArchiveRel( $oldName ); $this->archiveUrls[] = $this->file->getArchiveUrl( $oldName ); } @@ -1957,7 +1974,7 @@ * Add the old versions of the image to the batch * @return array List of archive names from old versions */ - function addOlds() { + public function addOlds() { $archiveNames = array(); $dbw = $this->file->repo->getMasterDB(); @@ -1978,7 +1995,7 @@ /** * @return array */ - function getOldRels() { + protected function getOldRels() { if ( !isset( $this->srcRels['.'] ) ) { $oldRels =& $this->srcRels; $deleteCurrent = false; @@ -2050,7 +2067,7 @@ return $hashes; } - function doDBInserts() { + protected function doDBInserts() { $dbw = $this->file->repo->getMasterDB(); $encTimestamp = $dbw->addQuotes( $dbw->timestamp() ); $encUserId = $dbw->addQuotes( $this->user->getId() ); @@ -2165,8 +2182,8 @@ * Run the transaction * @return FileRepoStatus */ - function execute() { - + public function execute() { + $repo = $this->file->getRepo(); $this->file->lock(); // Prepare deletion batch @@ -2180,7 +2197,7 @@ if ( isset( $hashes[$name] ) ) { $hash = $hashes[$name]; $key = $hash . $dotExt; - $dstRel = $this->file->repo->getDeletedHashPath( $key ) . $key; + $dstRel = $repo->getDeletedHashPath( $key ) . $key; $this->deletionBatch[$name] = array( $srcRel, $dstRel ); } } @@ -2193,20 +2210,22 @@ // them in a separate transaction, then run the file ops, then update the fa_name fields. $this->doDBInserts(); - // Removes non-existent file from the batch, so we don't get errors. - // This also handles files in the 'deleted' zone deleted via revision deletion. - $checkStatus = $this->removeNonexistentFiles( $this->deletionBatch ); - if ( !$checkStatus->isGood() ) { - $this->status->merge( $checkStatus ); - return $this->status; - } - $this->deletionBatch = $checkStatus->value; + if ( !$repo->hasSha1Storage() ) { + // Removes non-existent file from the batch, so we don't get errors. + // This also handles files in the 'deleted' zone deleted via revision deletion. + $checkStatus = $this->removeNonexistentFiles( $this->deletionBatch ); + if ( !$checkStatus->isGood() ) { + $this->status->merge( $checkStatus ); + return $this->status; + } + $this->deletionBatch = $checkStatus->value; - // Execute the file deletion batch - $status = $this->file->repo->deleteBatch( $this->deletionBatch ); + // Execute the file deletion batch + $status = $this->file->repo->deleteBatch( $this->deletionBatch ); - if ( !$status->isGood() ) { - $this->status->merge( $status ); + if ( !$status->isGood() ) { + $this->status->merge( $status ); + } } if ( !$this->status->isOK() ) { @@ -2232,7 +2251,7 @@ * @param array $batch * @return Status */ - function removeNonexistentFiles( $batch ) { + protected function removeNonexistentFiles( $batch ) { $files = $newBatch = array(); foreach ( $batch as $batchItem ) { @@ -2293,7 +2312,7 @@ * Add a file by ID * @param int $fa_id */ - function addId( $fa_id ) { + public function addId( $fa_id ) { $this->ids[] = $fa_id; } @@ -2301,14 +2320,14 @@ * Add a whole lot of files by ID * @param int[] $ids */ - function addIds( $ids ) { + public function addIds( $ids ) { $this->ids = array_merge( $this->ids, $ids ); } /** * Add all revisions of the file */ - function addAll() { + public function addAll() { $this->all = true; } @@ -2320,12 +2339,13 @@ * So we save the batch and let the caller call cleanup() * @return FileRepoStatus */ - function execute() { + public function execute() { global $wgLang; + $repo = $this->file->getRepo(); if ( !$this->all && !$this->ids ) { // Do nothing - return $this->file->repo->newGood(); + return $repo->newGood(); } $lockOwnsTrx = $this->file->lock(); @@ -2382,9 +2402,9 @@ continue; } - $deletedRel = $this->file->repo->getDeletedHashPath( $row->fa_storage_key ) . + $deletedRel = $repo->getDeletedHashPath( $row->fa_storage_key ) . $row->fa_storage_key; - $deletedUrl = $this->file->repo->getVirtualUrl() . '/deleted/' . $deletedRel; + $deletedUrl = $repo->getVirtualUrl() . '/deleted/' . $deletedRel; if ( isset( $row->fa_sha1 ) ) { $sha1 = $row->fa_sha1; @@ -2498,27 +2518,29 @@ $status->error( 'undelete-missing-filearchive', $id ); } - // Remove missing files from batch, so we don't get errors when undeleting them - $checkStatus = $this->removeNonexistentFiles( $storeBatch ); - if ( !$checkStatus->isGood() ) { - $status->merge( $checkStatus ); - return $status; - } - $storeBatch = $checkStatus->value; + if ( !$repo->hasSha1Storage() ) { + // Remove missing files from batch, so we don't get errors when undeleting them + $checkStatus = $this->removeNonexistentFiles( $storeBatch ); + if ( !$checkStatus->isGood() ) { + $status->merge( $checkStatus ); + return $status; + } + $storeBatch = $checkStatus->value; - // Run the store batch - // Use the OVERWRITE_SAME flag to smooth over a common error - $storeStatus = $this->file->repo->storeBatch( $storeBatch, FileRepo::OVERWRITE_SAME ); - $status->merge( $storeStatus ); + // Run the store batch + // Use the OVERWRITE_SAME flag to smooth over a common error + $storeStatus = $this->file->repo->storeBatch( $storeBatch, FileRepo::OVERWRITE_SAME ); + $status->merge( $storeStatus ); - if ( !$status->isGood() ) { - // Even if some files could be copied, fail entirely as that is the - // easiest thing to do without data loss - $this->cleanupFailedBatch( $storeStatus, $storeBatch ); - $status->ok = false; - $this->file->unlock(); + if ( !$status->isGood() ) { + // Even if some files could be copied, fail entirely as that is the + // easiest thing to do without data loss + $this->cleanupFailedBatch( $storeStatus, $storeBatch ); + $status->ok = false; + $this->file->unlock(); - return $status; + return $status; + } } // Run the DB updates @@ -2542,7 +2564,7 @@ } // If store batch is empty (all files are missing), deletion is to be considered successful - if ( $status->successCount > 0 || !$storeBatch ) { + if ( $status->successCount > 0 || !$storeBatch || $repo->hasSha1Storage() ) { if ( !$exists ) { wfDebug( __METHOD__ . " restored {$status->successCount} items, creating a new current\n" ); @@ -2565,7 +2587,7 @@ * @param array $triplets * @return Status */ - function removeNonexistentFiles( $triplets ) { + protected function removeNonexistentFiles( $triplets ) { $files = $filteredTriplets = array(); foreach ( $triplets as $file ) { $files[$file[0]] = $file[0]; @@ -2591,7 +2613,7 @@ * @param array $batch * @return array */ - function removeNonexistentFromCleanup( $batch ) { + protected function removeNonexistentFromCleanup( $batch ) { $files = $newBatch = array(); $repo = $this->file->repo; @@ -2616,7 +2638,7 @@ * This should be called from outside the transaction in which execute() was called. * @return FileRepoStatus */ - function cleanup() { + public function cleanup() { if ( !$this->cleanupBatch ) { return $this->file->repo->newGood(); } @@ -2635,7 +2657,7 @@ * @param Status $storeStatus * @param array $storeBatch */ - function cleanupFailedBatch( $storeStatus, $storeBatch ) { + protected function cleanupFailedBatch( $storeStatus, $storeBatch ) { $cleanupBatch = array(); foreach ( $storeStatus->success as $i => $success ) { @@ -2693,7 +2715,7 @@ /** * Add the current image to the batch */ - function addCurrent() { + public function addCurrent() { $this->cur = array( $this->oldRel, $this->newRel ); } @@ -2701,7 +2723,7 @@ * Add the old versions of the image to the batch * @return array List of archive names from old versions */ - function addOlds() { + public function addOlds() { $archiveBase = 'archive'; $this->olds = array(); $this->oldCount = 0; @@ -2751,7 +2773,7 @@ * Perform the move. * @return FileRepoStatus */ - function execute() { + public function execute() { $repo = $this->file->repo; $status = $repo->newGood(); @@ -2782,22 +2804,26 @@ wfDebugLog( 'imagemove', "Renamed {$this->file->getName()} in database: " . "{$statusDb->successCount} successes, {$statusDb->failCount} failures" ); - // Copy the files into their new location. - // If a prior process fataled copying or cleaning up files we tolerate any - // of the existing files if they are identical to the ones being stored. - $statusMove = $repo->storeBatch( $triplets, FileRepo::OVERWRITE_SAME ); - wfDebugLog( 'imagemove', "Moved files for {$this->file->getName()}: " . - "{$statusMove->successCount} successes, {$statusMove->failCount} failures" ); - if ( !$statusMove->isGood() ) { - // Delete any files copied over (while the destination is still locked) - $this->cleanupTarget( $triplets ); - $destFile->unlock(); - $this->file->unlockAndRollback(); // unlocks the destination - wfDebugLog( 'imagemove', "Error in moving files: " . $statusMove->getWikiText() ); - $statusMove->ok = false; + if ( !$repo->hasSha1Storage() ) { + // Copy the files into their new location. + // If a prior process fataled copying or cleaning up files we tolerate any + // of the existing files if they are identical to the ones being stored. + $statusMove = $repo->storeBatch( $triplets, FileRepo::OVERWRITE_SAME ); + wfDebugLog( 'imagemove', "Moved files for {$this->file->getName()}: " . + "{$statusMove->successCount} successes, {$statusMove->failCount} failures" ); + if ( !$statusMove->isGood() ) { + // Delete any files copied over (while the destination is still locked) + $this->cleanupTarget( $triplets ); + $destFile->unlock(); + $this->file->unlockAndRollback(); // unlocks the destination + wfDebugLog( 'imagemove', "Error in moving files: " . $statusMove->getWikiText() ); + $statusMove->ok = false; - return $statusMove; + return $statusMove; + } + $status->merge( $statusMove ); } + $destFile->unlock(); $this->file->unlock(); // done @@ -2805,7 +2831,6 @@ $this->cleanupSource( $triplets ); $status->merge( $statusDb ); - $status->merge( $statusMove ); return $status; } @@ -2816,7 +2841,7 @@ * * @return FileRepoStatus */ - function doDBUpdates() { + protected function doDBUpdates() { $repo = $this->file->repo; $status = $repo->newGood(); $dbw = $this->db; @@ -2868,7 +2893,7 @@ * Generate triplets for FileRepo::storeBatch(). * @return array */ - function getMoveTriplets() { + protected function getMoveTriplets() { $moves = array_merge( array( $this->cur ), $this->olds ); $triplets = array(); // The format is: (srcUrl, destZone, destUrl) @@ -2890,7 +2915,7 @@ * @param array $triplets * @return Status */ - function removeNonexistentFiles( $triplets ) { + protected function removeNonexistentFiles( $triplets ) { $files = array(); foreach ( $triplets as $file ) { @@ -2920,7 +2945,7 @@ * files. Called if something went wrong half way. * @param array $triplets */ - function cleanupTarget( $triplets ) { + protected function cleanupTarget( $triplets ) { // Create dest pairs from the triplets $pairs = array(); foreach ( $triplets as $triplet ) { @@ -2936,7 +2961,7 @@ * Called at the end of the move process if everything else went ok. * @param array $triplets */ - function cleanupSource( $triplets ) { + protected function cleanupSource( $triplets ) { // Create source file names from the triplets $files = array(); foreach ( $triplets as $triplet ) { diff --git a/maintenance/migrateFileRepoLayout.php b/maintenance/migrateFileRepoLayout.php new file mode 100644 index 0000000..78587ce --- /dev/null +++ b/maintenance/migrateFileRepoLayout.php @@ -0,0 +1,232 @@ +<?php +/** + * Copy all files in FileRepo to an originals container using SHA1 paths. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Copy all files in FileRepo to an originals container using SHA1 paths. + * + * This script should be run while the repo is still set to the old layout. + * + * @ingroup Maintenance + */ +class MigrateFileRepoLayout extends Maintenance { + public function __construct() { + parent::__construct(); + $this->mDescription = "Copy files in repo to a different layout."; + $this->addOption( 'oldlayout', "Old layout; one of 'name' or 'sha1'", true, true ); + $this->addOption( 'newlayout', "New layout; one of 'name' or 'sha1'", true, true ); + $this->addOption( 'since', "Copy only files from after this timestamp", false, true ); + $this->setBatchSize( 50 ); + } + + public function execute() { + $oldLayout = $this->getOption( 'oldlayout' ); + if ( !in_array( $oldLayout, array( 'name', 'sha1' ) ) ) { + $this->error( "Invalid old layout.", 1 ); + } + $newLayout = $this->getOption( 'newlayout' ); + if ( !in_array( $newLayout, array( 'name', 'sha1' ) ) ) { + $this->error( "Invalid new layout.", 1 ); + } + $since = $this->getOption( 'since' ); + + $repo = $this->getRepo(); + + $be = $repo->getBackend(); + if ( $be instanceof FileBackendDBRepoWrapper ) { + $be = $be->getInternalBackend(); // avoid path translations for this script + } + + $dbw = $repo->getMasterDB(); + + $origBase = $be->getContainerStoragePath( "{$repo->getName()}-original" ); + $startTime = wfTimestampNow(); + + // Do current and archived versions... + $conds = array(); + if ( $since ) { + $conds[] = 'img_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) ); + } + + $batch = array(); + $lastName = ''; + do { + $res = $dbw->select( 'image', array( 'img_name', 'img_sha1' ), + array_merge( array( 'img_name > ' . $dbw->addQuotes( $lastName ) ), $conds ), + __METHOD__, + array( 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'img_name' ) + ); + + foreach ( $res as $row ) { + $lastName = $row->img_name; + $sha1 = $row->img_sha1; + if ( !strlen( $sha1 ) ) { + $this->error( "Image SHA-1 not set for {$row->img_name}." ); + } else { + $file = $repo->newFile( $row->img_name ); + + if ( $oldLayout === 'sha1' ) { + $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $spath = $file->getPath(); + } + + if ( $newLayout === 'sha1' ) { + $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $dpath = $file->getPath(); + } + + $status = $be->prepare( array( 'dir' => dirname( $dpath ) ) ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrorsArray(), true ) ); + } + + $batch[] = array( 'op' => 'copy', 'overwrite' => true, + 'src' => $spath, 'dst' => $dpath, 'img' => $row->img_name ); + } + + foreach ( $file->getHistory() as $ofile ) { + $sha1 = $ofile->getSha1(); + if ( !strlen( $sha1 ) ) { + $this->error( "Image SHA-1 not set for {$ofile->getArchiveName()}." ); + continue; + } + + if ( $oldLayout === 'sha1' ) { + $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } elseif ( $ofile->isDeleted( File::DELETED_FILE ) ) { + $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) . + '/' . $repo->getDeletedHashPath( $sha1 ) . + $sha1 . '.' . $ofile->getExtension(); + } else { + $spath = $ofile->getPath(); + } + + if ( $newLayout === 'sha1' ) { + $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $dpath = $ofile->getPath(); + } + + $status = $be->prepare( array( 'dir' => dirname( $dpath ) ) ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrorsArray(), true ) ); + } + $batch[] = array( 'op' => 'copy', 'overwrite' => true, + 'src' => $spath, 'dst' => $dpath, 'img' => $ofile->getArchiveName() ); + } + + if ( count( $batch ) >= $this->mBatchSize ) { + $this->runBatch( $batch, $be ); + $batch = array(); + } + } + } while ( $res->numRows() ); + + if ( count( $batch ) ) { + $this->runBatch( $batch, $be ); + } + + // Do deleted versions... + $conds = array(); + if ( $since ) { + $conds[] = 'fa_deleted_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) ); + } + + $batch = array(); + $lastId = 0; + do { + $res = $dbw->select( 'filearchive', array( 'fa_storage_key', 'fa_id', 'fa_name' ), + array_merge( array( 'fa_id > ' . $dbw->addQuotes( $lastId ) ), $conds ), + __METHOD__, + array( 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'fa_id' ) + ); + + foreach ( $res as $row ) { + $lastId = $row->fa_id; + $sha1Key = $row->fa_storage_key; + if ( !strlen( $sha1Key ) ) { + $this->error( "Image SHA-1 not set for file #{$row->fa_id} (deleted)." ); + continue; + } + $sha1 = substr( $sha1Key, 0, strpos( $sha1Key, '.' ) ); + + if ( $oldLayout === 'sha1' ) { + $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) . + '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key; + } + + if ( $newLayout === 'sha1' ) { + $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $dpath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) . + '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key; + } + + $status = $be->prepare( array( 'dir' => dirname( $dpath ) ) ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrorsArray(), true ) ); + } + + $batch[] = array( 'op' => 'copy', 'src' => $spath, 'dst' => $dpath, + 'overwriteSame' => true, 'img' => "(ID {$row->fa_id}) {$row->fa_name}" ); + + if ( count( $batch ) >= $this->mBatchSize ) { + $this->runBatch( $batch, $be ); + $batch = array(); + } + } + } while ( $res->numRows() ); + + if ( count( $batch ) ) { + $this->runBatch( $batch, $be ); + } + + $this->output( "Done (started $startTime)\n" ); + } + + protected function getRepo() { + return RepoGroup::singleton()->getLocalRepo(); + } + + protected function runBatch( array $ops, FileBackend $be ) { + $this->output( "Migrating file batch:\n" ); + foreach ( $ops as $op ) { + $this->output( "\"{$op['img']}\" (dest: {$op['dst']})\n" ); + } + + $status = $be->doOperations( $ops ); + if ( !$status->isOK() ) { + $this->output( print_r( $status->getErrorsArray(), true ) ); + } + + $this->output( "Batch done\n\n" ); + } +} + +$maintClass = 'MigrateFileRepoLayout'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/tests/phpunit/includes/filerepo/FileBackendDBRepoWrapperTest.php b/tests/phpunit/includes/filerepo/FileBackendDBRepoWrapperTest.php new file mode 100644 index 0000000..681e368 --- /dev/null +++ b/tests/phpunit/includes/filerepo/FileBackendDBRepoWrapperTest.php @@ -0,0 +1,138 @@ +<?php + +class FileBackendDBRepoWrapperTest extends MediaWikiTestCase { + protected $backendName = 'foo-backend'; + protected $repoName = 'pureTestRepo'; + + /** + * @dataProvider getBackendPathsProvider + * @covers FileBackendDBRepoWrapper::getBackendPaths + */ + public function testGetBackendPaths( + $mocks, + $latest, + $dbReadsExpected, + $dbReturnValue, + $originalPath, + $expectedBackendPath, + $message ) { + list( $dbMock, $backendMock, $wrapperMock ) = $mocks; + + $dbMock->expects( $dbReadsExpected ) + ->method( 'selectField' ) + ->will( $this->returnValue( $dbReturnValue ) ); + + $newPaths = $wrapperMock->getBackendPaths( array( $originalPath ), $latest ); + + $this->assertEquals( + $expectedBackendPath, + $newPaths[0], + $message ); + } + + public function getBackendPathsProvider() { + $prefix = 'mwstore://' . $this->backendName . '/' . $this->repoName; + $mocksForCaching = $this->getMocks(); + + return array( + array( + $mocksForCaching, + false, + $this->once(), + '96246614d75ba1703bdfd5d7660bb57407aaf5d9', + $prefix . '-public/f/o/foobar.jpg', + $prefix . '-original/9/6/2/96246614d75ba1703bdfd5d7660bb57407aaf5d9', + 'Public path translated correctly', + ), + array( + $mocksForCaching, + false, + $this->never(), + '96246614d75ba1703bdfd5d7660bb57407aaf5d9', + $prefix . '-public/f/o/foobar.jpg', + $prefix . '-original/9/6/2/96246614d75ba1703bdfd5d7660bb57407aaf5d9', + 'LRU cache leveraged', + ), + array( + $this->getMocks(), + true, + $this->once(), + '96246614d75ba1703bdfd5d7660bb57407aaf5d9', + $prefix . '-public/f/o/foobar.jpg', + $prefix . '-original/9/6/2/96246614d75ba1703bdfd5d7660bb57407aaf5d9', + 'Latest obtained', + ), + array( + $this->getMocks(), + true, + $this->never(), + '96246614d75ba1703bdfd5d7660bb57407aaf5d9', + $prefix . '-deleted/f/o/foobar.jpg', + $prefix . '-original/f/o/o/foobar', + 'Deleted path translated correctly', + ), + array( + $this->getMocks(), + true, + $this->once(), + null, + $prefix . '-public/b/a/baz.jpg', + $prefix . '-public/b/a/baz.jpg', + 'Path left untouched if no sha1 can be found', + ), + ); + } + + /** + * @covers FileBackendDBRepoWrapper::getFileContentsMulti + */ + public function testGetFileContentsMulti() { + list( $dbMock, $backendMock, $wrapperMock ) = $this->getMocks(); + + $sha1Path = 'mwstore://' . $this->backendName . '/' . $this->repoName + . '-original/9/6/2/96246614d75ba1703bdfd5d7660bb57407aaf5d9'; + $filenamePath = 'mwstore://' . $this->backendName . '/' . $this->repoName + . '-public/f/o/foobar.jpg'; + + $dbMock->expects( $this->once() ) + ->method( 'selectField' ) + ->will( $this->returnValue( '96246614d75ba1703bdfd5d7660bb57407aaf5d9' ) ); + + $backendMock->expects( $this->once() ) + ->method( 'getFileContentsMulti') + ->will( $this->returnValue( array( $sha1Path => 'foo' ) ) ); + + $result = $wrapperMock->getFileContentsMulti( array( 'srcs' => array( $filenamePath ) ) ); + + $this->assertEquals( + array( $filenamePath => 'foo' ), + $result, + 'File contents paths translated properly' + ); + } + + protected function getMocks() { + $dbMock = $this->getMockBuilder( 'DatabaseMysql' ) + ->disableOriginalConstructor() + ->getMock(); + + $backendMock = $this->getMock( 'FSFileBackend', + array(), + array( array( + 'name' => $this->backendName, + 'wikiId' => wfWikiId() + ) ) ); + + $wrapperMock = $this->getMock( 'FileBackendDBRepoWrapper', + array( 'getDB' ), + array( array( + 'backend' => $backendMock, + 'repoName' => $this->repoName, + 'dbHandleFactory' => null + ) ) ); + + $wrapperMock->expects( $this->any() )->method( 'getDB' )->will( $this->returnValue( $dbMock ) ); + + return array( $dbMock, $backendMock, $wrapperMock ); + } +} diff --git a/tests/phpunit/includes/filerepo/MigrateFileRepoLayoutTest.php b/tests/phpunit/includes/filerepo/MigrateFileRepoLayoutTest.php new file mode 100644 index 0000000..65db7e4 --- /dev/null +++ b/tests/phpunit/includes/filerepo/MigrateFileRepoLayoutTest.php @@ -0,0 +1,114 @@ +<?php + +class MigrateFileRepoLayoutTest extends MediaWikiTestCase { + protected $tmpPrefix; + protected $migratorMock; + protected $tmpFilepath; + protected $text = 'testing'; + + protected function setUp() { + parent::setUp(); + + $filename = 'Foo.png'; + + $this->tmpPrefix = wfTempDir() . '/migratefilelayout-test-' . time() . '-' . mt_rand(); + + $backend = new FSFileBackend( array( + 'name' => 'local-migratefilerepolayouttest', + 'wikiId' => wfWikiID(), + 'containerPaths' => array( + 'migratefilerepolayouttest-original' => "{$this->tmpPrefix}-original", + 'migratefilerepolayouttest-public' => "{$this->tmpPrefix}-public", + 'migratefilerepolayouttest-thumb' => "{$this->tmpPrefix}-thumb", + 'migratefilerepolayouttest-temp' => "{$this->tmpPrefix}-temp", + 'migratefilerepolayouttest-deleted' => "{$this->tmpPrefix}-deleted", + ) + ) ); + + $dbMock = $this->getMockBuilder( 'DatabaseMysql' ) + ->disableOriginalConstructor() + ->getMock(); + + $imageRow = new stdClass; + $imageRow->img_name = $filename; + $imageRow->img_sha1 = sha1( $this->text ); + + $dbMock->expects( $this->any() ) + ->method( 'select' ) + ->will( $this->onConsecutiveCalls( + new FakeResultWrapper( array( $imageRow ) ), // image + new FakeResultWrapper( array() ), // image + new FakeResultWrapper( array() ) // filearchive + ) ); + + $repoMock = $this->getMock( 'LocalRepo', + array( 'getMasterDB' ), + array( array( + 'name' => 'migratefilerepolayouttest', + 'backend' => $backend + ) ) ); + + $repoMock->expects( $this->any() )->method( 'getMasterDB' )->will( $this->returnValue( $dbMock ) ); + + $this->migratorMock = $this->getMock( 'MigrateFileRepoLayout', array( 'getRepo' ) ); + $this->migratorMock->expects( $this->any() )->method( 'getRepo' )->will( $this->returnValue( $repoMock ) ); + + $this->tmpFilepath = TempFSFile::factory( 'migratefilelayout-test-', 'png' )->getPath(); + + file_put_contents( $this->tmpFilepath, $this->text ); + + $hashPath = $repoMock->getHashPath( $filename ); + + $status = $repoMock->store( $this->tmpFilepath, 'public', $hashPath . $filename, FileRepo::OVERWRITE ); + } + + protected function deleteFilesRecursively( $directory ) { + foreach ( glob( $directory . '/*' ) as $file ) { + if ( is_dir( $file ) ) { + $this->deleteFilesRecursively( $file ); + } else { + unlink( $file ); + } + } + + rmdir( $directory ); + } + + protected function tearDown() { + foreach ( glob( $this->tmpPrefix . '*' ) as $directory ) { + $this->deleteFilesRecursively( $directory ); + } + + unlink( $this->tmpFilepath ); + + parent::tearDown(); + } + + public function testMigration() { + $this->migratorMock->loadParamsAndArgs( null, array( 'oldlayout' => 'name', 'newlayout' => 'sha1' ) ); + + ob_start(); + + $this->migratorMock->execute(); + + ob_end_clean(); + + $sha1 = sha1( $this->text ); + + $expectedOriginalFilepath = $this->tmpPrefix + . '-original/' + . substr( $sha1, 0, 1 ) + . '/' + . substr( $sha1, 1, 1 ) + . '/' + . substr( $sha1, 2, 1 ) + . '/' + . $sha1 ; + + $this->assertEquals( file_get_contents( $expectedOriginalFilepath ), $this->text, 'New sha1 file should be exist and have the right contents' ); + + $expectedPublicFilepath = $this->tmpPrefix . '-public/f/f8/Foo.png'; + + $this->assertEquals( file_get_contents( $expectedPublicFilepath ), $this->text, 'Existing name file should still and have the right contents' ); + } +} -- To view, visit https://gerrit.wikimedia.org/r/127460 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iad46ad669c8ae3c02d10da10c3f7a16fe161663f Gerrit-PatchSet: 30 Gerrit-Project: mediawiki/core Gerrit-Branch: master Gerrit-Owner: Aaron Schulz <asch...@wikimedia.org> Gerrit-Reviewer: Aaron Schulz <asch...@wikimedia.org> Gerrit-Reviewer: Brian Wolff <bawolff...@gmail.com> Gerrit-Reviewer: Gergő Tisza <gti...@wikimedia.org> Gerrit-Reviewer: Gilles <gdu...@wikimedia.org> Gerrit-Reviewer: Marco <mai...@live.de> Gerrit-Reviewer: Nemo bis <federicol...@tiscali.it> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits