Krinkle has uploaded a new change for review. https://gerrit.wikimedia.org/r/206074
Change subject: [WIP] Implement git-cache-update script ...................................................................... [WIP] Implement git-cache-update script Bug: T96687 Change-Id: I5bce5c0f66e2c26d026f18e38d342fe717899d77 --- A bin/git-cache-update.php 1 file changed, 109 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/integration/jenkins refs/changes/74/206074/1 diff --git a/bin/git-cache-update.php b/bin/git-cache-update.php new file mode 100755 index 0000000..c0ebe9e --- /dev/null +++ b/bin/git-cache-update.php @@ -0,0 +1,109 @@ +#!/usr/bin/env php +<?php +/** + * Creates or updates a local cache of gerrit.wikimedia.org. + * + * Based on clone-all.php from mediawiki-tools-code-utils package. + * + * This script maintains bare clones relative to the current working directory. + * Be sure to change CWD before running this script. + * + * Copyright 2012 Platonides + * Copyright 2013 Reedy + * Copyright 2015 Krinkle + */ + +$gerritHost = 'gerrit.wikimedia.org'; +$excludes = array( + 'test*', + 'labs*', + 'wikimedia*', + 'mediawiki/packages*', + 'mediawiki/extensions', # Meta repo + 'All-Projects', # Meta repo + 'apps*', + 'search*', + 'webplatform.org*', + 'qa*', + 'glam*', +); + + +$gitServer = "https://$gerritHost/r/p";; + +// curl included with Linux or msysgit (Win) +$gerritProjectsApiUrl = "https://$gerritHost/r/projects/?d";; +$command = 'curl -H "Accept: application/json; Content-Type: application/json; charset=utf-8" -X GET ' . $gerritProjectsApiUrl; + +$totalStart = microtime( true ); +$p = popen( $command, 'r' ); +$json = stream_get_contents( $p ); +pclose( $p ); + +// Strip Gerrit's internal CSRF-protection characters that break json_decode +$json = str_replace( ")]}'", '', $json ); + +$projects = json_decode( $json ); + +foreach ( $projects as $project ) { + $repo = urldecode( $project->id ); + print "\n* $repo\n"; + + foreach ( $excludes as $exclude ) { + if ( fnmatch( $exclude, $repo, FNM_CASEFOLD ) ) { + echo "... excluded\n"; + continue 2; + } + } + + // Ensure git-clone exists + if ( !file_exists( "$repo.git" ) ) { + echo "... cloning\n"; + + $start = microtime( true ); + $exitCode = null; + passthru( "git clone --bare " . escapeshellarg( "$gitServer/$repo.git" ) . " " . escapeshellarg( "$repo.git" ), $exitCode ); + $end = microtime( true ); + echo "... completed $repo clone in ", $end - $start, " seconds.\n"; + if ( $exitCode ) { + echo "... git-clone failed.\n"; + exit( $exitCode ); + } + } else { + echo "... clone already exists\n"; + } + + // Ensure git config gc.auto is disabled so that repo is undisturbed + // when various Jenkins jobs interact with the git cache. + $exitCode = null; + passthru( "GIT_DIR='" . escapeshellarg( "$repo.git" ) . "' git config --global gc.auto 0", $exitCode ); + if ( $exitCode ) { + echo "... git-config failed.\n"; + exit( $exitCode ); + } else { + echo "... ran git-config.\n"; + } + + // Update existing clone + $exitCode = null; + passthru( "GIT_DIR='" . escapeshellarg( "$repo.git" ) . "' git fetch --prune --no-recurse-submodules", $exitCode ); + if ( $exitCode ) { + echo "... git-fetch failed.\n"; + exit( $exitCode ); + } else { + echo "... ran git-fetch.\n"; + } + + // Manually run gc now + $exitCode = null; + passthru( "GIT_DIR='" . escapeshellarg( "$repo.git" ) . "' git gc --auto", $exitCode ); + if ( $exitCode ) { + echo "... git-gc failed.\n"; + exit( $exitCode ); + } else { + echo "... ran git-gc.\n"; + } +} +$totalEnd = microtime( true ); + +echo "\nTotal time: ", $totalEnd - $totalStart, " seconds\n"; -- To view, visit https://gerrit.wikimedia.org/r/206074 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5bce5c0f66e2c26d026f18e38d342fe717899d77 Gerrit-PatchSet: 1 Gerrit-Project: integration/jenkins Gerrit-Branch: master Gerrit-Owner: Krinkle <krinklem...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
