Nikerabbit has uploaded a new change for review. https://gerrit.wikimedia.org/r/217944
Change subject: Clean up to repository scripts ...................................................................... Clean up to repository scripts Lots of cleanups in here, might contain traces of bugs. Bug: T87004 Change-Id: I67f475ca46c89609e3ac645ee4519bdca82849af --- A bin/clupdate-gerrit-repo A bin/clupdate-git-repo M bin/repocommit M bin/repocreate M bin/repoupdate M bin/update-reset-repo 6 files changed, 92 insertions(+), 220 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/translatewiki refs/changes/44/217944/1 diff --git a/bin/clupdate-gerrit-repo b/bin/clupdate-gerrit-repo new file mode 120000 index 0000000..184ea4f --- /dev/null +++ b/bin/clupdate-gerrit-repo @@ -0,0 +1 @@ +clupdate-git-repo \ No newline at end of file diff --git a/bin/clupdate-git-repo b/bin/clupdate-git-repo new file mode 100755 index 0000000..eca1039 --- /dev/null +++ b/bin/clupdate-git-repo @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +set -u + +DIRSCRIPT="`dirname \"$0\"`" +DIRSCRIPT="`( cd \"$DIRSCRIPT\" && pwd )`" + +ME=`basename "$0"` + +URL=$1 +DIR=$2 +BRANCH=${3:-master} + +if [ -d "$DIR/.git" ] +then + "$DIRSCRIPT/update-reset-repo" "$DIR" "" "$BRANCH" +else + git clone "$URL" "$DIR" -b "$BRANCH" + + if [ "$ME" = 'clupdate-gerrit-repo' ] + then + cd "$DIR" + git config user.name 'Translation updater bot' + git config user.email 'l10n-...@translatewiki.net' + git config gitreview.username 'l10n-bot' + + git review -r origin -s + fi +fi diff --git a/bin/repocommit b/bin/repocommit index 0891df5..4ddec10 100755 --- a/bin/repocommit +++ b/bin/repocommit @@ -9,37 +9,22 @@ DIR=$2 : ${DIR:=`pwd`} source ${DIRSCRIPT}/findexportroot -cd "$DIR" echo "$(date --rfc-3339=seconds --utc) [$(whoami) at $DIR] $0 $@" >> ${DIRLOG} COMMITMSG="Localisation updates from https://translatewiki.net." - gitCommit() { - local dir=$1 - cd "$dir" + cd "$1" git add . git commit -m "$COMMITMSG" - - # Check if $2 is set. If not, push to origin master - if [[ -z "$2" ]] - then - git push origin master - else - git push origin $2 - fi - - cd .. + git push origin "${2:-master}" } gitCommitGerrit() { - local dir=$1 - cd $PROJECT + cd $1 git add . - git commit -q -m "$COMMITMSG" && git fetch -q gerrit && git review -t l10n || : - mergeL10n-bot - cd .. + git commit -q -m "$COMMITMSG" && git review -r origin -t l10n || : } mergeL10n-bot() { @@ -74,16 +59,16 @@ for i in $GITPROJECTS; do if [ "$i" = "$PROJECT" ] then - gitCommit "$PROJECT" + gitCommit "$DIR/$PROJECT" if [ "$PROJECT" = "blockly" ] then - gitCommit blockly-games + gitCommit "$DIR/blockly-games" fi if [ "$PROJECT" = "osm" ] then - gitCommit potlatch2 + gitCommit "$DIR/potlatch2" fi exit 0 @@ -100,18 +85,19 @@ for i in $GERRITPROJECTS; do if [ "$i" = "$PROJECT" ] then - gitCommitGerrit "$PROJECT" + gitCommitGerrit "$DIR/$PROJECT" + mergeL10n-bot exit 0 fi done if [ "$PROJECT" = "etherpad-lite" ] then - gitCommit "$PROJECT" develop + gitCommit "$DIR/$PROJECT" develop elif [ "$PROJECT" = "fudforum" ] then - cd $PROJECT + cd "$DIR/$PROJECT" svn add -q --force * for i in `find . -name msg` do @@ -119,22 +105,19 @@ svn propset -q svn:eol-style native $i done svn commit --message "$COMMITMSG" - cd .. elif [ "$PROJECT" = "ihris" ] then - cd $PROJECT for MODULE in $REPO_IHRIS_MODULES do - cd $MODULE + cd "$DIR/$PROJECT/$MODULE" bzr add . bzr commit -m "$COMMITMSG" - cd .. done elif [ "$PROJECT" = "inaturalist" ] then - gitCommit "$PROJECT" i18n + gitCommit "$DIR/$PROJECT" i18n elif [ "$PROJECT" = "mediawiki" ] then @@ -142,67 +125,48 @@ then echo "Add REPO_MEDIAWIKI_BRANCHES to REPOCONF"; exit 1 fi - cd $PROJECT for BRANCH in $REPO_MEDIAWIKI_BRANCHES do - cd "$BRANCH" - git add . - git commit -m "$COMMITMSG" && git fetch gerrit && git review -t l10n || : - cd .. + gitCommitGerrit "$DIR/$PROJECT/$BRANCH" done - cd .. elif [ "$PROJECT" = "mediawiki-extensions" ] then - cd $PROJECT/extensions - for i in `ls` + for i in `ls "$DIR/$PROJECT/extensions"` do echo "--> $i" - cd $i - git add . - git commit -m "$COMMITMSG" && git fetch gerrit && git review -t l10n || : - cd .. + gitCommitGerrit "$DIR/$PROJECT/extensions/$i" done - - cd ../.. mergeL10n-bot elif [ "$PROJECT" = "mediawiki-skins" ] then - cd $PROJECT - for i in `ls` + for i in `ls $DIR/$PROJECT` do echo "--> $i" - cd $i - git add . - git commit -m "$COMMITMSG" && git fetch gerrit && git review -t l10n || : - cd .. + gitCommitGerrit "$DIR/$PROJECT/$i" done - - cd .. mergeL10n-bot elif [ "$PROJECT" = "mifos" ] then - gitCommit "$PROJECT" develop + gitCommit "$DIR/$PROJECT" develop elif [ "$PROJECT" = "nocc" ] then - cd $PROJECT + cd "$DIR/$PROJECT" svn add --force * --auto-props --parents --depth infinity -q svn commit --message "$COMMITMSG" - cd .. elif [ "$PROJECT" = "openimages" ] then - cd $PROJECT + cd "$DIR/$PROJECT" svn add -q --force *.properties svn propset -q svn:mime-type text/plain *.properties svn propset -q svn:eol-style native *.properties svn commit --message "$COMMITMSG" - cd .. else echo "`basename $0`: Unknown project" diff --git a/bin/repocreate b/bin/repocreate index ffc2bdd..6ff1773 100755 --- a/bin/repocreate +++ b/bin/repocreate @@ -13,17 +13,11 @@ echo "$(date --rfc-3339=seconds --utc) [$(whoami) at $DIR] $0 $@" >> ${DIRLOG} -gitCreateGerrit() { - if [ "$REPO_RW" = "yes" ] - then - pushd "$1" - git config user.name 'Translation updater bot' - git config user.email 'l10n-...@translatewiki.net' - git config gitreview.username 'l10n-bot' - git review -s - popd - fi -} + +if [ "${REPO_RW:-no}" = "yes" ] +then CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-gerrit-repo"; +else CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-git-repo"; +fi checkVar() { if [ -z "${!1}" ] @@ -131,11 +125,9 @@ checkVar 'REPO_MEDIAWIKI' checkVar 'REPO_MEDIAWIKI_BRANCHES' mkdir -p $PROJECT - cd $PROJECT for BRANCH in $REPO_MEDIAWIKI_BRANCHES do - git clone "$REPO_MEDIAWIKI" "$BRANCH" -b "$BRANCH" - gitCreateGerrit $BRANCH + "$CLUPDATE_GERRIT" "$REPO_MEDIAWIKI" "$DIR/$PROJECT/$BRANCH" $BRANCH done elif [ "$PROJECT" = "mediawiki-extensions" ] @@ -176,8 +168,7 @@ elif [ "$PROJECT" = "pywikibot" ] then checkVar 'REPO_PYWIKIBOT' - git clone "$REPO_PYWIKIBOT" $PROJECT - gitCreateGerrit $PROJECT + "$CLUPDATE_GERRIT" "$REPO_PYWIKIBOT" "$DIR/$PROJECT" elif [ "$PROJECT" = "vicuna" ] then @@ -197,20 +188,17 @@ elif [ "$PROJECT" = "wikimania" ] then checkVar 'REPO_WIKIMANIA' - git clone "$REPO_WIKIMANIA" $PROJECT - gitCreateGerrit $PROJECT + "$CLUPDATE_GERRIT" "$REPO_WIKIMANIA" "$DIR/$PROJECT" elif [ "$PROJECT" = "wikipedia-android" ] then checkVar 'REPO_WIKIPEDIAANDROID' - git clone "$REPO_WIKIPEDIAANDROID" $PROJECT - gitCreateGerrit $PROJECT + "$CLUPDATE_GERRIT" "$REPO_WIKIPEDIAANDROID" "$DIR/$PROJECT" elif [ "$PROJECT" = "wikipedia-ios" ] then checkVar 'REPO_WIKIPEDIAIOS' - git clone "$REPO_WIKIPEDIAIOS" $PROJECT - gitCreateGerrit $PROJECT + "$CLUPDATE_GERRIT" "$REPO_WIKIPEDIAIOS" "$DIR/$PROJECT" elif [ "$PROJECT" = "WikisourceMobile" ] then diff --git a/bin/repoupdate b/bin/repoupdate index d43958e..2d59e05 100755 --- a/bin/repoupdate +++ b/bin/repoupdate @@ -15,61 +15,29 @@ echo "$(date --rfc-3339=seconds --utc) [$(whoami) at $DIR] $0 $@" >> $DIRLOG gitupdate() { - local dir=$1 - cd $dir - git fetch -q --all - if [ -z "$2" ] - then - git reset -q --hard origin/master - else - git reset -q --hard origin/$2 - fi - git clean -q -f -d - git pull -q - cd .. + "$DIRSCRIPT/update-reset-repo" "$DIR" "$1" "$2" } -cloneGerritRepo() { - local url=$1 - local name=$2 - - git clone "$url" "$name" - - if [ "$REPO_RW" = "yes" ] +processGroups() { + if [ -z "$REPO_RW" ] then - cd "$name" - - git config user.name 'Translation updater bot' - git config user.email 'l10n-...@translatewiki.net' - git config gitreview.username 'l10n-bot' - - if [ ! -e ".gitreview" ] - then - git remote add -f gerrit "$url" - fi - - git review -s - - cd .. + sudo -u betawiki php "$WIKI/extensions/Translate/scripts/processMessageChanges.php" --quiet --group="$1" && + echo "Please check https://translatewiki.net/wiki/Special:ManageMessageGroups" fi } -# Ugly. Eventually, I think we should end up with a file similar to -# mediawiki-extensions.txt where we can simply add a line like this: -# repo=https://gerrit.wikimedia.org/r/p/oojs/ui.git -# and then things magically happen based on the other configuration -# settings. -processHacks() { - if [ ! -d "oojs-ui/.git" ] - then - cloneGerritRepo "https://gerrit.wikimedia.org/r/p/oojs/ui.git" "oojs-ui" - fi - - if [ ! -d "VisualEditorVisualEditor/.git" ] - then - cloneGerritRepo "https://gerrit.wikimedia.org/r/p/VisualEditor/VisualEditor.git" "VisualEditorVisualEditor" - fi +fetchReposFromGerrit() { + curl -s "https://gerrit.wikimedia.org/r/projects/?p=$1" | + grep "$1" | + cut -d '"' -f2 | + cut -d '/' -f3 | + sort } + +if [ "${REPO_RW:-no}" = "yes" ] +then CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-gerrit-repo"; +else CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-git-repo"; +fi GITUPDATEPROJECTS="\ eol \ @@ -141,36 +109,8 @@ then echo "Add REPO_MEDIAWIKI_BRANCHES to REPOCONF"; exit 1 fi - cd $PROJECT - for BRANCH in $REPO_MEDIAWIKI_BRANCHES - do - if [ ! -d "$BRANCH" ] - then - echo "$BRANCH not yet present. Cloning..." - git clone "$REPO_MEDIAWIKI" "$BRANCH" -b "$BRANCH" - if [ "$REPO_RW" = "yes" ] - then - cd "$BRANCH" - git config user.name 'Translation updater bot' - git config user.email 'l10n-...@translatewiki.net' - git config gitreview.username 'l10n-bot' - git review -s - fi - else - cd "$BRANCH" - git fetch -q --all - git reset -q --hard "origin/$BRANCH" - git clean -q -f -d - fi - cd .. - done - cd .. - - wait - if [ "$DIR" = "/resources/projects" ] - then - sudo -u betawiki php "$WIKI/extensions/Translate/scripts/createMessageIndex.php" --quiet || ${DIRSCRIPT}/udpcast Message index creation failed & - fi + echo "$REPO_MEDIAWIKI_BRANCHES" | tr ' ' '\n' | + xargs -P4 I___ "$CLUPDATE_GERRIT" "$REPO_MEDIAWIKI" "$DIR/$PROJECT/___" "___" elif [ "$PROJECT" = "mediawiki-extensions" ] then @@ -178,34 +118,14 @@ then echo "Add REPO_MWEXTGIT to REPOCONF"; exit 1 fi - cd $PROJECT - cd extensions + echo "$(fetchReposFromGerrit mediawiki/extensions/)" | + xargs -P4 -I___ "$CLUPDATE_GERRIT" "$REPO_MWEXTGIT/___.git" "$DIR/$PROJECT/extensions/___" - EXTENSIONS=$(curl -s https://gerrit.wikimedia.org/r/projects/?p=mediawiki/extensions/ |grep 'mediawiki/extensions/' |cut -d '"' -f 2 |cut -d '/' -f3 | sort) + "$CLUPDATE_GERRIT" "https://gerrit.wikimedia.org/r/p/oojs/ui.git" "$DIR/$PROJECT/extensions/oojs-ui" & + "$CLUPDATE_GERRIT" "https://gerrit.wikimedia.org/r/p/VisualEditor/VisualEditor.git" "$DIR/$PROJECT/extensions/VisualEditorVisualEditor" & + wait - # Clone extension repos that do not exist yet. - for EXTENSION in $EXTENSIONS - do - if [ ! -d "$EXTENSION/.git" ] - then - echo "Cloning extension $EXTENSION" - cloneGerritRepo "$REPO_MWEXTGIT/$EXTENSION.git" "$EXTENSION" - fi - done - - # This is where I disappoint Niklas a bit more... - processHacks - EXTENSIONS="$EXTENSIONS VisualEditorVisualEditor oojs-ui" - - # Update all repos (yes, there is duplication for repos that were just cloned) - echo $EXTENSIONS | - xargs -n1 -P7 ${DIRSCRIPT}/update-reset-repo "$DIR/$PROJECT/extensions" - - if [ -z "$REPO_RW" ] - then - sudo -u betawiki php "$WIKI/extensions/Translate/scripts/processMessageChanges.php" --quiet --group=ext-* && echo "Please check https://translatewiki.net/wiki/Special:ManageMessageGroups" - sudo -u betawiki php "$WIKI/extensions/Translate/scripts/createMessageIndex.php" --quiet || ${DIRSCRIPT}/udpcast Message index creation failed & - fi + processGroups "ext-*" elif [ "$PROJECT" = "mediawiki-skins" ] then @@ -213,29 +133,10 @@ then echo "Add REPO_MWSKINGIT to REPOCONF"; exit 1 fi - cd $PROJECT + echo "$(fetchReposFromGerrit mediawiki/skins/)" | + xargs -P4 -I___ "$CLUPDATE_GERRIT" "$REPO_MWSKINGIT/___.git" "$DIR/$PROJECT/___" - SKINS=$(curl -s https://gerrit.wikimedia.org/r/projects/?p=mediawiki/skins/ |grep 'mediawiki/skins/' |cut -d '"' -f 2 |cut -d '/' -f3 |sort) - - # Clone skin repos that do not exist yet. - for SKIN in $SKINS - do - if [ ! -d "$SKIN/.git" ] - then - echo "Cloning skin $SKIN" - cloneGerritRepo "$REPO_MWSKINGIT/$SKIN.git" "$SKIN" - fi - done - - # Update all repos (yes, there is duplication for repos that were just cloned) - echo $SKINS | - xargs -n1 -P7 ${DIRSCRIPT}/update-reset-repo "$DIR/$PROJECT" - - if [ -z "$REPO_RW" ] - then - sudo -u betawiki php "$WIKI/extensions/Translate/scripts/processMessageChanges.php" --quiet --group=mediawiki-skin-* && echo "Please check https://translatewiki.net/wiki/Special:ManageMessageGroups" - sudo -u betawiki php "$WIKI/extensions/Translate/scripts/createMessageIndex.php" --quiet || ${DIRSCRIPT}/udpcast Message index creation failed & - fi + processGroups "mediawiki-skin-*" elif [ "$PROJECT" = "nocc" ] then diff --git a/bin/update-reset-repo b/bin/update-reset-repo index dafb734..900fb56 100755 --- a/bin/update-reset-repo +++ b/bin/update-reset-repo @@ -1,4 +1,5 @@ #!/bin/bash +set -e # Allow for passing a base path and repo folder name in two parameters # or a single full path as parameter. @@ -10,26 +11,14 @@ DIR=$1 fi +BRANCH=${3:-master} + # Updates the repository in the current directory and resets it. -# Does not retry in case of errors. # Sets OUTPUT to the output of the git operations # Sets CODE to the return value of the pipeline. update_reset() { - OUTPUT="$(git ls-remote --exit-code origin refs/heads/master)" + OUTPUT=$(git fetch -q && git reset -q --hard origin/$BRANCH && git clean -q -f -d 2>&1 >&-) CODE=$? - - case "$CODE" in - 0) - # Actual fetching, resetting, ... - OUTPUT=$(git fetch -q --all && git reset -q --hard origin/master && git clean -q -f -d 2>&1 >&-) - CODE=$? - ;; - 2) - # remote does not have refs/heads/master - # No need to try fetching, resetting, ... - CODE=0 - ;; - esac } cd $DIR -- To view, visit https://gerrit.wikimedia.org/r/217944 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I67f475ca46c89609e3ac645ee4519bdca82849af Gerrit-PatchSet: 1 Gerrit-Project: translatewiki Gerrit-Branch: master Gerrit-Owner: Nikerabbit <niklas.laxst...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits