BryanDavis has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/386763 )
Change subject: git-sync-upstream: formatting and comment tweaks
......................................................................
git-sync-upstream: formatting and comment tweaks
* Use logging for all output
* Add timestamp to log messages
* Add a few comments
* Pedantic bd808 code style changes
Change-Id: I05beeb3bd323d00b57ebcba1afcc674470531487
---
M modules/puppetmaster/files/git-sync-upstream
1 file changed, 59 insertions(+), 36 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/63/386763/1
diff --git a/modules/puppetmaster/files/git-sync-upstream
b/modules/puppetmaster/files/git-sync-upstream
index 16993fb..1b490dc 100755
--- a/modules/puppetmaster/files/git-sync-upstream
+++ b/modules/puppetmaster/files/git-sync-upstream
@@ -1,5 +1,8 @@
#!/usr/bin/python3
-
+# Fetch new git changes from upstream repos and rebase local changes on top.
+# Rebase step is done in a temporary git clone that shares objects with the
+# repo being rebased. This helps avoid consumers of the clone seeing partial
+# application of local changes do to non-atomic operations.
import datetime
import logging
import os
@@ -9,12 +12,24 @@
# Send all git output to stdout
# This has to be set before git is imported.
os.environ["GIT_PYTHON_TRACE"] = 'full'
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s %(levelname)-8s %(name)s: %(message)s',
+ datefmt='%Y-%m-%dT%H:%M:%SZ'
+)
+logging.captureWarnings(True)
+logger = logging.getLogger('sync-upstream')
import git
def rebase_repo(repo_path, track_branch):
+ """Rebase the current HEAD of the given clone on top of the given tracking
+ branch of it's origin repo.
+
+ :param repo_path: git clone to rebase
+ :param track_branch: origin branch to track
+ """
datestring = datetime.datetime.now().strftime('%Y%m%d%H%M')
branchname = "oot-branch-%s" % datestring
tagname = "snapshot-%s" % datestring
@@ -27,7 +42,7 @@
# diff index against working copy
if repo.index.diff(None):
- print("Local diffs detected. Commit your changes!")
+ logger.error("Local diffs detected. Commit your changes!")
return False
repo.remotes.origin.fetch()
@@ -38,70 +53,78 @@
latest_merged_commit = repo.git.merge_base(upstream_branch, "HEAD")
if latest_upstream_commit == latest_merged_commit:
- print("Up-to-date: %s" % repo_path)
+ logger.info("Up-to-date: %s" % repo_path)
return True
try:
- # Rebase in a tempdir to avoid changing the state of the current
workdir.
- # (Rebasing in place causes an occasional puppet race)
+ # Perform rebase in a temporary workdir to avoid altering the state of
+ # the current workdir. Rebasing in place can lead to Puppet using
+ # files after the update but before local patches are re-applied.
#
# This next bit is largely cribbed from
- # https://github.com/encukou/bin/blob/master/oot-rebase
- #
+ # https://github.com/encukou/bin/blob/master/oot-rebase
os.makedirs(tempdir)
tmprepo = git.Repo.init(tempdir)
- # This bit of magic should prevent us from needing to create a full
duplicate
- # of all the objects in repo_path
- with open(os.path.join(tempdir,".git/objects/info/alternates"), "w")
as alternates:
+ # This bit of magic should prevent us from needing to create a full
+ # duplicate of all the objects in repo_path.
+ # See: https://git-scm.com/docs/gitrepository-layout
+ alt_file = os.path.join(tempdir,".git/objects/info/alternates")
+ with open(alt_file, "w") as alternates:
alternates.write("%s/.git/objects" % repo_path)
# Get ready to rebase in tmprepo: fetch from upstream, and create and
- # check out a branch 'oot-rebase' that matches the state of the
- # main repo in repo_path:
- tmprepo.git.fetch("-n", repo_path,
- "%s:oot-rebase/%s" % (current_branch,
current_branch),
- "%s:oot-rebase/%s" % (upstream_branch,
upstream_branch))
+ # check out a branch 'oot-rebase' that matches the state of the main
+ # repo in repo_path:
+ tmprepo.git.fetch(
+ "-n", repo_path,
+ "%s:oot-rebase/%s" % (current_branch, current_branch),
+ "%s:oot-rebase/%s" % (upstream_branch, upstream_branch))
tmprepo.git.checkout("oot-rebase/%s" % current_branch)
# And... rebase.
- tmprepo.git.rebase("--preserve-merges",
- "--stat",
- "--strategy=recursive",
- "--strategy-option=patience",
- "oot-rebase/%s" % upstream_branch)
+ tmprepo.git.rebase(
+ "--preserve-merges",
+ "--stat",
+ "--strategy=recursive",
+ "--strategy-option=patience",
+ "oot-rebase/%s" % upstream_branch)
# Now 'oot-rebase' in tmprepo has the final state we need. Push that
- # branch to a temporary branch ('branchname') in the main repo.
- tmprepo.git.push("--force-with-lease=%s:%s" % (current_branch,
latest_commit),
- repo_path,
- "oot-rebase/%s:%s" % (current_branch,branchname))
+ # branch to a temporary branch ('branchname') in the main repo.
+ tmprepo.git.push(
+ "--force-with-lease=%s:%s" % (current_branch, latest_commit),
+ repo_path,
+ "oot-rebase/%s:%s" % (current_branch,branchname))
- # And reset our original repo to this new branch and discard the
'branchname' branch
+ # Finally reset our original repo to this new branch and discard the
+ # 'branchname' branch
repo.git.reset("--hard", branchname)
repo.git.branch("-D", branchname)
shutil.rmtree(tempdir)
except git.exc.GitCommandError:
- print("Rebase failed!")
+ logger.error("Rebase failed!")
shutil.rmtree(tempdir)
return False
+ # Ensure that submodules are up to date in the local clone
repo.git.submodule("update", "--init", "--recursive")
- # For the sake of future rollbacks, tag the repo in the
- # state we've just set up
+ # For the sake of future rollbacks, tag the repo in the state we've just
+ # set up
repo.create_tag(tagname)
+ logger.info("Tagged as %s" % tagname)
- print("Tagged as %s" % tagname)
-
- print("Local hacks:")
- repo.git.log("--color",
- "--pretty=oneline",
- "--abbrev-commit",
- "origin/HEAD..HEAD")
+ logger.info("Local hacks:")
+ repo.git.log(
+ "--color",
+ "--pretty=oneline",
+ "--abbrev-commit",
+ "origin/HEAD..HEAD")
return True
+
if rebase_repo("/var/lib/git/operations/puppet", "production"):
rebase_repo("/var/lib/git/labs/private", "master")
--
To view, visit https://gerrit.wikimedia.org/r/386763
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I05beeb3bd323d00b57ebcba1afcc674470531487
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <bda...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
