[PATCH 2 of 3] vfs: extract 'vfs' class and related code to a new 'vfs' module (API)

[Pierre-Yves David]

# HG changeset patch
# User Pierre-Yves David 
# Date 1488362412 -3600
#  Wed Mar 01 11:00:12 2017 +0100
# Node ID 69ef1663d5fe2a3bf7bd4138b98ae373b1848591
# Parent  a588ab7f18eccd4bf734416d8d1fbf2858ace67c
# EXP-Topic vfs.cleanup
# Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/
#  hg pull 
https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r 69ef1663d5fe
vfs: extract 'vfs' class and related code to a new 'vfs' module (API)

The 'scmutil' is growing large (1500+ lines) and 2/5 of it is related to vfs.
We extract the 'vfs' related code in its own module get both module back to a
better scale and clearer contents.

We keep all the references available in 'scmutil' for now as many reference
needs to be updated.

diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py
--- a/mercurial/scmutil.py
+++ b/mercurial/scmutil.py
@@ -7,17 +7,12 @@
 
 from __future__ import absolute_import
 
-import contextlib
 import errno
 import glob
 import hashlib
 import os
 import re
-import shutil
 import socket
-import stat
-import tempfile
-import threading
 
 from .i18n import _
 from .node import wdirrev
@@ -32,6 +27,7 @@ from . import (
 revsetlang,
 similar,
 util,
+vfs as vfsmod,
 )
 
 if pycompat.osname == 'nt':
@@ -336,455 +332,16 @@ def filteredhash(repo, maxrev):
 key = s.digest()
 return key
 
-class abstractvfs(object):
-"""Abstract base class; cannot be instantiated"""
-
-def __init__(self, *args, **kwargs):
-'''Prevent instantiation; don't call this from subclasses.'''
-raise NotImplementedError('attempted instantiating ' + str(type(self)))
-
-def tryread(self, path):
-'''gracefully return an empty string for missing files'''
-try:
-return self.read(path)
-except IOError as inst:
-if inst.errno != errno.ENOENT:
-raise
-return ""
-
-def tryreadlines(self, path, mode='rb'):
-'''gracefully return an empty array for missing files'''
-try:
-return self.readlines(path, mode=mode)
-except IOError as inst:
-if inst.errno != errno.ENOENT:
-raise
-return []
-
-@util.propertycache
-def open(self):
-'''Open ``path`` file, which is relative to vfs root.
-
-Newly created directories are marked as "not to be indexed by
-the content indexing service", if ``notindexed`` is specified
-for "write" mode access.
-'''
-return self.__call__
-
-def read(self, path):
-with self(path, 'rb') as fp:
-return fp.read()
-
-def readlines(self, path, mode='rb'):
-with self(path, mode=mode) as fp:
-return fp.readlines()
-
-def write(self, path, data, backgroundclose=False):
-with self(path, 'wb', backgroundclose=backgroundclose) as fp:
-return fp.write(data)
-
-def writelines(self, path, data, mode='wb', notindexed=False):
-with self(path, mode=mode, notindexed=notindexed) as fp:
-return fp.writelines(data)
-
-def append(self, path, data):
-with self(path, 'ab') as fp:
-return fp.write(data)
-
-def basename(self, path):
-"""return base element of a path (as os.path.basename would do)
-
-This exists to allow handling of strange encoding if needed."""
-return os.path.basename(path)
-
-def chmod(self, path, mode):
-return os.chmod(self.join(path), mode)
-
-def dirname(self, path):
-"""return dirname element of a path (as os.path.dirname would do)
-
-This exists to allow handling of strange encoding if needed."""
-return os.path.dirname(path)
-
-def exists(self, path=None):
-return os.path.exists(self.join(path))
-
-def fstat(self, fp):
-return util.fstat(fp)
-
-def isdir(self, path=None):
-return os.path.isdir(self.join(path))
-
-def isfile(self, path=None):
-return os.path.isfile(self.join(path))
-
-def islink(self, path=None):
-return os.path.islink(self.join(path))
-
-def isfileorlink(self, path=None):
-'''return whether path is a regular file or a symlink
-
-Unlike isfile, this doesn't follow symlinks.'''
-try:
-st = self.lstat(path)
-except OSError:
-return False
-mode = st.st_mode
-return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
-
-def reljoin(self, *paths):
-"""join various elements of a path together (as os.path.join would do)
-
-The vfs base is not injected so that path stay relative. This exists
-to allow handling of strange encoding if needed."""
-return os.path.join(*paths)
-
-def split(self, path):
-"""split top-most element of a path (as os.path.split would do)
-
-This exists to allow handling of strange encoding if needed."""
-return os.path.spli

[PATCH 1 of 3] vfs: replace 'scmutil.opener' usage with 'scmutil.vfs'

[Pierre-Yves David]

# HG changeset patch
# User Pierre-Yves David 
# Date 1488423156 -3600
#  Thu Mar 02 03:52:36 2017 +0100
# Node ID a588ab7f18eccd4bf734416d8d1fbf2858ace67c
# Parent  c32f9eeec75445bfbbc55df4c1fcc584d3cf45cd
# EXP-Topic vfs.cleanup
# Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/
#  hg pull 
https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r a588ab7f18ec
vfs: replace 'scmutil.opener' usage with 'scmutil.vfs'

The 'vfs' class is the first class citizen for years. We remove all usages of
the older API. This will let us remove the old API eventually.

diff --git a/contrib/undumprevlog b/contrib/undumprevlog
--- a/contrib/undumprevlog
+++ b/contrib/undumprevlog
@@ -17,7 +17,7 @@ from mercurial import (
 for fp in (sys.stdin, sys.stdout, sys.stderr):
 util.setbinary(fp)
 
-opener = scmutil.opener('.', False)
+opener = scmutil.vfs('.', False)
 tr = transaction.transaction(sys.stderr.write, opener, {'store': opener},
  "undump.journal")
 while True:
diff --git a/hgext/convert/subversion.py b/hgext/convert/subversion.py
--- a/hgext/convert/subversion.py
+++ b/hgext/convert/subversion.py
@@ -1146,8 +1146,8 @@ class svn_sink(converter_sink, commandli
 self.run0('checkout', path, wcpath)
 
 self.wc = wcpath
-self.opener = scmutil.opener(self.wc)
-self.wopener = scmutil.opener(self.wc)
+self.opener = scmutil.vfs(self.wc)
+self.wopener = scmutil.vfs(self.wc)
 self.childmap = mapfile(ui, self.join('hg-childmap'))
 if util.checkexec(self.wc):
 self.is_exec = util.isexec
diff --git a/hgext/largefiles/lfutil.py b/hgext/largefiles/lfutil.py
--- a/hgext/largefiles/lfutil.py
+++ b/hgext/largefiles/lfutil.py
@@ -144,7 +144,7 @@ def openlfdirstate(ui, repo, create=True
 '''
 vfs = repo.vfs
 lfstoredir = longname
-opener = scmutil.opener(vfs.join(lfstoredir))
+opener = scmutil.vfs(vfs.join(lfstoredir))
 lfdirstate = largefilesdirstate(opener, ui, repo.root,
  repo.dirstate._validate)
 
diff --git a/hgext/mq.py b/hgext/mq.py
--- a/hgext/mq.py
+++ b/hgext/mq.py
@@ -434,7 +434,7 @@ class queue(object):
 except IOError:
 curpath = os.path.join(path, 'patches')
 self.path = patchdir or curpath
-self.opener = scmutil.opener(self.path)
+self.opener = scmutil.vfs(self.path)
 self.ui = ui
 self.baseui = baseui
 self.applieddirty = False
diff --git a/hgext/transplant.py b/hgext/transplant.py
--- a/hgext/transplant.py
+++ b/hgext/transplant.py
@@ -60,7 +60,7 @@ class transplants(object):
 self.opener = opener
 
 if not opener:
-self.opener = scmutil.opener(self.path)
+self.opener = scmutil.vfs(self.path)
 self.transplants = {}
 self.dirty = False
 self.read()
@@ -103,7 +103,7 @@ class transplanter(object):
 def __init__(self, ui, repo, opts):
 self.ui = ui
 self.path = repo.join('transplant')
-self.opener = scmutil.opener(self.path)
+self.opener = scmutil.vfs(self.path)
 self.transplants = transplants(self.path, 'transplants',
opener=self.opener)
 def getcommiteditor():
diff --git a/mercurial/archival.py b/mercurial/archival.py
--- a/mercurial/archival.py
+++ b/mercurial/archival.py
@@ -249,7 +249,7 @@ class fileit(object):
 
 def __init__(self, name, mtime):
 self.basedir = name
-self.opener = scmutil.opener(self.basedir)
+self.opener = scmutil.vfs(self.basedir)
 
 def addfile(self, name, mode, islink, data):
 if islink:
diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py
--- a/mercurial/cmdutil.py
+++ b/mercurial/cmdutil.py
@@ -583,7 +583,7 @@ def openrevlog(repo, cmd, file_, opts):
 raise error.CommandError(cmd, _('invalid arguments'))
 if not os.path.isfile(file_):
 raise error.Abort(_("revlog '%s' not found") % file_)
-r = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False),
+r = revlog.revlog(scmutil.vfs(pycompat.getcwd(), audit=False),
   file_[:-2] + ".i")
 return r
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -76,7 +76,7 @@ def debugancestor(ui, repo, *args):
 """find the ancestor revision of two revisions in a given index"""
 if len(args) == 3:
 index, rev1, rev2 = args
-r = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False), 
index)
+r = revlog.revlog(scmutil.vfs(pycompat.getcwd(), audit=False), index)
 lookup = r.lookup
 elif len(args) == 2:
 if not repo:
@@ -452,7 +452,7 @@ def debugdag(ui, repo, file_=None, *revs
 spaces = opts.get('spaces')
 dots = opts.get('dots')
 if file_:
-

[PATCH 3 of 3] vfs: use 'vfs' module directly in 'mercurial.hg'

[Pierre-Yves David]

# HG changeset patch
# User Pierre-Yves David 
# Date 1488457662 -3600
#  Thu Mar 02 13:27:42 2017 +0100
# Node ID b033990d12c81527cb1be32b6941086e5f419ad1
# Parent  69ef1663d5fe2a3bf7bd4138b98ae373b1848591
# EXP-Topic vfs.cleanup
# Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/
#  hg pull 
https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r b033990d12c8
vfs: use 'vfs' module directly in 'mercurial.hg'

Now that the 'vfs' classes moved in their own module, lets use the new module
directly. We update code iteratively to help with possible bisect needs in the
future.

diff --git a/mercurial/hg.py b/mercurial/hg.py
--- a/mercurial/hg.py
+++ b/mercurial/hg.py
@@ -40,6 +40,7 @@ from . import (
 url,
 util,
 verify as verifymod,
+vfs as vfsmod,
 )
 
 release = lock.release
@@ -219,8 +220,8 @@ def share(ui, source, dest=None, update=
 
 sharedpath = srcrepo.sharedpath # if our source is already sharing
 
-destwvfs = scmutil.vfs(dest, realpath=True)
-destvfs = scmutil.vfs(os.path.join(destwvfs.base, '.hg'), realpath=True)
+destwvfs = vfsmod.vfs(dest, realpath=True)
+destvfs = vfsmod.vfs(os.path.join(destwvfs.base, '.hg'), realpath=True)
 
 if destvfs.lexists():
 raise error.Abort(_('destination already exists'))
@@ -312,8 +313,8 @@ def copystore(ui, srcrepo, destpath):
 else:
 ui.progress(topic, pos + num)
 srcpublishing = srcrepo.publishing()
-srcvfs = scmutil.vfs(srcrepo.sharedpath)
-dstvfs = scmutil.vfs(destpath)
+srcvfs = vfsmod.vfs(srcrepo.sharedpath)
+dstvfs = vfsmod.vfs(destpath)
 for f in srcrepo.store.copylist():
 if srcpublishing and f.endswith('phaseroots'):
 continue
@@ -369,7 +370,7 @@ def clonewithshare(ui, peeropts, sharepa
 if e.errno != errno.EEXIST:
 raise
 
-poolvfs = scmutil.vfs(pooldir)
+poolvfs = vfsmod.vfs(pooldir)
 basename = os.path.basename(sharepath)
 
 with lock.lock(poolvfs, '%s.lock' % basename):
@@ -474,7 +475,7 @@ def clone(ui, peeropts, source, dest=Non
 if not dest:
 raise error.Abort(_("empty destination path is not valid"))
 
-destvfs = scmutil.vfs(dest, expandpath=True)
+destvfs = vfsmod.vfs(dest, expandpath=True)
 if destvfs.lexists():
 if not destvfs.isdir():
 raise error.Abort(_("destination '%s' already exists") % dest)
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 1 of 3 V3] chgcache: introduce a new global state

[Jun Wu]

This is actually a redesign of "chgcache", focusing on "cache" - how to get,
set, invalidate things.

It seems to me that the IPC, fork, repo path reporting etc. fit better in
chgserver, dispatch and elsewhere. This layout looks cleaner to me.

Excerpts from Jun Wu's message of 2017-03-07 22:35:57 -0800:
> # HG changeset patch
> # User Jun Wu 
> # Date 1488947963 28800
> #  Tue Mar 07 20:39:23 2017 -0800
> # Node ID 60eb2c2b5196a62d635dbe0eb1e29fdd945d5058
> # Parent  c32f9eeec75445bfbbc55df4c1fcc584d3cf45cd
> # Available At https://bitbucket.org/quark-zju/hg-draft 
> #  hg pull https://bitbucket.org/quark-zju/hg-draft  -r 
> 60eb2c2b5196
> chgcache: introduce a new global state
> 
> The new chgcache module will be used to store some cache state of chg, which
> is useful to preload repo-related contents and speed up related operations.
> 
> Motivation:
> 
>   Currently, chg only preloads extensions and one chg server is responsible
>   for multiple repos (to avoid unnecessary memory usage).
> 
>   With the new module, eventually the master chg server will have a state
>   storing multiple components of multiple repos, like the changelog index
>   (with radix tree prebuilt), the dirstate, the phasecache, the bookmarks,
>   the obsstore, etc.
> 
> Why a new module:
> 
>   The feature is low-level and relatively separate from chgserver (which is
>   focused on client/server stuff). So it's made a new module. It could be
>   used without chg in theory, although it depends on chg practically.
> 
> This patch only contains very basic "set" and "get" methods.
> 
> diff --git a/mercurial/chgcache.py b/mercurial/chgcache.py
> new file mode 100644
> --- /dev/null
> +++ b/mercurial/chgcache.py
> @@ -0,0 +1,22 @@
> +# chgcache.py - caching state for chg
> +#
> +# Copyright 2017 Facebook, Inc.
> +#
> +# This software may be used and distributed according to the terms of the
> +# GNU General Public License version 2 or any later version.
> +
> +from __future__ import absolute_import
> +
> +_cache = {}
> +
> +def get(key):
> +"""look up an entry. returns None if key is not found"""
> +return _cache.get(key)
> +
> +def set(key, value):
> +"""write to the cache. if value is None, remove the entry"""
> +if value is None:
> +if key in _cache:
> +del _cache[key]
> +else:
> +_cache[key] = value
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 3 of 3 V3] chgcache: implement repocache

[Jun Wu]

# HG changeset patch
# User Jun Wu 
# Date 1488953311 28800
#  Tue Mar 07 22:08:31 2017 -0800
# Node ID d136f214b3a5bd4698dfd96c641ad73f96a743cb
# Parent  f0bded8d53c5c9a5cfb25d29dd99cf4eb3fb79b2
# Available At https://bitbucket.org/quark-zju/hg-draft
#  hg pull https://bitbucket.org/quark-zju/hg-draft -r d136f214b3a5
chgcache: implement repocache

The repocache is based on smartcache. It will be used widely because most
objects interested to stateful chg are repo-related.

In the future, we may want to move part of localrepository to a thin class,
but for now we just use the repo object directly.

diff --git a/mercurial/chgcache.py b/mercurial/chgcache.py
--- a/mercurial/chgcache.py
+++ b/mercurial/chgcache.py
@@ -80,2 +80,7 @@ class smartcache(object):
 set(fullkey, (newhash, newvalue))
 return newvalue
+
+class repocache(smartcache):
+def __init__(self, repo, loadfunctable):
+keyprefix = 'repo\0%s\0' % repo.root
+super(repocache, self).__init__(keyprefix, repo, loadfunctable)
diff --git a/tests/test-chgcache.py b/tests/test-chgcache.py
--- a/tests/test-chgcache.py
+++ b/tests/test-chgcache.py
@@ -4,6 +4,9 @@ import os
 
 from mercurial import (
+changelog,
 chgcache,
+localrepo,
 scmutil,
+ui as uimod,
 )
 
@@ -55,2 +58,35 @@ printcache() # None, will invalidate the
 vfs.write(filename, 'ef')
 printcache() # cache miss, 'ef'
+
+def loadchangelog(repo, oldhash, oldvalue):
+# NOTE: This does not take care of corner cases. See "readfoo".
+newhash = repo.svfs.stat('00changelog.i').st_size
+if newhash == oldhash:
+print('changelog cache hit')
+return oldhash, oldvalue
+else:
+print('changelog cache miss')
+newvalue = changelog.changelog(repo.svfs)
+return newhash, newvalue
+
+repoloadfunctable = {'changelog': loadchangelog}
+
+ui = uimod.ui()
+ui.setconfig('ui', 'allowemptycommit', '1')
+
+repo = localrepo.localrepository(
+ui,
+os.path.join(os.environ['TESTTMP'], 'repo1'),
+create=True)
+repocache = chgcache.repocache(repo, repoloadfunctable)
+
+def printrepocache():
+print('changelog has %d revisions' % len(repocache.get('changelog')))
+
+repo.commit('foo')
+printrepocache()
+printrepocache()
+
+repo.commit('bar')
+printrepocache()
+printrepocache()
diff --git a/tests/test-chgcache.py.out b/tests/test-chgcache.py.out
--- a/tests/test-chgcache.py.out
+++ b/tests/test-chgcache.py.out
@@ -11,2 +11,10 @@ cache["foo"] = None
 cache miss
 cache["foo"] = 'ef'
+changelog cache miss
+changelog has 1 revisions
+changelog cache hit
+changelog has 1 revisions
+changelog cache miss
+changelog has 2 revisions
+changelog cache hit
+changelog has 2 revisions
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 2 of 3 V3] chgcache: implement a smartcache layer

[Jun Wu]

# HG changeset patch
# User Jun Wu 
# Date 1488949878 28800
#  Tue Mar 07 21:11:18 2017 -0800
# Node ID f0bded8d53c5c9a5cfb25d29dd99cf4eb3fb79b2
# Parent  60eb2c2b5196a62d635dbe0eb1e29fdd945d5058
# Available At https://bitbucket.org/quark-zju/hg-draft
#  hg pull https://bitbucket.org/quark-zju/hg-draft -r f0bded8d53c5
chgcache: implement a smartcache layer

See the docstring of smartcache. Basically it sets up a pattern where
everything in the cache has a corresponding hash value to help test if it's
valid quickly.

It's mainly designed to be used with the repo state. See the next patch.

diff --git a/mercurial/chgcache.py b/mercurial/chgcache.py
--- a/mercurial/chgcache.py
+++ b/mercurial/chgcache.py
@@ -21,2 +21,61 @@ def set(key, value):
 else:
 _cache[key] = value
+
+class smartcache(object):
+"""cache knowing how to load and invalidate values, for predefined keys
+
+The cache object will only answer to a key who is in loadfunctable. The
+loadfunctable stores load functions which will do hashing and loading.
+smartcache will update or invalidate entries according to the hash, and
+provide the hash and value to load functions being called next time.
+
+There is no "set" method. To pre-populate the cache, call "get" instead.
+This will make sure the hash values are always correctly set.
+
+The end users using smartcache.get will only notice the values, the hashes
+and the cache is transparent to them.
+
+A load function has the signature:
+
+(state, oldhash, oldvalue) -> (newhash, newvalue)
+
+Where state is where the load function reads information. oldhash, oldvalue
+is what currently being stored in the cache. The returned hash and value
+will be used to update the cache.
+
+A load function usually looks like:
+
+def loadfunc(state, oldhash, oldvalue):
+hash = state.quickhash()
+if hash == oldhash:
+return oldhash, oldvalue
+value = state.loadvalue()
+hash = hashvalue(value)
+# or, if hashvalue is expensive
+hashagain = state.quickhash()
+if hashagain != hash:
+# invalidate the cache entry without filling a new one
+hash = None
+return hash, value
+
+If predefined keys are not flexible enough, loadfunctable could be an
+object implementing "get" which generates load functions dynamically.
+"""
+
+def __init__(self, keyprefix, state, loadfunctable):
+self.keyprefix = keyprefix
+self.state = state
+self.loadfunctable = loadfunctable
+
+def get(self, key):
+loadfunc = self.loadfunctable.get(key)
+if loadfunc is None:
+return None
+fullkey = self.keyprefix + key
+oldhash, oldvalue = get(fullkey) or [None, None]
+newhash, newvalue = loadfunc(self.state, oldhash, oldvalue)
+if newhash is None:
+set(fullkey, None)
+elif newvalue is not oldvalue or newhash != oldhash:
+set(fullkey, (newhash, newvalue))
+return newvalue
diff --git a/tests/test-chgcache.py b/tests/test-chgcache.py
new file mode 100644
--- /dev/null
+++ b/tests/test-chgcache.py
@@ -0,0 +1,56 @@
+from __future__ import absolute_import, print_function
+
+import os
+
+from mercurial import (
+chgcache,
+scmutil,
+)
+
+filename = 'foo'
+
+def readfoo(vfs, oldhash, oldvalue):
+# NOTE: st_size is intentional for this test. Do not use it in real code if
+# the file could be rewritten (not append-only).
+try:
+newhash = vfs.stat(filename).st_size
+except OSError:
+return None, None
+if oldhash == newhash:
+print('cache hit')
+return oldhash, oldvalue
+else:
+print('cache miss')
+value = vfs.read(filename)
+# NOTE: This is wrong. In reality, we need to calculate the hash again,
+# and return None as the "newhash" if hashes mismatch, to mitigate
+# filesystem race conditions.
+# That said, in this test we do know nobody else will touch the file,
+# so it's fine.
+return newhash, value
+
+loadfuncs = {'foo': readfoo}
+vfs = scmutil.vfs(os.environ['TESTTMP'])
+
+cache = chgcache.smartcache('vfs', vfs, loadfuncs)
+
+def printcache():
+print('cache["foo"] = %r' % cache.get('foo'))
+
+printcache() # None, because the file does not exist
+
+vfs.write(filename, 'a')
+printcache() # cache miss, 'a'
+printcache() # cache hit, 'a'
+
+vfs.write(filename, 'ab')
+printcache() # cache miss, 'ab'
+
+vfs.write(filename, 'cd')
+printcache() # cache hit, 'ab'
+
+vfs.unlink('foo')
+printcache() # None, will invalidate the cache
+
+vfs.write(filename, 'ef')
+printcache() # cache miss, 'ef'
diff --git a/tests/test-chgcache.py.out b/tests/test-chgcache.py.out
new file mode 100644
--- /dev/null
+++ b/tests/test-chgcache.py.out
@@ -0,0 +1,12 @@
+cache["foo"] = None
+ca

[PATCH 1 of 3 V3] chgcache: introduce a new global state

[Jun Wu]

# HG changeset patch
# User Jun Wu 
# Date 1488947963 28800
#  Tue Mar 07 20:39:23 2017 -0800
# Node ID 60eb2c2b5196a62d635dbe0eb1e29fdd945d5058
# Parent  c32f9eeec75445bfbbc55df4c1fcc584d3cf45cd
# Available At https://bitbucket.org/quark-zju/hg-draft
#  hg pull https://bitbucket.org/quark-zju/hg-draft -r 60eb2c2b5196
chgcache: introduce a new global state

The new chgcache module will be used to store some cache state of chg, which
is useful to preload repo-related contents and speed up related operations.

Motivation:

  Currently, chg only preloads extensions and one chg server is responsible
  for multiple repos (to avoid unnecessary memory usage).

  With the new module, eventually the master chg server will have a state
  storing multiple components of multiple repos, like the changelog index
  (with radix tree prebuilt), the dirstate, the phasecache, the bookmarks,
  the obsstore, etc.

Why a new module:

  The feature is low-level and relatively separate from chgserver (which is
  focused on client/server stuff). So it's made a new module. It could be
  used without chg in theory, although it depends on chg practically.

This patch only contains very basic "set" and "get" methods.

diff --git a/mercurial/chgcache.py b/mercurial/chgcache.py
new file mode 100644
--- /dev/null
+++ b/mercurial/chgcache.py
@@ -0,0 +1,22 @@
+# chgcache.py - caching state for chg
+#
+# Copyright 2017 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+_cache = {}
+
+def get(key):
+"""look up an entry. returns None if key is not found"""
+return _cache.get(key)
+
+def set(key, value):
+"""write to the cache. if value is None, remove the entry"""
+if value is None:
+if key in _cache:
+del _cache[key]
+else:
+_cache[key] = value
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH website] downloads: add link to PPA for ubuntu/debian

[Sean Farley]

Kevin Bullock  writes:

>> On Feb 26, 2017, at 02:52, Anton Shestakov  wrote:
>> 
>> On Sat, 25 Feb 2017 13:02:15 -0800
>> heatherboo  wrote:
>> 
>>> # HG changeset patch
>>> # User heatherboo 
>>> # Date 1488054963 28800
>>> #  Sat Feb 25 12:36:03 2017 -0800
>>> # Node ID b91f88bda2404c91722a62a2eb9afc63867a172b
>>> # Parent  f8cee73d5cf0e9b4d46859c949ed0e23d65dd38c
>>> downloads: add link to PPA for ubuntu/debian
>>> 
>>> diff --git a/templates/downloads/index.html b/templates/downloads/index.html
>>> --- a/templates/downloads/index.html
>>> +++ b/templates/downloads/index.html
>>> @@ -114,7 +114,9 @@
>>> GUI the most. It integrates Mercurial directly into your explorer.
>>> 
>>> Packages for common Linux, BSD and Solaris distributions can be
>>> -installed from the system specific repositories
>>> +installed from the system specific repositories:
>>> +(Debian/Ubuntu: If you need a more recent version than what is 
>>> available through apt-get, you can >> href="https://launchpad.net/~mercurial-ppa/+archive/ubuntu/releases";>try 
>>> the PPA.)
>>> +
>> 
>> I don't think it should recommend PPA for Debian too, because using
>> Ubuntu PPAs in Debian AFAIU is literally just rebuilding binary packages
>> from source packages. Which is easier with `make deb`.
>
> I wonder if we should also suggest backports for Debian here? I very much 
> like the idea of having the links to more up-to-date repos in the sidebar, 
> and instructions for `make deb` aren't likely to fit very well here.

I kinda feel as if the entire site needs a new coat of paint. Maybe we
can discuss this at the sprint.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH website] downloads: add link to PPA for ubuntu/debian

[Sean Farley]

Anton Shestakov  writes:

> On Sat, 25 Feb 2017 13:02:15 -0800
> heatherboo  wrote:
>
>> # HG changeset patch
>> # User heatherboo 
>> # Date 1488054963 28800
>> #  Sat Feb 25 12:36:03 2017 -0800
>> # Node ID b91f88bda2404c91722a62a2eb9afc63867a172b
>> # Parent  f8cee73d5cf0e9b4d46859c949ed0e23d65dd38c
>> downloads: add link to PPA for ubuntu/debian
>> 
>> diff --git a/templates/downloads/index.html b/templates/downloads/index.html
>> --- a/templates/downloads/index.html
>> +++ b/templates/downloads/index.html
>> @@ -114,7 +114,9 @@
>>  GUI the most. It integrates Mercurial directly into your explorer.
>>  
>>  Packages for common Linux, BSD and Solaris distributions can be
>> -installed from the system specific repositories
>> +installed from the system specific repositories:
>> +(Debian/Ubuntu: If you need a more recent version than what is 
>> available through apt-get, you can > href="https://launchpad.net/~mercurial-ppa/+archive/ubuntu/releases";>try the 
>> PPA.)
>> +
>
> I don't think it should recommend PPA for Debian too, because using
> Ubuntu PPAs in Debian AFAIU is literally just rebuilding binary packages
> from source packages. Which is easier with `make deb`.
>
> Otherwise LGTM.

I didn't mind it too much but it makes sense. I'll remove 'Debian/' and queue.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 07 of 10 V2] context: remove uses of manifest.matches

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488937965 28800
#  Tue Mar 07 17:52:45 2017 -0800
# Node ID 822259e9f117b11cd54d6a13565a3202b508eae8
# Parent  d2850df6c891a20585a0d5eac370c1c2f4463cad
context: remove uses of manifest.matches

This removes the uses of manifest.matches in context.py in favor of the new
manifest.diff(match) api. This is part of removing manifest.matches since it is
O(manifest).

To drop the dependency on ctx._manifestmatches(s) we transfer responsibilty for
creating status oriented manifests over to ctx._buildstatusmanifest(s). This
already existed for workingctx, we just need to implement a simple version for
basectx. The old _manifestmatches functionality is basically identical to the
_buildstatusmanifest functionality (minus the matching part), so no behavior
should be lost.

diff --git a/mercurial/context.py b/mercurial/context.py
--- a/mercurial/context.py
+++ b/mercurial/context.py
@@ -18,7 +18,6 @@ from .node import (
 bin,
 hex,
 modifiednodeid,
-newnodeid,
 nullid,
 nullrev,
 short,
@@ -91,14 +90,11 @@ class basectx(object):
 def __iter__(self):
 return iter(self._manifest)
 
-def _manifestmatches(self, match, s):
-"""generate a new manifest filtered by the match argument
-
-This method is for internal use only and mainly exists to provide an
-object oriented way for other contexts to customize the manifest
-generation.
-"""
-return self.manifest().matches(match)
+def _buildstatusmanifest(self, status):
+"""Builds a manifest that includes the given status results, if this is
+a working copy context. For non-working copy contexts, it just returns
+the normal manifest."""
+return self.manifest()
 
 def _matchstatus(self, other, match):
 """return match.always if match is none
@@ -119,17 +115,17 @@ class basectx(object):
 # delta application.
 mf2 = None
 if self.rev() is not None and self.rev() < other.rev():
-mf2 = self._manifestmatches(match, s)
-mf1 = other._manifestmatches(match, s)
+mf2 = self._buildstatusmanifest(s)
+mf1 = other._buildstatusmanifest(s)
 if mf2 is None:
-mf2 = self._manifestmatches(match, s)
+mf2 = self._buildstatusmanifest(s)
 
 modified, added = [], []
 removed = []
 clean = []
 deleted, unknown, ignored = s.deleted, s.unknown, s.ignored
 deletedset = set(deleted)
-d = mf1.diff(mf2, clean=listclean)
+d = mf1.diff(mf2, match=match, clean=listclean)
 for fn, value in d.iteritems():
 if fn in deletedset:
 continue
@@ -156,8 +152,10 @@ class basectx(object):
 
 if removed:
 # need to filter files if they are already reported as removed
-unknown = [fn for fn in unknown if fn not in mf1]
-ignored = [fn for fn in ignored if fn not in mf1]
+unknown = [fn for fn in unknown if fn not in mf1 and
+   (not match or match(fn))]
+ignored = [fn for fn in ignored if fn not in mf1 and
+   (not match or match(fn))]
 # if they're deleted, don't report them as removed
 removed = [fn for fn in removed if fn not in deletedset]
 
@@ -1581,22 +1579,6 @@ class workingctx(committablectx):
 pass
 return modified, fixup
 
-def _manifestmatches(self, match, s):
-"""Slow path for workingctx
-
-The fast path is when we compare the working directory to its parent
-which means this function is comparing with a non-parent; therefore we
-need to build a manifest and return what matches.
-"""
-mf = self._repo['.']._manifestmatches(match, s)
-for f in s.modified + s.added:
-mf[f] = newnodeid
-mf.setflag(f, self.flags(f))
-for f in s.removed:
-if f in mf:
-del mf[f]
-return mf
-
 def _dirstatestatus(self, match=None, ignored=False, clean=False,
 unknown=False):
 '''Gets the status from the dirstate -- internal use only.'''
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 09 of 10 V2] treemanifest: add tests covering hg diff of partial trees

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488940198 28800
#  Tue Mar 07 18:29:58 2017 -0800
# Node ID 611fac63adb09c326912e56df59c828ad12ffd9f
# Parent  b76e7da4fb9ab00c3b98e5613d7caa0b4aceaf8b
treemanifest: add tests covering hg diff of partial trees

Previously the hg files tests also covered the logic (i.e.
treemanifest._matches) that governed how hg diff limited its diff. In a future
patch we will be switching treemanifest.diff() to have a custom implementation,
so let's go ahead and add equivalent test coverage for hg diff.

diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t
--- a/tests/test-treemanifest.t
+++ b/tests/test-treemanifest.t
@@ -437,6 +437,16 @@ Create deeper repo with tree manifests.
 
   $ hg ci -Aqm 'initial'
 
+  $ echo >> .A/one.txt
+  $ echo >> .A/two.txt
+  $ echo >> b/bar/fruits.txt
+  $ echo >> b/bar/orange/fly/gnat.py
+  $ echo >> b/bar/orange/fly/housefly.txt
+  $ echo >> b/foo/apple/bees/flower.py
+  $ echo >> c.txt
+  $ echo >> d.py
+  $ hg ci -Aqm 'second'
+
 We'll see that visitdir works by removing some treemanifest revlogs and running
 the files command with various parameters.
 
@@ -468,6 +478,12 @@ Test files for a subdirectory.
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
   b/foo/apple/bees/flower.py (glob)
+  $ hg diff -r '.^' -r . --stat b
+   b/bar/fruits.txt  |  1 +
+   b/bar/orange/fly/gnat.py  |  1 +
+   b/bar/orange/fly/housefly.txt |  1 +
+   b/foo/apple/bees/flower.py|  1 +
+   4 files changed, 4 insertions(+), 0 deletions(-)
   $ cp -R .hg/store-copy/. .hg/store
 
 Test files with just includes and excludes.
@@ -477,6 +493,9 @@ Test files with just includes and exclud
   $ rm -r .hg/store/meta/b/foo/apple/bees
   $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X 
path:b/foo/apple/bees
   b/bar/fruits.txt (glob)
+  $ hg diff -r '.^' -r . --stat -I path:b/bar -X path:b/bar/orange/fly -I 
path:b/foo -X path:b/foo/apple/bees
+   b/bar/fruits.txt |  1 +
+   1 files changed, 1 insertions(+), 0 deletions(-)
   $ cp -R .hg/store-copy/. .hg/store
 
 Test files for a subdirectory, excluding a directory within it.
@@ -487,6 +506,11 @@ Test files for a subdirectory, excluding
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
+  $ hg diff -r '.^' -r . --stat -X path:b/foo b
+   b/bar/fruits.txt  |  1 +
+   b/bar/orange/fly/gnat.py  |  1 +
+   b/bar/orange/fly/housefly.txt |  1 +
+   3 files changed, 3 insertions(+), 0 deletions(-)
   $ cp -R .hg/store-copy/. .hg/store
 
 Test files for a sub directory, including only a directory within it, and
@@ -497,6 +521,10 @@ including an unrelated directory.
   $ hg files -r . -I path:b/bar/orange -I path:a b
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
+  $ hg diff -r '.^' -r . --stat -I path:b/bar/orange -I path:a b
+   b/bar/orange/fly/gnat.py  |  1 +
+   b/bar/orange/fly/housefly.txt |  1 +
+   2 files changed, 2 insertions(+), 0 deletions(-)
   $ cp -R .hg/store-copy/. .hg/store
 
 Test files for a pattern, including a directory, and excluding a directory
@@ -507,6 +535,9 @@ within that.
   $ rm -r .hg/store/meta/b/bar/orange
   $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange
   b/bar/fruits.txt (glob)
+  $ hg diff -r '.^' -r . --stat glob:**.txt -I path:b/bar -X path:b/bar/orange
+   b/bar/fruits.txt |  1 +
+   1 files changed, 1 insertions(+), 0 deletions(-)
   $ cp -R .hg/store-copy/. .hg/store
 
 Add some more changes to the deep repo
@@ -522,7 +553,7 @@ Verify works
   checking directory manifests
   crosschecking files in changesets and manifests
   checking files
-  8 files, 3 changesets, 10 total revisions
+  8 files, 4 changesets, 18 total revisions
 
 Dirlogs are included in fncache
   $ grep meta/.A/00manifest.i .hg/store/fncache
@@ -563,8 +594,9 @@ Verify reports missing dirlog
   checking directory manifests
0: empty or missing b/
b/@0: parent-directory manifest refers to unknown revision 67688a370455
-   b/@1: parent-directory manifest refers to unknown revision f38e85d334c5
-   b/@2: parent-directory manifest refers to unknown revision 99c9792fd4b0
+   b/@1: parent-directory manifest refers to unknown revision f065da70369e
+   b/@2: parent-directory manifest refers to unknown revision ac0d30948e0b
+   b/@3: parent-directory manifest refers to unknown revision 367152e6af28
   warning: orphan revlog 'meta/b/bar/00manifest.i'
   warning: orphan revlog 'meta/b/bar/orange/00manifest.i'
   warning: orphan revlog 'meta/b/bar/orange/fly/00manifest.i'
@@ -577,9 +609,9 @@ Verify reports missing dirlog
b/bar/orange/fly/housefly.txt@0: in changeset but not in manifest
b/foo/apple/bees/flower.py@0: in changeset but not in manifest
   checking files
-  8 files, 3 changesets, 10 total revisions
+  8 files, 4 changesets, 18 total revisions
   6 warnings encountered!
-  8 integrity errors encountered

[PATCH 10 of 10 V2] treemanifest: optimize diff using the matcher

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488943242 28800
#  Tue Mar 07 19:20:42 2017 -0800
# Node ID 541bf866729342f534bac425bd8f01b9fe7564e8
# Parent  611fac63adb09c326912e56df59c828ad12ffd9f
treemanifest: optimize diff using the matcher

This optimizes treemanifest.diff() to limit the tree traversal based on the
provided matcher. According to Martin's testing, `hg status --rev .~1 --rev .
foo/` goes from 1.3s to 0.18s on a tree version of Mozilla central. I'd expect
and even greater saving on larger internal repos at big companies.

A previous patch added test coverage for treemanifest diff with patterns.

diff --git a/mercurial/manifest.py b/mercurial/manifest.py
--- a/mercurial/manifest.py
+++ b/mercurial/manifest.py
@@ -1053,10 +1053,6 @@ class treemanifest(object):
 the nodeid will be None and the flags will be the empty
 string.
 '''
-if match:
-m1 = self._matches(match)
-m2 = m2._matches(match)
-return m1.diff(m2, clean=clean)
 result = {}
 emptytree = treemanifest()
 def _diff(t1, t2):
@@ -1065,26 +1061,31 @@ class treemanifest(object):
 t1._load()
 t2._load()
 for d, m1 in t1._dirs.iteritems():
-m2 = t2._dirs.get(d, emptytree)
-_diff(m1, m2)
+if not match or match.visitdir(os.path.join(t1.dir(), d[:-1])):
+m2 = t2._dirs.get(d, emptytree)
+_diff(m1, m2)
 
 for d, m2 in t2._dirs.iteritems():
 if d not in t1._dirs:
-_diff(emptytree, m2)
+if (not match or match.visitdir(os.path.join(t2.dir(),
+ d[:-1]))):
+_diff(emptytree, m2)
 
 for fn, n1 in t1._files.iteritems():
-fl1 = t1._flags.get(fn, '')
-n2 = t2._files.get(fn, None)
-fl2 = t2._flags.get(fn, '')
-if n1 != n2 or fl1 != fl2:
-result[t1._subpath(fn)] = ((n1, fl1), (n2, fl2))
-elif clean:
-result[t1._subpath(fn)] = None
+if not match or match(os.path.join(t1.dir(), fn)):
+fl1 = t1._flags.get(fn, '')
+n2 = t2._files.get(fn, None)
+fl2 = t2._flags.get(fn, '')
+if n1 != n2 or fl1 != fl2:
+result[t1._subpath(fn)] = ((n1, fl1), (n2, fl2))
+elif clean:
+result[t1._subpath(fn)] = None
 
 for fn, n2 in t2._files.iteritems():
 if fn not in t1._files:
-fl2 = t2._flags.get(fn, '')
-result[t2._subpath(fn)] = ((None, ''), (n2, fl2))
+if not match or match(os.path.join(t2.dir(), fn)):
+fl2 = t2._flags.get(fn, '')
+result[t2._subpath(fn)] = ((None, ''), (n2, fl2))
 
 _diff(self, m2)
 return result
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 04 of 10 V2] status: handle more node indicators in buildstatus

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488909371 28800
#  Tue Mar 07 09:56:11 2017 -0800
# Node ID 68644896cbdcfc956079662668e010038bfe8048
# Parent  ccec5eeceb85929c5df041b768d01694176b3a9d
status: handle more node indicators in buildstatus

There are several different node markers that indicate different working copy
states. The context._buildstatus function was only handling one of them, and
this patch makes it handle all of them (falling back to file content comparisons
when in one of these states).

This affects a future patch where we get rid of context._manifestmatches as part
of getting rid of manifest.matches(). context._manifestmatches is currently
hacky in that it uses the newnodeid for all added and modified files, which is
why the current newnodeid check is sufficient. When we get rid of this function
and use the normal manifest.diff function, we start to see the other indicators
in the nodes, so they need to be handled or else the tests fail.

diff --git a/mercurial/context.py b/mercurial/context.py
--- a/mercurial/context.py
+++ b/mercurial/context.py
@@ -23,6 +23,7 @@ from .node import (
 nullrev,
 short,
 wdirid,
+wdirnodes,
 )
 from . import (
 encoding,
@@ -140,7 +141,7 @@ class basectx(object):
 removed.append(fn)
 elif flag1 != flag2:
 modified.append(fn)
-elif node2 != newnodeid:
+elif node2 not in wdirnodes:
 # When comparing files between two commits, we save time by
 # not comparing the file contents when the nodeids differ.
 # Note that this means we incorrectly report a reverted change
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 03 of 10 V2] merge: remove uses of manifest.matches

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488940700 28800
#  Tue Mar 07 18:38:20 2017 -0800
# Node ID ccec5eeceb85929c5df041b768d01694176b3a9d
# Parent  f70310209acbbcb4cf7cae7c38d94e15c09c1ace
merge: remove uses of manifest.matches

This gets rid of the manifest.matches calls in merge.py in favor of the new api.
This is part of getting rid of manifest.matches since it is O(manifest).

diff --git a/mercurial/merge.py b/mercurial/merge.py
--- a/mercurial/merge.py
+++ b/mercurial/merge.py
@@ -27,6 +27,7 @@ from . import (
 copies,
 error,
 filemerge,
+match as matchmod,
 obsolete,
 pycompat,
 scmutil,
@@ -818,11 +819,10 @@ def manifestmerge(repo, wctx, p2, pa, br
 if any(wctx.sub(s).dirty() for s in wctx.substate):
 m1['.hgsubstate'] = modifiednodeid
 
-# Compare manifests
-if matcher is not None:
-m1 = m1.matches(matcher)
-m2 = m2.matches(matcher)
-diff = m1.diff(m2)
+diff = m1.diff(m2, match=matcher)
+
+if matcher is None:
+matcher = matchmod.always('', '')
 
 actions = {}
 for f, ((n1, fl1), (n2, fl2)) in diff.iteritems():
@@ -858,7 +858,7 @@ def manifestmerge(repo, wctx, p2, pa, br
 pass # we'll deal with it on m2 side
 elif f in movewithdir: # directory rename, move local
 f2 = movewithdir[f]
-if f2 in m2:
+if matcher(f2) and f2 in m2:
 actions[f2] = ('m', (f, f2, None, True, pa.node()),
"remote directory rename, both created")
 else:
@@ -887,7 +887,7 @@ def manifestmerge(repo, wctx, p2, pa, br
 pass # we'll deal with it on m1 side
 elif f in movewithdir:
 f2 = movewithdir[f]
-if f2 in m1:
+if matcher(f2) and f2 in m1:
 actions[f2] = ('m', (f2, f, None, False, pa.node()),
"local directory rename, both created")
 else:
@@ -895,7 +895,7 @@ def manifestmerge(repo, wctx, p2, pa, br
"local directory rename - get from " + f)
 elif f in copy:
 f2 = copy[f]
-if f2 in m2:
+if matcher(f2) and f2 in m2:
 actions[f] = ('m', (f2, f, f2, False, pa.node()),
   "remote copied from " + f2)
 else:
@@ -927,7 +927,7 @@ def manifestmerge(repo, wctx, p2, pa, br
 # new file added in a directory that was moved
 df = dirmove[d] + f[len(d):]
 break
-if df in m1:
+if matcher(df) and df in m1:
 actions[df] = ('m', (df, f, f, False, pa.node()),
 "local directory rename - respect move from " + f)
 elif acceptremote:
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 05 of 10 V2] context: move _manifest from committablectx to workingctx

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488938190 28800
#  Tue Mar 07 17:56:30 2017 -0800
# Node ID 36bcc5d848c6bdf33d604999631a0708d1b7f067
# Parent  68644896cbdcfc956079662668e010038bfe8048
context: move _manifest from committablectx to workingctx

committablectx had a _manifest implementation that was only used by the derived
workingctx class. The other derived versions, like memctx and metadataonlyctx,
define their own _manifest functions.

Let's move the function down to workingctx, and let's break it into two parts,
the _manifest part that reads from self._status, and the part that actually
builds the new manifest. This separation will let us reuse the builder code in a
future patch to answer _buildstatus with varying status inputs, since workingctx
has special behavior for _buildstatus that the other ctx's don't have.

diff --git a/mercurial/context.py b/mercurial/context.py
--- a/mercurial/context.py
+++ b/mercurial/context.py
@@ -1266,35 +1266,6 @@ class committablectx(basectx):
 return self._repo.dirstate.flagfunc(self._buildflagfunc)
 
 @propertycache
-def _manifest(self):
-"""generate a manifest corresponding to the values in self._status
-
-This reuse the file nodeid from parent, but we append an extra letter
-when modified. Modified files get an extra 'm' while added files get
-an extra 'a'. This is used by manifests merge to see that files
-are different and by update logic to avoid deleting newly added files.
-"""
-parents = self.parents()
-
-man = parents[0].manifest().copy()
-
-ff = self._flagfunc
-for i, l in ((addednodeid, self._status.added),
- (modifiednodeid, self._status.modified)):
-for f in l:
-man[f] = i
-try:
-man.setflag(f, ff(f))
-except OSError:
-pass
-
-for f in self._status.deleted + self._status.removed:
-if f in man:
-del man[f]
-
-return man
-
-@propertycache
 def _status(self):
 return self._repo.status()
 
@@ -1655,6 +1626,41 @@ class workingctx(committablectx):
 
 return s
 
+@propertycache
+def _manifest(self):
+"""generate a manifest corresponding to the values in self._status
+
+This reuse the file nodeid from parent, but we use special node
+identifiers for added and modified files. This is used by manifests
+merge to see that files are different and by update logic to avoid
+deleting newly added files.
+"""
+return self._buildstatusmanifest(self._status)
+
+def _buildstatusmanifest(self, status):
+"""Builds a manifest that includes the given status results, if this is
+a working copy context. For non-working copy contexts, it just returns
+the normal manifest."""
+parents = self.parents()
+
+man = parents[0].manifest().copy()
+
+ff = self._flagfunc
+for i, l in ((addednodeid, status.added),
+ (modifiednodeid, status.modified)):
+for f in l:
+man[f] = i
+try:
+man.setflag(f, ff(f))
+except OSError:
+pass
+
+for f in status.deleted + status.removed:
+if f in man:
+del man[f]
+
+return man
+
 def _buildstatus(self, other, s, match, listignored, listclean,
  listunknown):
 """build a status with respect to another context
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 02 of 10 V2] copies: remove use of manifest.matches

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488909371 28800
#  Tue Mar 07 09:56:11 2017 -0800
# Node ID f70310209acbbcb4cf7cae7c38d94e15c09c1ace
# Parent  5ede3fa3bae6e964b4bce8b36681d203c05e94f7
copies: remove use of manifest.matches

Convert the existing use of manifest.matches to use the new api. This is part
of getting rid of manifest.matches, since it is O(manifest).

diff --git a/mercurial/copies.py b/mercurial/copies.py
--- a/mercurial/copies.py
+++ b/mercurial/copies.py
@@ -149,10 +149,7 @@ def _computeforwardmissing(a, b, match=N
 """
 ma = a.manifest()
 mb = b.manifest()
-if match:
-ma = ma.matches(match)
-mb = mb.matches(match)
-return mb.filesnotin(ma)
+return mb.filesnotin(ma, match=match)
 
 def _forwardcopies(a, b, match=None):
 '''find {dst@b: src@a} copy mapping where a is an ancestor of b'''
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 08 of 10 V2] manifest: rename matches to _matches

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488937983 28800
#  Tue Mar 07 17:53:03 2017 -0800
# Node ID b76e7da4fb9ab00c3b98e5613d7caa0b4aceaf8b
# Parent  822259e9f117b11cd54d6a13565a3202b508eae8
manifest: rename matches to _matches

Now that there are no external consumers of manifest.matches, let's rename it to
_matches so it remains internal. This means alternative manifest implementations
no longer have to implement it, and can instead implement efficient versions of
diff() and filesnotin().

diff --git a/mercurial/manifest.py b/mercurial/manifest.py
--- a/mercurial/manifest.py
+++ b/mercurial/manifest.py
@@ -448,8 +448,8 @@ class manifestdict(object):
 def filesnotin(self, m2, match=None):
 '''Set of files in this manifest that are not in the other'''
 if match:
-m1 = self.matches(match)
-m2 = m2.matches(match)
+m1 = self._matches(match)
+m2 = m2._matches(match)
 return m1.filesnotin(m2)
 diff = self.diff(m2)
 files = set(filepath
@@ -510,7 +510,7 @@ class manifestdict(object):
 if not self.hasdir(fn):
 match.bad(fn, None)
 
-def matches(self, match):
+def _matches(self, match):
 '''generate a new manifest filtered by the match argument'''
 if match.always():
 return self.copy()
@@ -543,8 +543,8 @@ class manifestdict(object):
 string.
 '''
 if match:
-m1 = self.matches(match)
-m2 = m2.matches(match)
+m1 = self._matches(match)
+m2 = m2._matches(match)
 return m1.diff(m2, clean=clean)
 return self._lm.diff(m2._lm, clean)
 
@@ -917,8 +917,8 @@ class treemanifest(object):
 def filesnotin(self, m2, match=None):
 '''Set of files in this manifest that are not in the other'''
 if match:
-m1 = self.matches(match)
-m2 = m2.matches(match)
+m1 = self._matches(match)
+m2 = m2._matches(match)
 return m1.filesnotin(m2)
 
 files = set()
@@ -1002,7 +1002,7 @@ class treemanifest(object):
 for f in self._dirs[p]._walk(match):
 yield f
 
-def matches(self, match):
+def _matches(self, match):
 '''generate a new manifest filtered by the match argument'''
 if match.always():
 return self.copy()
@@ -1054,8 +1054,8 @@ class treemanifest(object):
 string.
 '''
 if match:
-m1 = self.matches(match)
-m2 = m2.matches(match)
+m1 = self._matches(match)
+m2 = m2._matches(match)
 return m1.diff(m2, clean=clean)
 result = {}
 emptytree = treemanifest()
diff --git a/tests/test-manifest.py b/tests/test-manifest.py
--- a/tests/test-manifest.py
+++ b/tests/test-manifest.py
@@ -233,7 +233,7 @@ class basemanifesttests(object):
 self.assertEqual(want, m['foo'])
 # make sure the suffix survives a copy
 match = matchmod.match('', '', ['re:foo'])
-m2 = m.matches(match)
+m2 = m._matches(match)
 self.assertEqual(want, m2['foo'])
 self.assertEqual(1, len(m2))
 m2 = m.copy()
@@ -255,7 +255,7 @@ class basemanifesttests(object):
 assert False
 return True
 match.matchfn = filt
-self.assertRaises(AssertionError, m.matches, match)
+self.assertRaises(AssertionError, m._matches, match)
 
 def testRemoveItem(self):
 m = self.parsemanifest(A_SHORT_MANIFEST)
@@ -358,7 +358,7 @@ class basemanifesttests(object):
 
 match = matchmod.match('/', '',
 ['file1', 'file200', 'file300'], exact=True)
-m2 = m.matches(match)
+m2 = m._matches(match)
 
 w = ('file1\0%sx\n'
  'file200\0%sl\n'
@@ -374,7 +374,7 @@ class basemanifesttests(object):
 match = matchmod.match('/', '',
 ['a/b/c/bar.txt', 'a/b/d/qux.py', 'readme.txt', 'nonexistent'],
 exact=True)
-m2 = m.matches(match)
+m2 = m._matches(match)
 
 self.assertEqual(
 ['a/b/c/bar.txt', 'a/b/d/qux.py', 'readme.txt'],
@@ -386,7 +386,7 @@ class basemanifesttests(object):
 m = self.parsemanifest(A_DEEPER_MANIFEST)
 
 match = matchmod.match('/', '', ['a/f'], default='relpath')
-m2 = m.matches(match)
+m2 = m._matches(match)
 
 self.assertEqual([], m2.keys())
 
@@ -397,7 +397,7 @@ class basemanifesttests(object):
 
 flist = m.keys()[80:300]
 match = matchmod.match('/', '', flist, exact=True)
-m2 = m.matches(match)
+m2 = m._matches(match)
 
 self.assertEqual(flist, m2.keys())
 
@@ -406,7 +406,7 @@ class basemanifesttests(object):
 m = self.parsemanifest(A_DEEPER_MANIFEST)
 
 match = matchmod.match('/', '', [''])
-m2 = m.matches(match)
+m2 = m._

[PATCH 06 of 10 V2] context: remove duplicate manifest creation during _buildstatus

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488937790 28800
#  Tue Mar 07 17:49:50 2017 -0800
# Node ID d2850df6c891a20585a0d5eac370c1c2f4463cad
# Parent  36bcc5d848c6bdf33d604999631a0708d1b7f067
context: remove duplicate manifest creation during _buildstatus

Previously we called self.manifest() in some cases to preload the first
manifest. It turns out that self.manifest() may do extra logic that
_manifestmatches() does not, so it may be causing us extra work. The fix is to
just only do the work once.

diff --git a/mercurial/context.py b/mercurial/context.py
--- a/mercurial/context.py
+++ b/mercurial/context.py
@@ -117,10 +117,12 @@ class basectx(object):
 # 1000 and cache it so that when you read 1001, we just need to apply a
 # delta to what's in the cache. So that's one full reconstruction + one
 # delta application.
+mf2 = None
 if self.rev() is not None and self.rev() < other.rev():
-self.manifest()
+mf2 = self._manifestmatches(match, s)
 mf1 = other._manifestmatches(match, s)
-mf2 = self._manifestmatches(match, s)
+if mf2 is None:
+mf2 = self._manifestmatches(match, s)
 
 modified, added = [], []
 removed = []
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 01 of 10 V2] manifest: add match argument to diff and filesnotin

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488909371 28800
#  Tue Mar 07 09:56:11 2017 -0800
# Node ID 5ede3fa3bae6e964b4bce8b36681d203c05e94f7
# Parent  7433b3bc55eebfa9149280339b406bd4cec64efb
manifest: add match argument to diff and filesnotin

As part of removing manifest.matches (since it is O(manifest)), let's start by
adding match arguments to diff and filesnotin. As we'll see in later patches,
these are the only flows that actually use matchers, so by moving the matching
into the actual functions, other manifest implementations can make more 
efficient
algorithsm.

For instance, this will allow treemanifest diff's to only iterate over the files
that are different AND meet the match criteria.

No consumers are changed in this patches, but the code is fairly easy to verify
visually. Future patches will convert consumers to use it.

One test was affected because it did not use the kwargs version of the clean
parameter.

diff --git a/mercurial/manifest.py b/mercurial/manifest.py
--- a/mercurial/manifest.py
+++ b/mercurial/manifest.py
@@ -445,8 +445,12 @@ class manifestdict(object):
 def keys(self):
 return list(self.iterkeys())
 
-def filesnotin(self, m2):
+def filesnotin(self, m2, match=None):
 '''Set of files in this manifest that are not in the other'''
+if match:
+m1 = self.matches(match)
+m2 = m2.matches(match)
+return m1.filesnotin(m2)
 diff = self.diff(m2)
 files = set(filepath
 for filepath, hashflags in diff.iteritems()
@@ -523,7 +527,7 @@ class manifestdict(object):
 m._lm = self._lm.filtercopy(match)
 return m
 
-def diff(self, m2, clean=False):
+def diff(self, m2, match=None, clean=False):
 '''Finds changes between the current manifest and m2.
 
 Args:
@@ -538,6 +542,10 @@ class manifestdict(object):
 the nodeid will be None and the flags will be the empty
 string.
 '''
+if match:
+m1 = self.matches(match)
+m2 = m2.matches(match)
+return m1.diff(m2, clean=clean)
 return self._lm.diff(m2._lm, clean)
 
 def setflag(self, key, flag):
@@ -906,8 +914,13 @@ class treemanifest(object):
 copy._copyfunc = self._copyfunc
 return copy
 
-def filesnotin(self, m2):
+def filesnotin(self, m2, match=None):
 '''Set of files in this manifest that are not in the other'''
+if match:
+m1 = self.matches(match)
+m2 = m2.matches(match)
+return m1.filesnotin(m2)
+
 files = set()
 def _filesnotin(t1, t2):
 if t1._node == t2._node and not t1._dirty and not t2._dirty:
@@ -1025,7 +1038,7 @@ class treemanifest(object):
 ret._dirty = True
 return ret
 
-def diff(self, m2, clean=False):
+def diff(self, m2, match=None, clean=False):
 '''Finds changes between the current manifest and m2.
 
 Args:
@@ -1040,6 +1053,10 @@ class treemanifest(object):
 the nodeid will be None and the flags will be the empty
 string.
 '''
+if match:
+m1 = self.matches(match)
+m2 = m2.matches(match)
+return m1.diff(m2, clean=clean)
 result = {}
 emptytree = treemanifest()
 def _diff(t1, t2):
diff --git a/tests/test-manifest.py b/tests/test-manifest.py
--- a/tests/test-manifest.py
+++ b/tests/test-manifest.py
@@ -320,7 +320,7 @@ class basemanifesttests(object):
 'bar/baz/qux.py': None,
 'foo': (MISSING, (BIN_HASH_1, '')),
 }
-self.assertEqual(want, pruned.diff(short, True))
+self.assertEqual(want, pruned.diff(short, clean=True))
 
 def testReversedLines(self):
 backwards = ''.join(
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 5 of 6 py3] dispatch: enforce bytes when converting boolean flags to config items

[Augie Fackler]

> On Mar 7, 2017, at 22:07, Durham Goode  wrote:
> 
> 
> 
> On 3/7/17 8:25 AM, Augie Fackler wrote:
>> # HG changeset patch
>> # User Augie Fackler 
>> # Date 1488570207 18000
>> #  Fri Mar 03 14:43:27 2017 -0500
>> # Node ID 4801067dee2c77ff4e720c931d8b19cf32515beb
>> # Parent  a6e8bb19707e0c7505ccfdf44f7e1b19a0f65d48
>> dispatch: enforce bytes when converting boolean flags to config items
>> 
>> This fixes --verbose on Python 3.
>> 
>> diff --git a/mercurial/dispatch.py b/mercurial/dispatch.py
>> --- a/mercurial/dispatch.py
>> +++ b/mercurial/dispatch.py
>> @@ -744,6 +744,8 @@ def _dispatch(req):
>> if options['verbose'] or options['debug'] or options['quiet']:
>> for opt in ('verbose', 'debug', 'quiet'):
>> val = str(bool(options[opt]))
>> +if pycompat.ispy3:
>> +val = val.encode('latin1')
> 
> Should we have a util function for turning str() output into bytes? Or event 
> a strbytes() function?  On py2 it could just return str.  My encoding 
> knowledge is approximately zero, which is why I'd love to be able to choose 
> from some easy functions like `util.tobytesfromstr()` instead of knowing that 
> encode('latin1') is how I get ascii bytes.

I'm not sure - in this case I knew latin1 was safe because it's the repr() of a 
bool, but I don't know if there's a general-purpose solution possible here. 
Yuya might have an idea though?
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 5 of 6 py3] dispatch: enforce bytes when converting boolean flags to config items

[Durham Goode]

On 3/7/17 8:25 AM, Augie Fackler wrote:

# HG changeset patch
# User Augie Fackler 
# Date 1488570207 18000
#  Fri Mar 03 14:43:27 2017 -0500
# Node ID 4801067dee2c77ff4e720c931d8b19cf32515beb
# Parent  a6e8bb19707e0c7505ccfdf44f7e1b19a0f65d48
dispatch: enforce bytes when converting boolean flags to config items

This fixes --verbose on Python 3.

diff --git a/mercurial/dispatch.py b/mercurial/dispatch.py
--- a/mercurial/dispatch.py
+++ b/mercurial/dispatch.py
@@ -744,6 +744,8 @@ def _dispatch(req):
 if options['verbose'] or options['debug'] or options['quiet']:
 for opt in ('verbose', 'debug', 'quiet'):
 val = str(bool(options[opt]))
+if pycompat.ispy3:
+val = val.encode('latin1')


Should we have a util function for turning str() output into bytes? Or 
event a strbytes() function?  On py2 it could just return str.  My 
encoding knowledge is approximately zero, which is why I'd love to be 
able to choose from some easy functions like `util.tobytesfromstr()` 
instead of knowing that encode('latin1') is how I get ascii bytes.

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 3 of 6 py3] extensions: tapdance to get reasonable import error formatting

[Augie Fackler]

> On Mar 7, 2017, at 21:59, Durham Goode  wrote:
> 
> 
> 
> On 3/7/17 8:25 AM, Augie Fackler wrote:
>> # HG changeset patch
>> # User Augie Fackler 
>> # Date 1488568082 18000
>> #  Fri Mar 03 14:08:02 2017 -0500
>> # Node ID adf7419abb89aef332ac5eac48a0f1c9d3eab527
>> # Parent  3e82376d7d3f3e11d11f09fceb8b4c79233057a8
>> extensions: tapdance to get reasonable import error formatting
>> 
>> I'm not thrilled with this, but it seems to work.
>> 
>> diff --git a/mercurial/extensions.py b/mercurial/extensions.py
>> --- a/mercurial/extensions.py
>> +++ b/mercurial/extensions.py
>> @@ -18,6 +18,7 @@ from .i18n import (
>> 
>> from . import (
>> cmdutil,
>> +encoding,
>> error,
>> pycompat,
>> util,
>> @@ -104,11 +105,19 @@ def _importext(name, path=None, reportfu
>> mod = _importh(name)
>> return mod
>> 
>> +def _forbytes(inst):
>> +"""Portably format an import error into a form suitable for
>> +%-formatting into bytestrings."""
>> +if pycompat.ispy3:
>> +return str(inst).encode('utf-8')
>> +return inst
>> +
>> def _reportimporterror(ui, err, failed, next):
>> # note: this ui.debug happens before --debug is processed,
>> #   Use --config ui.debug=1 to see them.
>> -ui.debug('could not import %s (%s): trying %s\n'
>> - % (failed, err, next))
>> +msg = 'could not import %s (%s): trying %s\n' % (
>> +failed, _forbytes(err), next)
>> +ui.debug(encoding.tolocal(msg))
>> if ui.debugflag:
>> ui.traceback()
>> 
>> @@ -168,12 +177,13 @@ def loadall(ui):
>> except KeyboardInterrupt:
>> raise
>> except Exception as inst:
>> +inst = _forbytes(inst)
>> if path:
>> -ui.warn(_("*** failed to import extension %s from %s: %s\n")
>> -% (name, path, inst))
>> +fmt = _("*** failed to import extension %s from %s: %s\n")
>> +ui.warn(encoding.tolocal(fmt % (name, path, inst)))
>> else:
>> -ui.warn(_("*** failed to import extension %s: %s\n")
>> -% (name, inst))
>> +fmt = _("*** failed to import extension %s: %s\n")
>> +ui.warn(encoding.tolocal(fmt % (name, inst)))
>> ui.traceback()
>> 
>> for name in _order[newindex:]:
>> diff --git a/tests/test-check-py3-commands.t 
>> b/tests/test-check-py3-commands.t
>> --- a/tests/test-check-py3-commands.t
>> +++ b/tests/test-check-py3-commands.t
>> @@ -12,3 +12,12 @@ The full traceback is hidden to have a s
>>   warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
>>   debuginstall
>>   TypeError: Can't convert 'bytes' object to str implicitly
>> +
>> +  $ cat > included-hgrc <> +  > [extensions]
>> +  > babar = imaginary_elephant
>> +  > EOF
>> +  $ cat >> $HGRCPATH <> +  > %include $TESTTMP/included-hgrc
>> +  > EOF
>> +  $ $PYTHON3 `which hg` version --config ui.debug=1
> 
> Should this have outputted an error?

D'oh. I must have missed something here. I'll take a look in the morning (it's 
late here) and do a resend.

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 3 of 6 py3] extensions: tapdance to get reasonable import error formatting

[Durham Goode]

On 3/7/17 6:59 PM, Durham Goode wrote:



On 3/7/17 8:25 AM, Augie Fackler wrote:

# HG changeset patch
# User Augie Fackler 
# Date 1488568082 18000
#  Fri Mar 03 14:08:02 2017 -0500
# Node ID adf7419abb89aef332ac5eac48a0f1c9d3eab527
# Parent  3e82376d7d3f3e11d11f09fceb8b4c79233057a8
extensions: tapdance to get reasonable import error formatting

I'm not thrilled with this, but it seems to work.

diff --git a/mercurial/extensions.py b/mercurial/extensions.py
--- a/mercurial/extensions.py
+++ b/mercurial/extensions.py
@@ -18,6 +18,7 @@ from .i18n import (

 from . import (
 cmdutil,
+encoding,
 error,
 pycompat,
 util,
@@ -104,11 +105,19 @@ def _importext(name, path=None, reportfu
 mod = _importh(name)
 return mod

+def _forbytes(inst):
+"""Portably format an import error into a form suitable for
+%-formatting into bytestrings."""
+if pycompat.ispy3:
+return str(inst).encode('utf-8')
+return inst
+
 def _reportimporterror(ui, err, failed, next):
 # note: this ui.debug happens before --debug is processed,
 #   Use --config ui.debug=1 to see them.
-ui.debug('could not import %s (%s): trying %s\n'
- % (failed, err, next))
+msg = 'could not import %s (%s): trying %s\n' % (
+failed, _forbytes(err), next)
+ui.debug(encoding.tolocal(msg))
 if ui.debugflag:
 ui.traceback()

@@ -168,12 +177,13 @@ def loadall(ui):
 except KeyboardInterrupt:
 raise
 except Exception as inst:
+inst = _forbytes(inst)
 if path:
-ui.warn(_("*** failed to import extension %s from %s:
%s\n")
-% (name, path, inst))
+fmt = _("*** failed to import extension %s from %s:
%s\n")
+ui.warn(encoding.tolocal(fmt % (name, path, inst)))
 else:
-ui.warn(_("*** failed to import extension %s: %s\n")
-% (name, inst))
+fmt = _("*** failed to import extension %s: %s\n")
+ui.warn(encoding.tolocal(fmt % (name, inst)))
 ui.traceback()

 for name in _order[newindex:]:
diff --git a/tests/test-check-py3-commands.t
b/tests/test-check-py3-commands.t
--- a/tests/test-check-py3-commands.t
+++ b/tests/test-check-py3-commands.t
@@ -12,3 +12,12 @@ The full traceback is hidden to have a s
   warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.
   debuginstall
   TypeError: Can't convert 'bytes' object to str implicitly
+
+  $ cat > included-hgrc < [extensions]
+  > babar = imaginary_elephant
+  > EOF
+  $ cat >> $HGRCPATH < %include $TESTTMP/included-hgrc
+  > EOF
+  $ $PYTHON3 `which hg` version --config ui.debug=1


Should this have outputted an error?


Er, I see the next patch has the error?  Did this file belong in the 
next patch?

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 3 of 6 py3] extensions: tapdance to get reasonable import error formatting

[Durham Goode]

On 3/7/17 8:25 AM, Augie Fackler wrote:

# HG changeset patch
# User Augie Fackler 
# Date 1488568082 18000
#  Fri Mar 03 14:08:02 2017 -0500
# Node ID adf7419abb89aef332ac5eac48a0f1c9d3eab527
# Parent  3e82376d7d3f3e11d11f09fceb8b4c79233057a8
extensions: tapdance to get reasonable import error formatting

I'm not thrilled with this, but it seems to work.

diff --git a/mercurial/extensions.py b/mercurial/extensions.py
--- a/mercurial/extensions.py
+++ b/mercurial/extensions.py
@@ -18,6 +18,7 @@ from .i18n import (

 from . import (
 cmdutil,
+encoding,
 error,
 pycompat,
 util,
@@ -104,11 +105,19 @@ def _importext(name, path=None, reportfu
 mod = _importh(name)
 return mod

+def _forbytes(inst):
+"""Portably format an import error into a form suitable for
+%-formatting into bytestrings."""
+if pycompat.ispy3:
+return str(inst).encode('utf-8')
+return inst
+
 def _reportimporterror(ui, err, failed, next):
 # note: this ui.debug happens before --debug is processed,
 #   Use --config ui.debug=1 to see them.
-ui.debug('could not import %s (%s): trying %s\n'
- % (failed, err, next))
+msg = 'could not import %s (%s): trying %s\n' % (
+failed, _forbytes(err), next)
+ui.debug(encoding.tolocal(msg))
 if ui.debugflag:
 ui.traceback()

@@ -168,12 +177,13 @@ def loadall(ui):
 except KeyboardInterrupt:
 raise
 except Exception as inst:
+inst = _forbytes(inst)
 if path:
-ui.warn(_("*** failed to import extension %s from %s: %s\n")
-% (name, path, inst))
+fmt = _("*** failed to import extension %s from %s: %s\n")
+ui.warn(encoding.tolocal(fmt % (name, path, inst)))
 else:
-ui.warn(_("*** failed to import extension %s: %s\n")
-% (name, inst))
+fmt = _("*** failed to import extension %s: %s\n")
+ui.warn(encoding.tolocal(fmt % (name, inst)))
 ui.traceback()

 for name in _order[newindex:]:
diff --git a/tests/test-check-py3-commands.t b/tests/test-check-py3-commands.t
--- a/tests/test-check-py3-commands.t
+++ b/tests/test-check-py3-commands.t
@@ -12,3 +12,12 @@ The full traceback is hidden to have a s
   warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   debuginstall
   TypeError: Can't convert 'bytes' object to str implicitly
+
+  $ cat > included-hgrc < [extensions]
+  > babar = imaginary_elephant
+  > EOF
+  $ cat >> $HGRCPATH < %include $TESTTMP/included-hgrc
+  > EOF
+  $ $PYTHON3 `which hg` version --config ui.debug=1


Should this have outputted an error?
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: stable ordering of test output

[Matt Harbison]

On Fri, 03 Mar 2017 17:45:56 -0500, Danek Duvall   
wrote:



I frequently get failures like this:

--- .../mercurial.hg/tests/test-bundle2-exchange.t
+++ .../mercurial.hg/tests/test-bundle2-exchange.t.err
@@ -1042,11 +1042,11 @@
   $ hg --config devel.legacy.exchange=bundle1 clone  
ssh://user@dummy/bundle2onlyserver not-bundle2-ssh

   requesting all changes
   adding changesets
-  remote: abort: incompatible Mercurial client; bundle2 required
-  remote: (see  
https://www.mercurial-scm.org/wiki/IncompatibleClient)

   transaction abort!
   rollback completed
   abort: stream ended unexpectedly (got 0 bytes, expected 4)
+  remote: abort: incompatible Mercurial client; bundle2 required
+  remote: (see  
https://www.mercurial-scm.org/wiki/IncompatibleClient)

   [255]
  $ cat > bundle2onlyserver/.hg/hgrc << EOF

ERROR: test-bundle2-exchange.t output changed

It's usually fairly consistent, at least for a period of time, and then  
it

goes away.  Presumably it's some sort of fairly stable timing issue, and
possibly unique to the environment I'm running in (at least, I assume  
that

the official tests aren't showing this).


The symptoms seem similar to what was happening on Windows a few years ago.

https://www.mercurial-scm.org/repo/hg/rev/83f6c4733ecc
https://www.mercurial-scm.org/repo/hg/rev/2abbf4750915

The commit referenced in the first link's message might also be of  
interest.


I could patch the tests locally to reorder the lines, but if it's really  
an

environmental issue, then that's guaranteed to work consistently even for
me.

Is this (ever? frequently?) an issue for anyone else?

I can't think of any particularly satisfying solution to this, other than
perhaps separating the remote lines from the local lines, and comparing
each of those independently.  Would that make sense?

Thanks,
Danek
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] fsmonitor: remove use of repo.opener

[Augie Fackler]

On Tue, Mar 07, 2017 at 12:53:25PM -0800, Durham Goode wrote:
> # HG changeset patch
> # User Durham Goode 
> # Date 1488919920 28800
> #  Tue Mar 07 12:52:00 2017 -0800
> # Node ID 4d485e46a3e5c83ff8d2ae0a4ba8e484a284f65b
> # Parent  7433b3bc55eebfa9149280339b406bd4cec64efb
> fsmonitor: remove use of repo.opener

Queued, thanks.

>
> This has been deprecated, so we need to switch to the appropriate vfs apis.
>
> diff --git a/hgext/fsmonitor/state.py b/hgext/fsmonitor/state.py
> --- a/hgext/fsmonitor/state.py
> +++ b/hgext/fsmonitor/state.py
> @@ -20,7 +20,7 @@ from mercurial import pathutil
>
>  class state(object):
>  def __init__(self, repo):
> -self._opener = repo.opener
> +self._vfs = repo.vfs
>  self._ui = repo.ui
>  self._rootdir = pathutil.normasprefix(repo.root)
>  self._lastclock = None
> @@ -33,7 +33,7 @@ class state(object):
>
>  def get(self):
>  try:
> -file = self._opener('fsmonitor.state', 'rb')
> +file = self._vfs('fsmonitor.state', 'rb')
>  except IOError as inst:
>  if inst.errno != errno.ENOENT:
>  raise
> @@ -91,7 +91,7 @@ class state(object):
>  return
>
>  try:
> -file = self._opener('fsmonitor.state', 'wb', atomictemp=True)
> +file = self._vfs('fsmonitor.state', 'wb', atomictemp=True)
>  except (IOError, OSError):
>  self._ui.warn(_("warning: unable to write out fsmonitor 
> state\n"))
>  return
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 3 of 6] merge: remove uses of manifest.matches

[Durham Goode]

On 3/7/17 4:41 PM, Martin von Zweigbergk wrote:

On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:

# HG changeset patch
# User Durham Goode 
# Date 1488519936 28800
#  Thu Mar 02 21:45:36 2017 -0800
# Node ID 883bb49a3b40609074d56257aab7619f0c306efc
# Parent  4cebdd029399cf7c3b0fff73faf1f41af0e895d1
merge: remove uses of manifest.matches

This gets rid of the manifest.matches calls in merge.py in favor of the new api.
This is part of getting rid of manifest.matches since it is O(manifest).

diff --git a/mercurial/merge.py b/mercurial/merge.py
--- a/mercurial/merge.py
+++ b/mercurial/merge.py
@@ -818,11 +818,7 @@ def manifestmerge(repo, wctx, p2, pa, br
 if any(wctx.sub(s).dirty() for s in wctx.substate):
 m1['.hgsubstate'] = modifiednodeid

-# Compare manifests
-if matcher is not None:
-m1 = m1.matches(matcher)
-m2 = m2.matches(matcher)
-diff = m1.diff(m2)
+diff = m1.diff(m2, match=matcher)

 actions = {}
 for f, ((n1, fl1), (n2, fl2)) in diff.iteritems():
@@ -858,7 +854,7 @@ def manifestmerge(repo, wctx, p2, pa, br
 pass # we'll deal with it on m2 side
 elif f in movewithdir: # directory rename, move local
 f2 = movewithdir[f]
-if f2 in m2:
+if f2 in m2 and (not matcher or matcher(f2)):


Would it be easier to set matcher = match.always(...) (or however you
create such matchers)  when matcher is None, so we don't have to check
in 4 places?


I figured an `if None` check was faster than a match.always() check, 
especially since this is a hotpath. I could change it though.



Also, will it be better to swap the order of the conditions to "if
matcher(f2) and f2 in m2" to avoid loading some treemanifests if
matcher(f2) is false? I guess we can change that later if it turns out
to be better.


Hmm, yes.


(For this case the lazy manifest.match() approach would have been mean
cleaner. It's a little unfortunate that we have to remember to check
the matcher ever time, but I don't see a better way of doing it.)


The ugliness of checking it 4 times here is because I'm retrofitting a 
piece of code that wasn't designed with this in mind and I was trying to 
avoid any semantic changes. I'd expect code that was built to expect a 
matcher would not be so ugly.



 actions[f2] = ('m', (f, f2, None, True, pa.node()),
"remote directory rename, both created")
 else:
@@ -887,7 +883,7 @@ def manifestmerge(repo, wctx, p2, pa, br
 pass # we'll deal with it on m1 side
 elif f in movewithdir:
 f2 = movewithdir[f]
-if f2 in m1:
+if f2 in m1 and (not matcher or matcher(f2)):
 actions[f2] = ('m', (f2, f, None, False, pa.node()),
"local directory rename, both created")
 else:
@@ -895,7 +891,7 @@ def manifestmerge(repo, wctx, p2, pa, br
"local directory rename - get from " + f)
 elif f in copy:
 f2 = copy[f]
-if f2 in m2:
+if f2 in m2 and (not matcher or matcher(f2)):
 actions[f] = ('m', (f2, f, f2, False, pa.node()),
   "remote copied from " + f2)
 else:
@@ -927,7 +923,7 @@ def manifestmerge(repo, wctx, p2, pa, br
 # new file added in a directory that was moved
 df = dirmove[d] + f[len(d):]
 break
-if df in m1:
+if df in m1 and (not matcher or matcher(df)):
 actions[df] = ('m', (df, f, f, False, pa.node()),
 "local directory rename - respect move from " + f)
 elif acceptremote:
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=nuarHzhP1wi1T9iURRCj1A&m=kiGdgxyWqhs0Cu0YjkuskFVobqCWr4TIOthkrkxFU5w&s=4vcpzhVAiK7FHCfzH2C5FUDNbQkM0oP5DXa9g8h-9Dk&e=

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 5 of 6] context: remove uses of manifest.matches

[Martin von Zweigbergk via Mercurial-devel]

On Tue, Mar 7, 2017 at 4:13 PM, Durham Goode  wrote:
>
>
> On 3/7/17 2:40 PM, Martin von Zweigbergk wrote:
>>
>> On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:
>>>
>>> # HG changeset patch
>>> # User Durham Goode 
>>> # Date 1488569391 28800
>>> #  Fri Mar 03 11:29:51 2017 -0800
>>> # Node ID 207763e895c7d24885df22f5b9c0df5494d77daf
>>> # Parent  78e0fb2bd1bc972444ae08e7b7165da66cbf53a3
>>> context: remove uses of manifest.matches
>>>
>>> This removes the uses of manifest.matches in context.py in favor of the
>>> new api.
>>> This is part of removing manifest.matches since it is O(manifest).
>>>
>>> diff --git a/mercurial/context.py b/mercurial/context.py
>>> --- a/mercurial/context.py
>>> +++ b/mercurial/context.py
>>> @@ -18,7 +18,6 @@ from .node import (
>>>  bin,
>>>  hex,
>>>  modifiednodeid,
>>> -newnodeid,
>>>  nullid,
>>>  nullrev,
>>>  short,
>>> @@ -91,15 +90,6 @@ class basectx(object):
>>>  def __iter__(self):
>>>  return iter(self._manifest)
>>>
>>> -def _manifestmatches(self, match, s):
>>> -"""generate a new manifest filtered by the match argument
>>> -
>>> -This method is for internal use only and mainly exists to
>>> provide an
>>> -object oriented way for other contexts to customize the manifest
>>> -generation.
>>> -"""
>>> -return self.manifest().matches(match)
>>> -
>>>  def _matchstatus(self, other, match):
>>>  """return match.always if match is none
>>>
>>> @@ -119,15 +109,15 @@ class basectx(object):
>>>  # delta application.
>>>  if self.rev() is not None and self.rev() < other.rev():
>>>  self.manifest()
>>> -mf1 = other._manifestmatches(match, s)
>>> -mf2 = self._manifestmatches(match, s)
>>> +mf1 = other.manifest()
>>> +mf2 = self.manifest()
>>>
>>>  modified, added = [], []
>>>  removed = []
>>>  clean = []
>>>  deleted, unknown, ignored = s.deleted, s.unknown, s.ignored
>>>  deletedset = set(deleted)
>>> -d = mf1.diff(mf2, clean=listclean)
>>> +d = mf1.diff(mf2, match=match, clean=listclean)
>>>  for fn, value in d.iteritems():
>>>  if fn in deletedset:
>>>  continue
>>> @@ -154,8 +144,10 @@ class basectx(object):
>>>
>>>  if removed:
>>>  # need to filter files if they are already reported as
>>> removed
>>> -unknown = [fn for fn in unknown if fn not in mf1]
>>> -ignored = [fn for fn in ignored if fn not in mf1]
>>> +unknown = [fn for fn in unknown if fn not in mf1 and
>>> +   (not match or match(fn))]
>>> +ignored = [fn for fn in ignored if fn not in mf1 and
>>> +   (not match or match(fn))]
>>>  # if they're deleted, don't report them as removed
>>>  removed = [fn for fn in removed if fn not in deletedset]
>>>
>>> @@ -1608,22 +1600,6 @@ class workingctx(committablectx):
>>>  pass
>>>  return modified, fixup
>>>
>>> -def _manifestmatches(self, match, s):
>>> -"""Slow path for workingctx
>>> -
>>> -The fast path is when we compare the working directory to its
>>> parent
>>> -which means this function is comparing with a non-parent;
>>> therefore we
>>> -need to build a manifest and return what matches.
>>> -"""
>>> -mf = self._repo['.']._manifestmatches(match, s)
>>> -for f in s.modified + s.added:
>>> -mf[f] = newnodeid
>>> -mf.setflag(f, self.flags(f))
>>> -for f in s.removed:
>>> -if f in mf:
>>> -del mf[f]
>>> -return mf
>>> -
>>
>>
>> Nice! But could you explain why this is no longer needed? Is it
>> completely thanks to the new manifest.diff() API or something else?
>
>
> My original reason was because self._manifest() could already generate this
> manifest (and does a more accurate job of it than this function), and since
> the matcher was moved to the diff layer, nothing here would be lost by
> switching to self._manifest().
>
> Upon looking deeper, it looks like the big benefit here is that the status
> is provided, and therefore we don't have to call status a second time, (i.e.
> when self._manifest calls 'self._status()'). This has the additional benefit
> that the matcher can be applied to the status output early.  But this only
> applies when diffing the working copy against a commit that is not the
> parent of the working copy, so a somewhat niche case.

Thanks for checking.

>
> I have some ideas about how I could refactor this area to address this, so
> I'll include them in V2. It might make the series 10-12 patches long though.

That sounds fine to me.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://ww

Re: [PATCH 3 of 6] merge: remove uses of manifest.matches

[Martin von Zweigbergk via Mercurial-devel]

On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:
> # HG changeset patch
> # User Durham Goode 
> # Date 1488519936 28800
> #  Thu Mar 02 21:45:36 2017 -0800
> # Node ID 883bb49a3b40609074d56257aab7619f0c306efc
> # Parent  4cebdd029399cf7c3b0fff73faf1f41af0e895d1
> merge: remove uses of manifest.matches
>
> This gets rid of the manifest.matches calls in merge.py in favor of the new 
> api.
> This is part of getting rid of manifest.matches since it is O(manifest).
>
> diff --git a/mercurial/merge.py b/mercurial/merge.py
> --- a/mercurial/merge.py
> +++ b/mercurial/merge.py
> @@ -818,11 +818,7 @@ def manifestmerge(repo, wctx, p2, pa, br
>  if any(wctx.sub(s).dirty() for s in wctx.substate):
>  m1['.hgsubstate'] = modifiednodeid
>
> -# Compare manifests
> -if matcher is not None:
> -m1 = m1.matches(matcher)
> -m2 = m2.matches(matcher)
> -diff = m1.diff(m2)
> +diff = m1.diff(m2, match=matcher)
>
>  actions = {}
>  for f, ((n1, fl1), (n2, fl2)) in diff.iteritems():
> @@ -858,7 +854,7 @@ def manifestmerge(repo, wctx, p2, pa, br
>  pass # we'll deal with it on m2 side
>  elif f in movewithdir: # directory rename, move local
>  f2 = movewithdir[f]
> -if f2 in m2:
> +if f2 in m2 and (not matcher or matcher(f2)):

Would it be easier to set matcher = match.always(...) (or however you
create such matchers)  when matcher is None, so we don't have to check
in 4 places?

Also, will it be better to swap the order of the conditions to "if
matcher(f2) and f2 in m2" to avoid loading some treemanifests if
matcher(f2) is false? I guess we can change that later if it turns out
to be better.

(For this case the lazy manifest.match() approach would have been mean
cleaner. It's a little unfortunate that we have to remember to check
the matcher ever time, but I don't see a better way of doing it.)

>  actions[f2] = ('m', (f, f2, None, True, pa.node()),
> "remote directory rename, both created")
>  else:
> @@ -887,7 +883,7 @@ def manifestmerge(repo, wctx, p2, pa, br
>  pass # we'll deal with it on m1 side
>  elif f in movewithdir:
>  f2 = movewithdir[f]
> -if f2 in m1:
> +if f2 in m1 and (not matcher or matcher(f2)):
>  actions[f2] = ('m', (f2, f, None, False, pa.node()),
> "local directory rename, both created")
>  else:
> @@ -895,7 +891,7 @@ def manifestmerge(repo, wctx, p2, pa, br
> "local directory rename - get from " + f)
>  elif f in copy:
>  f2 = copy[f]
> -if f2 in m2:
> +if f2 in m2 and (not matcher or matcher(f2)):
>  actions[f] = ('m', (f2, f, f2, False, pa.node()),
>"remote copied from " + f2)
>  else:
> @@ -927,7 +923,7 @@ def manifestmerge(repo, wctx, p2, pa, br
>  # new file added in a directory that was moved
>  df = dirmove[d] + f[len(d):]
>  break
> -if df in m1:
> +if df in m1 and (not matcher or matcher(df)):
>  actions[df] = ('m', (df, f, f, False, pa.node()),
>  "local directory rename - respect move from " + 
> f)
>  elif acceptremote:
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 1 of 5 V2] rebase: clear updatestate during rebase --abort in more cases

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488925148 28800
#  Tue Mar 07 14:19:08 2017 -0800
# Node ID 4eb06495f778ecce2b3644e0d50585495f26162b
# Parent  7433b3bc55eebfa9149280339b406bd4cec64efb
rebase: clear updatestate during rebase --abort in more cases

Previously, rebase --abort would only call update if you were on a node that had
already been rebased. This meant that if the rebase failed during the rebase of
the first commit, the working copy would be left dirty (with a .hg/updatestate
file) and rebase --abort would not have update to clean it up.

The fix is to also perform an update if you're still on the target node or on
the original working copy node (since the working copy may be dirty, we still
need to do the update). We don't want to perform an update in all cases though
because of issue4009.

A subsequent patch makes this case much more common, since it causes the entire
rebase transaction to rollback during unexpected exceptions. This causes the
existing test-rebase-abort.t to cover this case.

diff --git a/hgext/rebase.py b/hgext/rebase.py
--- a/hgext/rebase.py
+++ b/hgext/rebase.py
@@ -1156,8 +1156,11 @@ def abort(repo, originalwd, target, stat
 if rebased:
 strippoints = [
 c.node() for c in repo.set('roots(%ld)', rebased)]
-shouldupdate = len([
-c.node() for c in repo.set('. & (%ld)', rebased)]) > 0
+
+updateifonnodes = set(rebased)
+updateifonnodes.add(target)
+updateifonnodes.add(originalwd)
+shouldupdate = repo['.'].rev() in updateifonnodes
 
 # Update away from the rebase if necessary
 if shouldupdate or needupdate(repo, state):
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 2 of 5 V2] rebase: move storestatus onto rebaseruntime

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488924704 28800
#  Tue Mar 07 14:11:44 2017 -0800
# Node ID 9ee513d4a3884d19a85b2aadd7bac362d1530d11
# Parent  4eb06495f778ecce2b3644e0d50585495f26162b
rebase: move storestatus onto rebaseruntime

The rebaseruntime class already has the restorestatus function, so let's make it
own the store status function too. This get's rid of a lot of unnecessary
argument passing and will make a future patch cleaner that refactors storestatus
to support transactions.

diff --git a/hgext/rebase.py b/hgext/rebase.py
--- a/hgext/rebase.py
+++ b/hgext/rebase.py
@@ -159,6 +159,31 @@ class rebaseruntime(object):
 self.keepopen = opts.get('keepopen', False)
 self.obsoletenotrebased = {}
 
+def storestatus(self):
+"""Store the current status to allow recovery"""
+repo = self.repo
+f = repo.vfs("rebasestate", "w")
+f.write(repo[self.originalwd].hex() + '\n')
+f.write(repo[self.target].hex() + '\n')
+f.write(repo[self.external].hex() + '\n')
+f.write('%d\n' % int(self.collapsef))
+f.write('%d\n' % int(self.keepf))
+f.write('%d\n' % int(self.keepbranchesf))
+f.write('%s\n' % (self.activebookmark or ''))
+for d, v in self.state.iteritems():
+oldrev = repo[d].hex()
+if v >= 0:
+newrev = repo[v].hex()
+elif v == revtodo:
+# To maintain format compatibility, we have to use nullid.
+# Please do remove this special case when upgrading the format.
+newrev = hex(nullid)
+else:
+newrev = v
+f.write("%s:%s\n" % (oldrev, newrev))
+f.close()
+repo.ui.debug('rebase status stored\n')
+
 def restorestatus(self):
 """Restore a previously stored status"""
 repo = self.repo
@@ -358,10 +383,7 @@ class rebaseruntime(object):
  self.state,
  self.targetancestors,
  self.obsoletenotrebased)
-storestatus(repo, self.originalwd, self.target,
-self.state, self.collapsef, self.keepf,
-self.keepbranchesf, self.external,
-self.activebookmark)
+self.storestatus()
 storecollapsemsg(repo, self.collapsemsg)
 if len(repo[None].parents()) == 2:
 repo.ui.debug('resuming interrupted rebase\n')
@@ -1076,31 +1098,6 @@ def restorecollapsemsg(repo):
 raise error.Abort(_('no rebase in progress'))
 return collapsemsg
 
-def storestatus(repo, originalwd, target, state, collapse, keep, keepbranches,
-external, activebookmark):
-'Store the current status to allow recovery'
-f = repo.vfs("rebasestate", "w")
-f.write(repo[originalwd].hex() + '\n')
-f.write(repo[target].hex() + '\n')
-f.write(repo[external].hex() + '\n')
-f.write('%d\n' % int(collapse))
-f.write('%d\n' % int(keep))
-f.write('%d\n' % int(keepbranches))
-f.write('%s\n' % (activebookmark or ''))
-for d, v in state.iteritems():
-oldrev = repo[d].hex()
-if v >= 0:
-newrev = repo[v].hex()
-elif v == revtodo:
-# To maintain format compatibility, we have to use nullid.
-# Please do remove this special case when upgrading the format.
-newrev = hex(nullid)
-else:
-newrev = v
-f.write("%s:%s\n" % (oldrev, newrev))
-f.close()
-repo.ui.debug('rebase status stored\n')
-
 def clearstatus(repo):
 'Remove the status files'
 _clearrebasesetvisibiliy(repo)
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 4 of 5 V2] rebase: allow aborting if last-message.txt is missing

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488933031 28800
#  Tue Mar 07 16:30:31 2017 -0800
# Node ID 177c391caab3912e403e0eec469ea45a18c001f2
# Parent  8451bfc2e1d6260a0ce9dac505dd9f34fb3b19aa
rebase: allow aborting if last-message.txt is missing

Previously, if .hg/rebasestate existed but .hg/last-message.txt was missing, 'hg
rebase --abort' would say there's no rebase in progress but 'hg checkout foo'
would say 'abort: rebase in progress'. It turns out loading the collapse message
will throw a "no rebase in progress" error if the file doesn't exist, even
though .hg/rebasestate obviously indicates a rebase is in progress.

The fix is to only throw an exception if we're trying to --continue, and to just
eat the issues if we're doing --abort.

This issue is exposed by us writing the rebase state earlier in the process.
This will be used by later patches to ensure the user can appropriately 'hg
rebase --abort' if there's a crash before the first the first commit has
finished rebasing. Tests cover all of this. The only negative affect is we now
require a hg rebase --abort in a very specific exception case, as shown in the
test.

diff --git a/hgext/rebase.py b/hgext/rebase.py
--- a/hgext/rebase.py
+++ b/hgext/rebase.py
@@ -283,7 +283,7 @@ class rebaseruntime(object):
 def _prepareabortorcontinue(self, isabort):
 try:
 self.restorestatus()
-self.collapsemsg = restorecollapsemsg(self.repo)
+self.collapsemsg = restorecollapsemsg(self.repo, isabort)
 except error.RepoLookupError:
 if isabort:
 clearstatus(self.repo)
@@ -369,6 +369,10 @@ class rebaseruntime(object):
 if self.activebookmark:
 bookmarks.deactivate(repo)
 
+# Store the state before we begin so users can run 'hg rebase --abort'
+# if we fail before the transaction closes.
+self.storestatus()
+
 sortedrevs = repo.revs('sort(%ld, -topo)', self.state)
 cands = [k for k, v in self.state.iteritems() if v == revtodo]
 total = len(cands)
@@ -1092,7 +1096,7 @@ def clearcollapsemsg(repo):
 'Remove collapse message file'
 util.unlinkpath(repo.join("last-message.txt"), ignoremissing=True)
 
-def restorecollapsemsg(repo):
+def restorecollapsemsg(repo, isabort):
 'Restore previously stored collapse message'
 try:
 f = repo.vfs("last-message.txt")
@@ -1101,7 +1105,11 @@ def restorecollapsemsg(repo):
 except IOError as err:
 if err.errno != errno.ENOENT:
 raise
-raise error.Abort(_('no rebase in progress'))
+if isabort:
+# Oh well, just abort like normal
+collapsemsg = ''
+else:
+raise error.Abort(_('missing .hg/last-message.txt for rebase'))
 return collapsemsg
 
 def clearstatus(repo):
diff --git a/tests/test-rebase-conflicts.t b/tests/test-rebase-conflicts.t
--- a/tests/test-rebase-conflicts.t
+++ b/tests/test-rebase-conflicts.t
@@ -218,6 +218,7 @@ Check that the right ancestors is used w
   
   $ hg rebase -s9 -d2 --debug # use debug to really check merge base used
   rebase onto 4bc80088dc6b starting from e31216eec445
+  rebase status stored
   ignoring null merge rebase of 3
   ignoring null merge rebase of 4
   ignoring null merge rebase of 6
diff --git a/tests/test-rebase-scenario-global.t 
b/tests/test-rebase-scenario-global.t
--- a/tests/test-rebase-scenario-global.t
+++ b/tests/test-rebase-scenario-global.t
@@ -272,7 +272,8 @@ G onto B - merge revision with both pare
   rebasing 6:eea13746799a "G"
   abort: cannot use revision 6 as base, result would have 3 parents
   [255]
-
+  $ hg rebase --abort
+  rebase aborted
 
 These will abort gracefully (using --base):
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 5 of 5 V2] rebase: move actual rebase into a single transaction

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488932852 28800
#  Tue Mar 07 16:27:32 2017 -0800
# Node ID 39116126d70a5cc2fe330c880093beffa640aa64
# Parent  177c391caab3912e403e0eec469ea45a18c001f2
rebase: move actual rebase into a single transaction

Previously, rebasing would open several transaction over the course of rebasing
several commits. Opening a transaction can have notable overhead (like copying
the dirstate) which can add up when rebasing many commits.

This patch adds a single large transaction around the actual commit rebase
operation, with a catch for intervention which serializes the current state if
we need to drop back to the terminal for user intervention. Amazingly, almost
all the tests seem to pass.

On large repos with large working copies, this can speed up rebasing 7 commits
by 25%. I'd expect the percentage to be a bit larger for rebasing even more
commits.

There are minor test changes because we're rolling back the entire transaction
during unexpected exceptions instead of just stopping mid-rebase, so there's no
more backup bundle. It also leave an unknown file in the working copy, since our
clean up 'hg update' doesn't delete unknown files.

diff --git a/hgext/rebase.py b/hgext/rebase.py
--- a/hgext/rebase.py
+++ b/hgext/rebase.py
@@ -343,7 +343,7 @@ class rebaseruntime(object):
 if dest.closesbranch() and not self.keepbranchesf:
 self.ui.status(_('reopening closed branch head %s\n') % dest)
 
-def _performrebase(self):
+def _performrebase(self, tr):
 repo, ui, opts = self.repo, self.ui, self.opts
 if self.keepbranchesf:
 # insert _savebranch at the start of extrafns so if
@@ -393,7 +393,7 @@ class rebaseruntime(object):
  self.state,
  self.targetancestors,
  self.obsoletenotrebased)
-self.storestatus()
+self.storestatus(tr=tr)
 storecollapsemsg(repo, self.collapsemsg)
 if len(repo[None].parents()) == 2:
 repo.ui.debug('resuming interrupted rebase\n')
@@ -711,7 +711,12 @@ def rebase(ui, repo, **opts):
 if retcode is not None:
 return retcode
 
-rbsrt._performrebase()
+with repo.transaction('rebase') as tr:
+try:
+rbsrt._performrebase(tr)
+except error.InterventionRequired:
+tr.close()
+raise
 rbsrt._finishrebase()
 finally:
 release(lock, wlock)
diff --git a/tests/test-rebase-abort.t b/tests/test-rebase-abort.t
--- a/tests/test-rebase-abort.t
+++ b/tests/test-rebase-abort.t
@@ -374,10 +374,11 @@ test aborting an interrupted series (iss
   $ hg --config extensions.n=$TESTDIR/failfilemerge.py rebase -s 3 -d tip
   rebasing 3:3a71550954f1 "b"
   rebasing 4:e80b69427d80 "c"
+  transaction abort!
+  rollback completed
   abort: ^C
   [255]
   $ hg rebase --abort
-  saved backup bundle to 
$TESTTMP/interrupted/.hg/strip-backup/3d8812cf300d-93041a90-backup.hg (glob)
   rebase aborted
   $ hg log -G --template "{rev} {desc} {bookmarks}"
   o  6 no-a
@@ -398,7 +399,7 @@ test aborting an interrupted series (iss
   parent: 0:df4f53cec30a 
base
   branch: default
-  commit: (clean)
+  commit: 1 unknown (clean)
   update: 6 new changesets (update)
   phases: 7 draft
 
diff --git a/tests/test-rebase-conflicts.t b/tests/test-rebase-conflicts.t
--- a/tests/test-rebase-conflicts.t
+++ b/tests/test-rebase-conflicts.t
@@ -225,7 +225,6 @@ Check that the right ancestors is used w
   ignoring null merge rebase of 8
   rebasing 9:e31216eec445 "more changes to f1"
future parents are 2 and -1
-  rebase status stored
update to 2:4bc80088dc6b
   resolving manifests
branchmerge: False, force: True, partial: False
@@ -251,7 +250,6 @@ Check that the right ancestors is used w
   rebased as 19c888675e13
   rebasing 10:2f2496ddf49d "merge" (tip)
future parents are 11 and 7
-  rebase status stored
already in target
merge against 10:2f2496ddf49d
  detach base 9:e31216eec445
@@ -269,6 +267,7 @@ Check that the right ancestors is used w
   committing changelog
   rebased as 2a7f09cac94c
   rebase merging completed
+  rebase status stored
   update back to initial working directory parent
   resolving manifests
branchmerge: False, force: False, partial: False
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 3 of 5 V2] rebase: add storestatus support for transactions

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488924269 28800
#  Tue Mar 07 14:04:29 2017 -0800
# Node ID 8451bfc2e1d6260a0ce9dac505dd9f34fb3b19aa
# Parent  9ee513d4a3884d19a85b2aadd7bac362d1530d11
rebase: add storestatus support for transactions

This let's the status writing logic support transactions. This will be useful in
a later patch where we add a transaction around the entire rebase.

diff --git a/hgext/rebase.py b/hgext/rebase.py
--- a/hgext/rebase.py
+++ b/hgext/rebase.py
@@ -159,10 +159,17 @@ class rebaseruntime(object):
 self.keepopen = opts.get('keepopen', False)
 self.obsoletenotrebased = {}
 
-def storestatus(self):
+def storestatus(self, tr=None):
 """Store the current status to allow recovery"""
+if tr:
+tr.addfilegenerator('rebasestate', ('rebasestate',),
+self._writestatus, location='plain')
+else:
+with self.repo.vfs("rebasestate", "w") as f:
+self._writestatus(f)
+
+def _writestatus(self, f):
 repo = self.repo
-f = repo.vfs("rebasestate", "w")
 f.write(repo[self.originalwd].hex() + '\n')
 f.write(repo[self.target].hex() + '\n')
 f.write(repo[self.external].hex() + '\n')
@@ -181,7 +188,6 @@ class rebaseruntime(object):
 else:
 newrev = v
 f.write("%s:%s\n" % (oldrev, newrev))
-f.close()
 repo.ui.debug('rebase status stored\n')
 
 def restorestatus(self):
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 5 of 6] context: remove uses of manifest.matches

[Durham Goode]

On 3/7/17 2:40 PM, Martin von Zweigbergk wrote:

On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:

# HG changeset patch
# User Durham Goode 
# Date 1488569391 28800
#  Fri Mar 03 11:29:51 2017 -0800
# Node ID 207763e895c7d24885df22f5b9c0df5494d77daf
# Parent  78e0fb2bd1bc972444ae08e7b7165da66cbf53a3
context: remove uses of manifest.matches

This removes the uses of manifest.matches in context.py in favor of the new api.
This is part of removing manifest.matches since it is O(manifest).

diff --git a/mercurial/context.py b/mercurial/context.py
--- a/mercurial/context.py
+++ b/mercurial/context.py
@@ -18,7 +18,6 @@ from .node import (
 bin,
 hex,
 modifiednodeid,
-newnodeid,
 nullid,
 nullrev,
 short,
@@ -91,15 +90,6 @@ class basectx(object):
 def __iter__(self):
 return iter(self._manifest)

-def _manifestmatches(self, match, s):
-"""generate a new manifest filtered by the match argument
-
-This method is for internal use only and mainly exists to provide an
-object oriented way for other contexts to customize the manifest
-generation.
-"""
-return self.manifest().matches(match)
-
 def _matchstatus(self, other, match):
 """return match.always if match is none

@@ -119,15 +109,15 @@ class basectx(object):
 # delta application.
 if self.rev() is not None and self.rev() < other.rev():
 self.manifest()
-mf1 = other._manifestmatches(match, s)
-mf2 = self._manifestmatches(match, s)
+mf1 = other.manifest()
+mf2 = self.manifest()

 modified, added = [], []
 removed = []
 clean = []
 deleted, unknown, ignored = s.deleted, s.unknown, s.ignored
 deletedset = set(deleted)
-d = mf1.diff(mf2, clean=listclean)
+d = mf1.diff(mf2, match=match, clean=listclean)
 for fn, value in d.iteritems():
 if fn in deletedset:
 continue
@@ -154,8 +144,10 @@ class basectx(object):

 if removed:
 # need to filter files if they are already reported as removed
-unknown = [fn for fn in unknown if fn not in mf1]
-ignored = [fn for fn in ignored if fn not in mf1]
+unknown = [fn for fn in unknown if fn not in mf1 and
+   (not match or match(fn))]
+ignored = [fn for fn in ignored if fn not in mf1 and
+   (not match or match(fn))]
 # if they're deleted, don't report them as removed
 removed = [fn for fn in removed if fn not in deletedset]

@@ -1608,22 +1600,6 @@ class workingctx(committablectx):
 pass
 return modified, fixup

-def _manifestmatches(self, match, s):
-"""Slow path for workingctx
-
-The fast path is when we compare the working directory to its parent
-which means this function is comparing with a non-parent; therefore we
-need to build a manifest and return what matches.
-"""
-mf = self._repo['.']._manifestmatches(match, s)
-for f in s.modified + s.added:
-mf[f] = newnodeid
-mf.setflag(f, self.flags(f))
-for f in s.removed:
-if f in mf:
-del mf[f]
-return mf
-


Nice! But could you explain why this is no longer needed? Is it
completely thanks to the new manifest.diff() API or something else?


My original reason was because self._manifest() could already generate 
this manifest (and does a more accurate job of it than this function), 
and since the matcher was moved to the diff layer, nothing here would be 
lost by switching to self._manifest().


Upon looking deeper, it looks like the big benefit here is that the 
status is provided, and therefore we don't have to call status a second 
time, (i.e. when self._manifest calls 'self._status()'). This has the 
additional benefit that the matcher can be applied to the status output 
early.  But this only applies when diffing the working copy against a 
commit that is not the parent of the working copy, so a somewhat niche case.


I have some ideas about how I could refactor this area to address this, 
so I'll include them in V2. It might make the series 10-12 patches long 
though.

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH evolve-ext] evolve: switch away from deprecated repo.opener

[Pierre-Yves David]

On 03/07/2017 06:47 PM, Martin von Zweigbergk via Mercurial-devel wrote:

# HG changeset patch
# User Martin von Zweigbergk 
# Date 1488908857 28800
#  Tue Mar 07 09:47:37 2017 -0800
# Node ID 5cf7a405f0e5c089a6da86583eed7c3f99c4ccde
# Parent  a4b09789117a64e4672ba6a2626bbaf965476e91
evolve: switch away from deprecated repo.opener


Pushed, thanks :-)

--
Pierre-Yves David
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 6 of 6] manifest: rename matches to _matches

[Martin von Zweigbergk via Mercurial-devel]

On Tue, Mar 7, 2017 at 2:45 PM, Durham Goode  wrote:
>
>
> On 3/7/17 2:29 PM, Martin von Zweigbergk wrote:
>>
>> On Tue, Mar 7, 2017 at 9:37 AM, Durham Goode  wrote:
>>>
>>>
>>>
>>> On 3/6/17 10:31 PM, Martin von Zweigbergk wrote:


 On Mon, Mar 6, 2017 at 5:40 PM, Durham Goode  wrote:
>
>
>
>
> On 3/6/17 5:32 PM, Martin von Zweigbergk wrote:
>>
>>
>>
>> On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:
>>>
>>>
>>>
>>> # HG changeset patch
>>> # User Durham Goode 
>>> # Date 1488517595 28800
>>> #  Thu Mar 02 21:06:35 2017 -0800
>>> # Node ID 1bed8ee65389133923b9076909e04f66ef26f1b8
>>> # Parent  207763e895c7d24885df22f5b9c0df5494d77daf
>>> manifest: rename matches to _matches
>>>
>>> Now that there are no external consumers of manifest.matches, let's
>>> rename it to
>>> _matches so it remains internal.
>>
>>
>> treemanifest already had a _matches(), so now it has two...
>>
>>> This means alternative manifest
>>> implementations
>>> no longer have to implement it, and can instead implement efficient
>>> versions of
>>> diff() and filesnotin().
>>
>>
>>
>>
>> matches() still seems like a sensible method. Once "hg files" gains a
>> "-v" or "--flags" option so it can really replace "hg manifest" (or if
>> "hg manifest" started accepting a file pattern), it seems natural to
>> implement that using manifest.matches(). Would it be a problem to
>> queue patches 1-5 but not this one?
>
>
>
>
> It's not a problem to queue 1-5 without 6, but given the performance
> criticalness of the manifest, I'd lean towards not giving it functions
> that
> could enable future bad algorithms.  If we need the ability to iterate
> over
> the entire manifest in the future, maybe we could add a
> manifest.walk(match=None) function that returns an iterator.



 That already exists, but it walks the filenames only, not the
 (filename,node,flags) tuples :-) But that's besides the point, I know;
 we'd just need to find a different name.

> The 'walk()'
> part exposes caller to the fact that it is going to be an expensive
> operation, and the iterator aspect means we don't have to allocate an
> entire
> dict and we don't have to worry about people abusing it as a way of
> doing
> naive set math using manifests.  That gets the manifest a little closer
> to
> resembling a dirstate as well.



 That makes sense. However, there's no such warning to users who do
 *not* use a matcher. I think I took care of all those cases two years
 ago, but if we want to make it less likely that someone does
 set(mf.match(m)), then it seems even more important to prevent
 set(mf).  So isn't it __iter__ and friends we should make explicit?
>>>
>>>
>>>
>>> Yep, we should probably head in the direction of making all full
>>> iteration
>>> operations explicit.
>>>
 I'd also like to be able to verify that this is going in the right
 direction. Could you share the remainder of the series too (as a
 link)? Thanks.
>>>
>>>
>>>
>>> The only new commits on top of this so far are to change
>>> treemanifest.diff()
>>> to actually limit it's diff based on the visitdir() responses of the
>>> matcher.  You can see it here:
>>> https://urldefense.proofpoint.com/v2/url?u=https-3A__bpaste.net_show_cba5f57820f1&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=nuarHzhP1wi1T9iURRCj1A&m=uvgL5NLjKD18j2hB6yeJEE-7YJg1wLpx_R1fsWaDkwg&s=oWl_cfGafh-_1LwCNrhAraVcMcVOLDKn3rcReKm5BY4&e=
>>> (two
>>> patches, one to update tests, one to adjust the diff implementation. I'm
>>> working on migrating our internal treemanifest imlpementation to use this
>>> pattern now, so I can get perf numbers. Then I'll come back and do the
>>> same
>>> for treemanifest.filesnotin()
>>
>>
>> I'd really like to see the performance gain from this series, since
>> that's the motivating force behind it. I've spent some time trying it
>> myself. I added these two lines in treemanifest.diff._diff():
>>
>> if match and not match.visitdir(t1._dir or '.'):
>> return
>>
>> IIUC, the point of the series is to make diffs and status calls
>> (across revisions) faster when there are many matching files but a
>> small diff. I tried "time hg st --rev .~1 --rev . js" on some
>> arbitrary commit I picked from a treemanifest version of the Mozilla
>> repo. That speeds up from 1.3s to 0.18s. Nice! It's such a simple fix
>> to get that speedup, so can I get you to include it in the series? I
>> would have preferred to see something like:
>
>
> Is the treemanifest version of mozilla available somewhere? Or would I need
> to convert it?

Unfortunately not, and I don't know of an easy way of sharing it.

>
> I can attach my two remaining patches (tests + diff optimization) to this
> series if you want.

Re: [PATCH 6 of 6] manifest: rename matches to _matches

[Durham Goode]

On 3/7/17 2:29 PM, Martin von Zweigbergk wrote:

On Tue, Mar 7, 2017 at 9:37 AM, Durham Goode  wrote:



On 3/6/17 10:31 PM, Martin von Zweigbergk wrote:


On Mon, Mar 6, 2017 at 5:40 PM, Durham Goode  wrote:




On 3/6/17 5:32 PM, Martin von Zweigbergk wrote:



On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:



# HG changeset patch
# User Durham Goode 
# Date 1488517595 28800
#  Thu Mar 02 21:06:35 2017 -0800
# Node ID 1bed8ee65389133923b9076909e04f66ef26f1b8
# Parent  207763e895c7d24885df22f5b9c0df5494d77daf
manifest: rename matches to _matches

Now that there are no external consumers of manifest.matches, let's
rename it to
_matches so it remains internal.


treemanifest already had a _matches(), so now it has two...


This means alternative manifest
implementations
no longer have to implement it, and can instead implement efficient
versions of
diff() and filesnotin().




matches() still seems like a sensible method. Once "hg files" gains a
"-v" or "--flags" option so it can really replace "hg manifest" (or if
"hg manifest" started accepting a file pattern), it seems natural to
implement that using manifest.matches(). Would it be a problem to
queue patches 1-5 but not this one?




It's not a problem to queue 1-5 without 6, but given the performance
criticalness of the manifest, I'd lean towards not giving it functions
that
could enable future bad algorithms.  If we need the ability to iterate
over
the entire manifest in the future, maybe we could add a
manifest.walk(match=None) function that returns an iterator.



That already exists, but it walks the filenames only, not the
(filename,node,flags) tuples :-) But that's besides the point, I know;
we'd just need to find a different name.


The 'walk()'
part exposes caller to the fact that it is going to be an expensive
operation, and the iterator aspect means we don't have to allocate an
entire
dict and we don't have to worry about people abusing it as a way of doing
naive set math using manifests.  That gets the manifest a little closer
to
resembling a dirstate as well.



That makes sense. However, there's no such warning to users who do
*not* use a matcher. I think I took care of all those cases two years
ago, but if we want to make it less likely that someone does
set(mf.match(m)), then it seems even more important to prevent
set(mf).  So isn't it __iter__ and friends we should make explicit?



Yep, we should probably head in the direction of making all full iteration
operations explicit.


I'd also like to be able to verify that this is going in the right
direction. Could you share the remainder of the series too (as a
link)? Thanks.



The only new commits on top of this so far are to change treemanifest.diff()
to actually limit it's diff based on the visitdir() responses of the
matcher.  You can see it here: 
https://urldefense.proofpoint.com/v2/url?u=https-3A__bpaste.net_show_cba5f57820f1&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=nuarHzhP1wi1T9iURRCj1A&m=uvgL5NLjKD18j2hB6yeJEE-7YJg1wLpx_R1fsWaDkwg&s=oWl_cfGafh-_1LwCNrhAraVcMcVOLDKn3rcReKm5BY4&e=
  (two
patches, one to update tests, one to adjust the diff implementation. I'm
working on migrating our internal treemanifest imlpementation to use this
pattern now, so I can get perf numbers. Then I'll come back and do the same
for treemanifest.filesnotin()


I'd really like to see the performance gain from this series, since
that's the motivating force behind it. I've spent some time trying it
myself. I added these two lines in treemanifest.diff._diff():

if match and not match.visitdir(t1._dir or '.'):
return

IIUC, the point of the series is to make diffs and status calls
(across revisions) faster when there are many matching files but a
small diff. I tried "time hg st --rev .~1 --rev . js" on some
arbitrary commit I picked from a treemanifest version of the Mozilla
repo. That speeds up from 1.3s to 0.18s. Nice! It's such a simple fix
to get that speedup, so can I get you to include it in the series? I
would have preferred to see something like:


Is the treemanifest version of mozilla available somewhere? Or would I 
need to convert it?


I can attach my two remaining patches (tests + diff optimization) to 
this series if you want. The two lines you added aren't quite enough to 
be a complete patch (t1._dir isn't the full path, and we need to handle 
file matching as well).



1. add match argument to diff() and filesnotin() and optimize them
2. make copies use new manifest.diff() api and see how it's faster
3. same for merge
4. ...

That would make it an easier sell for me. But again, what I cared most
about was data to back up your claim, so I'd really like at least to
have that part of the series.


I went with this approach because it put all the easy, 
non-logic-affecting changes at the front for easy queueing. That put the 
least blockers between the refactor getting into core and me being able 
to make the needed changes to our internal treemanifest to ge

Re: [PATCH 5 of 6] context: remove uses of manifest.matches

[Martin von Zweigbergk via Mercurial-devel]

On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:
> # HG changeset patch
> # User Durham Goode 
> # Date 1488569391 28800
> #  Fri Mar 03 11:29:51 2017 -0800
> # Node ID 207763e895c7d24885df22f5b9c0df5494d77daf
> # Parent  78e0fb2bd1bc972444ae08e7b7165da66cbf53a3
> context: remove uses of manifest.matches
>
> This removes the uses of manifest.matches in context.py in favor of the new 
> api.
> This is part of removing manifest.matches since it is O(manifest).
>
> diff --git a/mercurial/context.py b/mercurial/context.py
> --- a/mercurial/context.py
> +++ b/mercurial/context.py
> @@ -18,7 +18,6 @@ from .node import (
>  bin,
>  hex,
>  modifiednodeid,
> -newnodeid,
>  nullid,
>  nullrev,
>  short,
> @@ -91,15 +90,6 @@ class basectx(object):
>  def __iter__(self):
>  return iter(self._manifest)
>
> -def _manifestmatches(self, match, s):
> -"""generate a new manifest filtered by the match argument
> -
> -This method is for internal use only and mainly exists to provide an
> -object oriented way for other contexts to customize the manifest
> -generation.
> -"""
> -return self.manifest().matches(match)
> -
>  def _matchstatus(self, other, match):
>  """return match.always if match is none
>
> @@ -119,15 +109,15 @@ class basectx(object):
>  # delta application.
>  if self.rev() is not None and self.rev() < other.rev():
>  self.manifest()
> -mf1 = other._manifestmatches(match, s)
> -mf2 = self._manifestmatches(match, s)
> +mf1 = other.manifest()
> +mf2 = self.manifest()
>
>  modified, added = [], []
>  removed = []
>  clean = []
>  deleted, unknown, ignored = s.deleted, s.unknown, s.ignored
>  deletedset = set(deleted)
> -d = mf1.diff(mf2, clean=listclean)
> +d = mf1.diff(mf2, match=match, clean=listclean)
>  for fn, value in d.iteritems():
>  if fn in deletedset:
>  continue
> @@ -154,8 +144,10 @@ class basectx(object):
>
>  if removed:
>  # need to filter files if they are already reported as removed
> -unknown = [fn for fn in unknown if fn not in mf1]
> -ignored = [fn for fn in ignored if fn not in mf1]
> +unknown = [fn for fn in unknown if fn not in mf1 and
> +   (not match or match(fn))]
> +ignored = [fn for fn in ignored if fn not in mf1 and
> +   (not match or match(fn))]
>  # if they're deleted, don't report them as removed
>  removed = [fn for fn in removed if fn not in deletedset]
>
> @@ -1608,22 +1600,6 @@ class workingctx(committablectx):
>  pass
>  return modified, fixup
>
> -def _manifestmatches(self, match, s):
> -"""Slow path for workingctx
> -
> -The fast path is when we compare the working directory to its parent
> -which means this function is comparing with a non-parent; therefore 
> we
> -need to build a manifest and return what matches.
> -"""
> -mf = self._repo['.']._manifestmatches(match, s)
> -for f in s.modified + s.added:
> -mf[f] = newnodeid
> -mf.setflag(f, self.flags(f))
> -for f in s.removed:
> -if f in mf:
> -del mf[f]
> -return mf
> -

Nice! But could you explain why this is no longer needed? Is it
completely thanks to the new manifest.diff() API or something else?

>  def _dirstatestatus(self, match=None, ignored=False, clean=False,
>  unknown=False):
>  '''Gets the status from the dirstate -- internal use only.'''
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 6 of 6] manifest: rename matches to _matches

[Martin von Zweigbergk via Mercurial-devel]

On Tue, Mar 7, 2017 at 9:37 AM, Durham Goode  wrote:
>
>
> On 3/6/17 10:31 PM, Martin von Zweigbergk wrote:
>>
>> On Mon, Mar 6, 2017 at 5:40 PM, Durham Goode  wrote:
>>>
>>>
>>>
>>> On 3/6/17 5:32 PM, Martin von Zweigbergk wrote:


 On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:
>
>
> # HG changeset patch
> # User Durham Goode 
> # Date 1488517595 28800
> #  Thu Mar 02 21:06:35 2017 -0800
> # Node ID 1bed8ee65389133923b9076909e04f66ef26f1b8
> # Parent  207763e895c7d24885df22f5b9c0df5494d77daf
> manifest: rename matches to _matches
>
> Now that there are no external consumers of manifest.matches, let's
> rename it to
> _matches so it remains internal.

treemanifest already had a _matches(), so now it has two...

> This means alternative manifest
> implementations
> no longer have to implement it, and can instead implement efficient
> versions of
> diff() and filesnotin().



 matches() still seems like a sensible method. Once "hg files" gains a
 "-v" or "--flags" option so it can really replace "hg manifest" (or if
 "hg manifest" started accepting a file pattern), it seems natural to
 implement that using manifest.matches(). Would it be a problem to
 queue patches 1-5 but not this one?
>>>
>>>
>>>
>>> It's not a problem to queue 1-5 without 6, but given the performance
>>> criticalness of the manifest, I'd lean towards not giving it functions
>>> that
>>> could enable future bad algorithms.  If we need the ability to iterate
>>> over
>>> the entire manifest in the future, maybe we could add a
>>> manifest.walk(match=None) function that returns an iterator.
>>
>>
>> That already exists, but it walks the filenames only, not the
>> (filename,node,flags) tuples :-) But that's besides the point, I know;
>> we'd just need to find a different name.
>>
>>> The 'walk()'
>>> part exposes caller to the fact that it is going to be an expensive
>>> operation, and the iterator aspect means we don't have to allocate an
>>> entire
>>> dict and we don't have to worry about people abusing it as a way of doing
>>> naive set math using manifests.  That gets the manifest a little closer
>>> to
>>> resembling a dirstate as well.
>>
>>
>> That makes sense. However, there's no such warning to users who do
>> *not* use a matcher. I think I took care of all those cases two years
>> ago, but if we want to make it less likely that someone does
>> set(mf.match(m)), then it seems even more important to prevent
>> set(mf).  So isn't it __iter__ and friends we should make explicit?
>
>
> Yep, we should probably head in the direction of making all full iteration
> operations explicit.
>
>> I'd also like to be able to verify that this is going in the right
>> direction. Could you share the remainder of the series too (as a
>> link)? Thanks.
>
>
> The only new commits on top of this so far are to change treemanifest.diff()
> to actually limit it's diff based on the visitdir() responses of the
> matcher.  You can see it here: https://bpaste.net/show/cba5f57820f1 (two
> patches, one to update tests, one to adjust the diff implementation. I'm
> working on migrating our internal treemanifest imlpementation to use this
> pattern now, so I can get perf numbers. Then I'll come back and do the same
> for treemanifest.filesnotin()

I'd really like to see the performance gain from this series, since
that's the motivating force behind it. I've spent some time trying it
myself. I added these two lines in treemanifest.diff._diff():

if match and not match.visitdir(t1._dir or '.'):
return

IIUC, the point of the series is to make diffs and status calls
(across revisions) faster when there are many matching files but a
small diff. I tried "time hg st --rev .~1 --rev . js" on some
arbitrary commit I picked from a treemanifest version of the Mozilla
repo. That speeds up from 1.3s to 0.18s. Nice! It's such a simple fix
to get that speedup, so can I get you to include it in the series? I
would have preferred to see something like:

1. add match argument to diff() and filesnotin() and optimize them
2. make copies use new manifest.diff() api and see how it's faster
3. same for merge
4. ...

That would make it an easier sell for me. But again, what I cared most
about was data to back up your claim, so I'd really like at least to
have that part of the series.

Btw, an alternative approach would have been to make
manifest.matches() lazy, but I think making it explicit like your
series does is better.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH RFC] help: align description of 'base rev' with reality [issue5488]

[Jun Wu]

Excerpts from Augie Fackler's message of 2017-03-07 13:02:18 -0500:
> On Tue, Feb 28, 2017 at 03:33:36PM +0100, Kim Alvefur wrote:
> > # HG changeset patch
> > # User Kim Alvefur 
> > # Date 1488291548 -3600
> > #  Tue Feb 28 15:19:08 2017 +0100
> > # Node ID 397c0aa1cd18439d2ce90a68cd5737fb8dafab49
> > # Parent  a185b903bda3c64678d1f9399bfa0be8d326dbff
> > help: align description of 'base rev' with reality [issue5488]
> 
> Confirmed by reading the code this is correct. Very nice catch.

AFAIK indygreg did a same fix in his doc update patches.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] rebase: move actual rebase into a single transaction

[Jun Wu]

Excerpts from Jun Wu's message of 2017-03-07 13:51:17 -0800:
> I think we either just update so the behavior will "look like" what it was
> before, or mark this as a "BC".

After a second thought, it does not seem that we could keep the old
behavior. So it's probably a BC either way.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] rebase: move actual rebase into a single transaction

[Jun Wu]

Excerpts from Durham Goode's message of 2017-03-07 13:16:29 -0800:
> 
> On 3/7/17 10:08 AM, Augie Fackler wrote:
> > On Sat, Mar 04, 2017 at 02:08:26PM -0800, Durham Goode wrote:
> >> # HG changeset patch
> >> # User Durham Goode 
> >> # Date 1488665211 28800
> >> #  Sat Mar 04 14:06:51 2017 -0800
> >> # Node ID 9c3ea2112952f398aaa7625f43dcfa36cfd34379
> >> # Parent  b4cd912d7704cd976e1bee3a3c927e0e578ec88f
> >> rebase: move actual rebase into a single transaction
> >>
> >> Previously, rebasing would open several transaction over the course of 
> >> rebasing
> >> several commits. Opening a transaction can have notable overhead (like 
> >> copying
> >> the dirstate) which can add up when rebasing many commits.
> >>
> >> This patch adds a single large transaction around the actual commit rebase
> >> operation, with a catch for intervention which serializes the current 
> >> state if
> >> we need to drop back to the terminal for user intervention. Amazingly, 
> >> almost
> >> all the tests seem to pass.
> >>
> >> On large repos with large working copies, this can speed up rebasing 7 
> >> commits
> >> by 25%. I'd expect the percentage to be a bit larger for rebasing even more
> >> commits.
> >
> > I like it, seems like the correct thing to do in any case. However...
> >
> > [...]
> >
> >> diff --git a/tests/test-rebase-abort.t b/tests/test-rebase-abort.t
> >> --- a/tests/test-rebase-abort.t
> >> +++ b/tests/test-rebase-abort.t
> >>
> > [...]
> >
> >> @@ -398,7 +399,7 @@ test aborting an interrupted series (iss
> >>parent: 0:df4f53cec30a
> >> base
> >>branch: default
> >> -  commit: (clean)
> >> +  commit: 1 unknown (interrupted update)
> >>update: 6 new changesets (update)
> >>phases: 7 draft
> >
> > What's up with this? Shouldn't rebase --abort put us back in a clean state?
> 
> This is actually a bug in normal rebase.  If the rebase aborts while the 
> working copy is on the commit that it started on, then it doesn't need 
> to perform any update during rebase --abort, which means it doesn't 
> clean up the .hg/updatestate file. The single transaction makes this 
> more obvious since when the whole transaction fails, it dumps everything 
> back to the way things were (i.e. you're back on the initial commit), 
> but leaves the working copy dirty.  I guess we could fix this by also 
> checking if status is clean when deciding whether to perform an update 
> during rebase --abort.  Or maybe we just run the update every time, 
> since it's extremely likely to be needed.
> 
> This raises another issue though, that the rebase state file is not 
> managed by the transaction, so it is being written mid transaction and 
> doesn't get rolled back appropriately.  I'll look into fixing both of 
> these and resending.

I think we either just update so the behavior will "look like" what it was
before, or mark this as a "BC".
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] rebase: move actual rebase into a single transaction

[Durham Goode]

On 3/7/17 10:08 AM, Augie Fackler wrote:

On Sat, Mar 04, 2017 at 02:08:26PM -0800, Durham Goode wrote:

# HG changeset patch
# User Durham Goode 
# Date 1488665211 28800
#  Sat Mar 04 14:06:51 2017 -0800
# Node ID 9c3ea2112952f398aaa7625f43dcfa36cfd34379
# Parent  b4cd912d7704cd976e1bee3a3c927e0e578ec88f
rebase: move actual rebase into a single transaction

Previously, rebasing would open several transaction over the course of rebasing
several commits. Opening a transaction can have notable overhead (like copying
the dirstate) which can add up when rebasing many commits.

This patch adds a single large transaction around the actual commit rebase
operation, with a catch for intervention which serializes the current state if
we need to drop back to the terminal for user intervention. Amazingly, almost
all the tests seem to pass.

On large repos with large working copies, this can speed up rebasing 7 commits
by 25%. I'd expect the percentage to be a bit larger for rebasing even more
commits.


I like it, seems like the correct thing to do in any case. However...

[...]


diff --git a/tests/test-rebase-abort.t b/tests/test-rebase-abort.t
--- a/tests/test-rebase-abort.t
+++ b/tests/test-rebase-abort.t


[...]


@@ -398,7 +399,7 @@ test aborting an interrupted series (iss
   parent: 0:df4f53cec30a
base
   branch: default
-  commit: (clean)
+  commit: 1 unknown (interrupted update)
   update: 6 new changesets (update)
   phases: 7 draft


What's up with this? Shouldn't rebase --abort put us back in a clean state?


This is actually a bug in normal rebase.  If the rebase aborts while the 
working copy is on the commit that it started on, then it doesn't need 
to perform any update during rebase --abort, which means it doesn't 
clean up the .hg/updatestate file. The single transaction makes this 
more obvious since when the whole transaction fails, it dumps everything 
back to the way things were (i.e. you're back on the initial commit), 
but leaves the working copy dirty.  I guess we could fix this by also 
checking if status is clean when deciding whether to perform an update 
during rebase --abort.  Or maybe we just run the update every time, 
since it's extremely likely to be needed.


This raises another issue though, that the rebase state file is not 
managed by the transaction, so it is being written mid transaction and 
doesn't get rolled back appropriately.  I'll look into fixing both of 
these and resending.

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 5 of 5] localrepo: deprecated 'repo.opener' (API)

[Durham Goode]

On 3/3/17 7:42 AM, Pierre-Yves David wrote:



On 03/03/2017 02:50 PM, Yuya Nishihara wrote:

On Thu, 02 Mar 2017 03:58:45 +0100, Pierre-Yves David wrote:

# HG changeset patch
# User Pierre-Yves David 
# Date 1470398170 -7200
#  Fri Aug 05 13:56:10 2016 +0200
# Node ID aacf8b01b81483b4815a974f17d0ff5d214c4d3d
# Parent  437a39859c33901ea29cd22341d93be752e4acc0
# EXP-Topic vfs.cleanup
localrepo: deprecated 'repo.opener' (API)


It appears fsmonitor still use repo.opener. Can you fix it as a follow
up?


Yep, sent. It appears, fsmonitor is not tested at all by default :-/


I didn't see the patch on the list (or my email search skills are just 
bad), so I sent a patch to fix this.

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH] fsmonitor: remove use of repo.opener

[Durham Goode]

# HG changeset patch
# User Durham Goode 
# Date 1488919920 28800
#  Tue Mar 07 12:52:00 2017 -0800
# Node ID 4d485e46a3e5c83ff8d2ae0a4ba8e484a284f65b
# Parent  7433b3bc55eebfa9149280339b406bd4cec64efb
fsmonitor: remove use of repo.opener

This has been deprecated, so we need to switch to the appropriate vfs apis.

diff --git a/hgext/fsmonitor/state.py b/hgext/fsmonitor/state.py
--- a/hgext/fsmonitor/state.py
+++ b/hgext/fsmonitor/state.py
@@ -20,7 +20,7 @@ from mercurial import pathutil
 
 class state(object):
 def __init__(self, repo):
-self._opener = repo.opener
+self._vfs = repo.vfs
 self._ui = repo.ui
 self._rootdir = pathutil.normasprefix(repo.root)
 self._lastclock = None
@@ -33,7 +33,7 @@ class state(object):
 
 def get(self):
 try:
-file = self._opener('fsmonitor.state', 'rb')
+file = self._vfs('fsmonitor.state', 'rb')
 except IOError as inst:
 if inst.errno != errno.ENOENT:
 raise
@@ -91,7 +91,7 @@ class state(object):
 return
 
 try:
-file = self._opener('fsmonitor.state', 'wb', atomictemp=True)
+file = self._vfs('fsmonitor.state', 'wb', atomictemp=True)
 except (IOError, OSError):
 self._ui.warn(_("warning: unable to write out fsmonitor state\n"))
 return
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH V2] merge: add `internal:dumpjson` tool to `resolve`, which outputs conflict state

[Phil Cohen]

# HG changeset patch
# User Phil Cohen 
# Date 1488915535 28800
#  Tue Mar 07 11:38:55 2017 -0800
# Node ID bbce62e3790220f19e7b37160a2f8351b7461272
# Parent  91e86a6c61c0c6a3b554eefeba906311aa29
merge: add `internal:dumpjson` tool to `resolve`, which outputs conflict state

This supersedes a previous change which added a --prep option.

Normally, `hg resolve` takes the user on a whistle-stop tour of each conflicted
file, pausing to launch the editor to resolve the conflicts. This is an
alternative workflow for resolving many conflicts in a random-access fashion. It
doesn't change/replace the default behavior.

This commit adds `--tool=internal:dumpjson`. It prints, for each conflict, the
"base", "other", and "ours" versions (the contents, as well as their exec/link
flags), and where the user/tool should write a resolved version (i.e., the
working copy) as JSON. The user will then resolve the conflicts at their leisure
and run `hg resolve --mark`.

diff --git a/mercurial/commands.py b/mercurial/commands.py
--- a/mercurial/commands.py
+++ b/mercurial/commands.py
@@ -33,6 +33,7 @@
 error,
 exchange,
 extensions,
+formatter,
 graphmod,
 hbisect,
 help,
@@ -4284,6 +4285,26 @@
 fm.end()
 return 0
 
+if opts.get('tool', '') == "internal:dumpjson":
+fm = ui.formatter('resolve', {'template': 'json'})
+ms = mergemod.mergestate.read(repo)
+m = scmutil.match(repo[None], pats, opts)
+wctx = repo[None]
+
+paths = []
+for f in ms:
+if not m(f):
+continue
+
+val = ms.internaldump(f, wctx)
+if val is not None:
+paths.append(val)
+
+fm.startitem()
+fm.write('conflicts', '%s\n', paths)
+fm.end()
+return 0
+
 with repo.wlock():
 ms = mergemod.mergestate.read(repo)
 
diff --git a/mercurial/filemerge.py b/mercurial/filemerge.py
--- a/mercurial/filemerge.py
+++ b/mercurial/filemerge.py
@@ -567,6 +567,40 @@
 "o": " [%s]" % labels[1],
 }
 
+def summarize(repo, fcd, fco, fca):
+back = None if fcd.isabsent() else \
+scmutil.origpath(repo.ui, repo, repo.wjoin(fcd.path()))
+
+def sum(ctx):
+return {
+'contents': ctx.data(),
+'isexec': ctx.isexec(),
+'issymlink': ctx.islink(),
+}
+
+ours = None
+if back and util.filestat(back).stat:
+# Replicate the schema of `base` and `other` for `ours`. Since you can
+# start a merge with a dirty working copy, `ours` must reflect that,
+# not the underlying commit. That's why we look at .orig version.
+ours = {
+'path': back,
+'contents': util.readfile(back),
+'isexec': util.isexec(back),
+'issymlink': util.statislink(util.filestat(back).stat)
+}
+
+output = sum(fcd)
+output['path'] = repo.wjoin(fcd.path())
+
+return {
+'path': fcd.path(),
+'base': sum(fca),
+'other': sum(fco),
+'output': output,
+'ours': ours,
+}
+
 def _filemerge(premerge, repo, mynode, orig, fcd, fco, fca, labels=None):
 """perform a 3-way merge in the working directory
 
diff --git a/mercurial/merge.py b/mercurial/merge.py
--- a/mercurial/merge.py
+++ b/mercurial/merge.py
@@ -456,6 +456,24 @@
 def extras(self, filename):
 return self._stateextras.setdefault(filename, {})
 
+def _internaldump(self, dfile, wctx):
+if self[dfile] in 'rd':
+return None
+stateentry = self._state[dfile]
+state, hash, lfile, afile, anode, ofile, onode, flags = stateentry
+octx = self._repo[self._other]
+extras = self.extras(dfile)
+anccommitnode = extras.get('ancestorlinknode')
+if anccommitnode:
+actx = self._repo[anccommitnode]
+else:
+actx = None
+fcd = self._filectxorabsent(hash, wctx, dfile)
+fco = self._filectxorabsent(onode, octx, ofile)
+fca = self._repo.filectx(afile, fileid=anode, changeid=actx)
+
+return filemerge.summarize(self._repo, fcd, fco, fca)
+
 def _resolve(self, preresolve, dfile, wctx):
 """rerun merge process for file path `dfile`"""
 if self[dfile] in 'rd':
@@ -543,6 +561,9 @@
 Returns whether the merge is complete, and the exit code."""
 return self._resolve(True, dfile, wctx)
 
+def internaldump(self, dfile, wctx):
+return self._internaldump(dfile, wctx)
+
 def resolve(self, dfile, wctx):
 """run merge process (assuming premerge was run) for dfile
 
diff --git a/tests/test-resolve-prep.t b/tests/test-resolve-prep.t
new file mode 100644
--- /dev/null
+++ b/tests/test-resolve-prep.t
@@ -0,0 +1,75 @@
+1) Make the repo
+  $ hg init
+
+2) Can't run prep outside a conflict
+  $ hg resolve --tool internal:dumpjson
+  abort: no files or directories specified
+  (use --all to re-merge al

[PATCH 3 of 3 py3] repoview: convert attribute names to unicodes on Python 3

[Pulkit Goyal]

# HG changeset patch
# User Pulkit Goyal <7895pul...@gmail.com>
# Date 1488914355 -19800
#  Wed Mar 08 00:49:15 2017 +0530
# Node ID 3f1b6aded120622ec0195ab180cddd8cf09dc59e
# Parent  6ff8f348d278599b1dd79aa6b4d7c7c231ca7b58
repoview: convert attribute names to unicodes on Python 3

In Python 3, the attribute names must be strings i.e. unicodes.

diff -r 6ff8f348d278 -r 3f1b6aded120 mercurial/repoview.py
--- a/mercurial/repoview.py Wed Mar 08 00:45:19 2017 +0530
+++ b/mercurial/repoview.py Wed Mar 08 00:49:15 2017 +0530
@@ -300,10 +300,10 @@
 """
 
 def __init__(self, repo, filtername):
-object.__setattr__(self, '_unfilteredrepo', repo)
-object.__setattr__(self, 'filtername', filtername)
-object.__setattr__(self, '_clcachekey', None)
-object.__setattr__(self, '_clcache', None)
+object.__setattr__(self, r'_unfilteredrepo', repo)
+object.__setattr__(self, r'filtername', filtername)
+object.__setattr__(self, r'_clcachekey', None)
+object.__setattr__(self, r'_clcache', None)
 
 # not a propertycache on purpose we shall implement a proper cache later
 @property
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 1 of 3 py3] store: slice over a bytestring to get characters instead of ascii values

[Pulkit Goyal]

# HG changeset patch
# User Pulkit Goyal <7895pul...@gmail.com>
# Date 1488913815 -19800
#  Wed Mar 08 00:40:15 2017 +0530
# Node ID f8ec9104d219a01357e8f6118286d9b212695bbe
# Parent  b9834a28d200fad7584c6c98efb1b81bdc5f4106
store: slice over a bytestring to get characters instead of ascii values

On Python 2,

>>> a = b'abc'
>>> a[1]
'b'

Whereas on python 3,

>>> a = b'abc'
>>> a[1]
98
>>> a[1:2]
b'b'

This does not change behaviour on python 2.

diff -r b9834a28d200 -r f8ec9104d219 mercurial/store.py
--- a/mercurial/store.pyTue Feb 28 15:19:08 2017 +0100
+++ b/mercurial/store.pyWed Mar 08 00:40:15 2017 +0530
@@ -101,7 +101,7 @@
 e = '_'
 if pycompat.ispy3:
 xchr = lambda x: bytes([x])
-asciistr = [bytes(a) for a in range(127)]
+asciistr = [bytes([a]) for a in range(127)]
 else:
 xchr = chr
 asciistr = map(chr, xrange(127))
@@ -128,7 +128,7 @@
 pass
 else:
 raise KeyError
-return (lambda s: ''.join([cmap[c] for c in s]),
+return (lambda s: ''.join([cmap[s[c:c+1]] for c in xrange(len(s))]),
 lambda s: ''.join(list(decode(s
 
 _encodefname, _decodefname = _buildencodefun()
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 2 of 3 py3] parsers: alias long to int on Python 3

[Pulkit Goyal]

# HG changeset patch
# User Pulkit Goyal <7895pul...@gmail.com>
# Date 1488914119 -19800
#  Wed Mar 08 00:45:19 2017 +0530
# Node ID 6ff8f348d278599b1dd79aa6b4d7c7c231ca7b58
# Parent  f8ec9104d219a01357e8f6118286d9b212695bbe
parsers: alias long to int on Python 3

diff -r f8ec9104d219 -r 6ff8f348d278 mercurial/pure/parsers.py
--- a/mercurial/pure/parsers.py Wed Mar 08 00:40:15 2017 +0530
+++ b/mercurial/pure/parsers.py Wed Mar 08 00:45:19 2017 +0530
@@ -14,6 +14,9 @@
 from . import pycompat
 stringio = pycompat.stringio
 
+if pycompat.ispy3:
+long = int
+
 _pack = struct.pack
 _unpack = struct.unpack
 _compress = zlib.compress
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 6 of 6 py3] schemes: move re construction to module-level and python3-ify

[Augie Fackler]

> On Mar 7, 2017, at 9:03 AM, Yuya Nishihara  wrote:
> 
>> diff --git a/hgext/schemes.py b/hgext/schemes.py
>> --- a/hgext/schemes.py
>> +++ b/hgext/schemes.py
>> @@ -63,6 +63,7 @@ command = cmdutil.command(cmdtable)
>> # leave the attribute unspecified.
>> testedwith = 'ships-with-hg-core'
>> 
>> +_partre = re.compile(r'\{(\d+)\}'.encode(u'latin1'))
> 
> Maybe this could be br'' ?

Very likely it should be. I just was doing enough tap dancing to get a little 
further on Python 3, hopefully the end of this series will help us make faster 
progress now that we have --debugger working in Python 3.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: Deltas, compression, and zstd

[Gregory Szorc]

On Tue, Jan 10, 2017 at 10:34 AM, Gregory Szorc 
wrote:

> A few of us started an off-list conversation with Yann Collett (author of
> lz4 and zstd and general compression guru) about deltas, compression, and
> zstd. Most of that conversation is suitable for the public domain since it
> revolves around the open source Mercurial project. So this thread will be
> the continuation of that discussion.
>
> My initial message (with minor modifications) to Yann follows.
>
> ---
>
> My immediate goals for Mercurial are to allow zstd to be used as a
> drop-in replacement for zlib wherever zlib is used, without changing /too/
> much of the architecture.
>
> For streaming compression (producing standalone "bundle" files and
> transferring data over the wire protocol), zstd has been nothing short of
> amazing. https://www.mercurial-scm.org/pipermail/mercurial-devel/2016
> -December/091797.html shows ~60% CPU usage while yielding better
> compression ratios. You can't ask for much better than that!
>
> The other major use of zlib in Mercurial is revlogs. You can read more
> about them at https://www.mercurial-scm.org/repo/hg/help/internals.revlogs.
> In short, revlogs are our storage structure for "blob" data. Each logical
> entity (the set of commits (changelog), the set of files in a commit
> (manifest), and each file/path tracked in history (filelog)) has its own
> revlog (read: data within a specific revlog tends to look very similar).
> Revlogs store data in delta chains. The first entry is stored as fulltext.
> Then we compute and store deltas/diffs for subsequent entries. Rebuilding
> the fulltext for a revision involves finding a base fulltext then walking
> the delta chain to apply deltas until you rebuild the fulltext. We attempt
> to zlib compress all deltas today, storing the zlib compressed result if it
> is smaller or plaintext if it isn't.
>
> In https://www.mercurial-scm.org/pipermail/mercurial-devel/2017
> -January/091928.html, I'm attempting to replace zlib in revlogs with zstd.
> It does "just work." But it isn't without its caveats. Many of them are
> listed in https://www.mercurial-scm.org/pipermail/mercurial-devel/2017
> -January/091982.html.
>
> Here are some of the areas where I could use your expertise:
>
> * Optimizing read performance. We'd really like changelog reads
> (changeset/commit data) to be as fast as possible. We've already removed
> delta chains there. I've also noticed that decompression with dictionaries
> is substantially faster than without them. Is there further tuning of
> compression parameters(read: not just integer comp level) to optimize for
> read speed? (Note: changelog entries tend to be 100-300 bytes raw, with
> some going into the tens of kilobytes or larger range.)
>
> * Everything dictionary compression. See https://www.mercurial-scm.org/
> pipermail/mercurial-devel/2017-January/091986.html. It seems like we
> could get some major compression ratio and speed wins with dictionaries.
> But that requires complications to build, store, and possibly transfer
> dictionaries. At the very least, I'd like your thoughts on whether
> Mercurial should ship pre-built dictionaries for specific use cases so
> people can get *some* dictionary compression benefits today without us
> having to implement complex dictionary management. If so, how could you
> provide guidance on training?
>
> * Could we emit/format revlog deltas differently or in a special way to
> optimize for zstd?
>
> * When should we compress versus store uncompressed? Right now, our
> heuristic for storing a compressed chunk in a revlog is to store compressed
> if it is smaller than the raw input. This feels sub-optimal to me because
> if your compressed data is only 1 byte smaller than raw, it feels better to
> just sacrifice that 1 byte in the name of performance. While I'm certainly
> capable of measuring this and making this a configurable parameter, I was
> wondering if you could provide insights for a good heuristic for
> determining a) should we compress string X b) should we store the
> compressed result.
>
> * Frame header overhead. In some cases, the longer zstd frame header is
> yielding larger storage sizes than zlib. This is mostly for very small
> inputs (<150 bytes). We're storing the content size in the frame header (so
> we can do one-shot decompression without over heap allocating). I don't
> want to reinvent the wheel, but it is certainly tempting to construct our
> own frame header for special cases (namely very small inputs).
>
> Those are shorter term questions.
>
> Down the road, we're thinking about more radical changes to the storage
> model. We don't want 1 file/revlog per logical tracked entity (due to inode
> scaling issues among other things). We want to put multiple logical
> entities in the same container. There's a ton of possibilities for radical
> compression and encoding techniques here.
>
>
I just published a blog post about Zstandard:
http://gregoryszorc.com/blog/2017/03/07/bet

[PATCH stable] pycompat: verify sys.argv exists before forwarding it (issue5493)

[Augie Fackler]

# HG changeset patch
# User Augie Fackler 
# Date 1488911064 18000
#  Tue Mar 07 13:24:24 2017 -0500
# Branch stable
# Node ID b1e649de4c276133444234f33638cde393cbbfc8
# Parent  6b00c3ecd15b26587de8cca6fab811069cba3b2f
pycompat: verify sys.argv exists before forwarding it (issue5493)

ISAPI_WSGI doesn't set up sys.argv, so we have to look for the
attribute before assuming it exists.

diff --git a/mercurial/pycompat.py b/mercurial/pycompat.py
--- a/mercurial/pycompat.py
+++ b/mercurial/pycompat.py
@@ -69,7 +69,8 @@ if ispy3:
 #
 # TODO: On Windows, the native argv is wchar_t, so we'll need a different
 # workaround to simulate the Python 2 (i.e. ANSI Win32 API) behavior.
-sysargv = list(map(os.fsencode, sys.argv))
+if getattr(sys, 'argv', None) is not None:
+sysargv = list(map(os.fsencode, sys.argv))
 
 def sysstr(s):
 """Return a keyword str to be passed to Python functions such as
@@ -165,7 +166,8 @@ else:
 stdin = sys.stdin
 stdout = sys.stdout
 stderr = sys.stderr
-sysargv = sys.argv
+if getattr(sys, 'argv', None) is not None:
+sysargv = sys.argv
 sysplatform = sys.platform
 getcwd = os.getcwd
 sysexecutable = sys.executable
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] rebase: move actual rebase into a single transaction

[Augie Fackler]

On Sat, Mar 04, 2017 at 02:08:26PM -0800, Durham Goode wrote:
> # HG changeset patch
> # User Durham Goode 
> # Date 1488665211 28800
> #  Sat Mar 04 14:06:51 2017 -0800
> # Node ID 9c3ea2112952f398aaa7625f43dcfa36cfd34379
> # Parent  b4cd912d7704cd976e1bee3a3c927e0e578ec88f
> rebase: move actual rebase into a single transaction
>
> Previously, rebasing would open several transaction over the course of 
> rebasing
> several commits. Opening a transaction can have notable overhead (like copying
> the dirstate) which can add up when rebasing many commits.
>
> This patch adds a single large transaction around the actual commit rebase
> operation, with a catch for intervention which serializes the current state if
> we need to drop back to the terminal for user intervention. Amazingly, almost
> all the tests seem to pass.
>
> On large repos with large working copies, this can speed up rebasing 7 commits
> by 25%. I'd expect the percentage to be a bit larger for rebasing even more
> commits.

I like it, seems like the correct thing to do in any case. However...

[...]

> diff --git a/tests/test-rebase-abort.t b/tests/test-rebase-abort.t
> --- a/tests/test-rebase-abort.t
> +++ b/tests/test-rebase-abort.t
>
[...]

> @@ -398,7 +399,7 @@ test aborting an interrupted series (iss
>parent: 0:df4f53cec30a
> base
>branch: default
> -  commit: (clean)
> +  commit: 1 unknown (interrupted update)
>update: 6 new changesets (update)
>phases: 7 draft

What's up with this? Shouldn't rebase --abort put us back in a clean state?

>
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: stable ordering of test output

[Danek Duvall]

Pierre-Yves David wrote:

> >>It's also a problem on the FreeBSD buildbot. I don't know enough about
> >>the bundle2 code to understand how to fix it, but maybe we can figure
> >>out a way to get marmoute an account on a machine taht would help him
> >>diagnose? Danek, do you have a solaris machine you could get him
> >>access to for testing purposes?
> >
> >Yeah, I tried to setup the BSD on the gcc compile farm to debug this,
> >but the machine have such ancient everything else that I finally gave up
> >(after recompiling my own python and a couple of dependency). I did not
> >had time to spend on this since that last attempt. Having an account on
> >something showing the issue would help.
> >
> >On the other hand, this is probably not so bundle2 specific. We have
> >some "select" logic to read stdout and stderr as soon as possible. This
> >is the main suspect as it is possible that this logic behave different
> >under linux and other unix (not too much effort have been put into it).
> >So there is not need of a deep knowledge of bundle2 to debug this if
> >someone else want to give it a shot.
> 
> As perr Augie request on IRC let me be more specific: I would have a look at
> the behavior of the logic in sshpeer.py using "doublepipe" class and the
> associated "util.poll".
> 
> It is responsible for reading the first stream that has data (of stderr and
> stdout). This kind of change in output seems to imply that either the server
> is flushing the stream in different order or that the "ready to read"
> detection is disabled or behaving differently.

It is in fact not specific to bundle2, as I'm also currently seeing the
problem in test-ssh-bundle1.

I would think this is an inherently racy problem, as with any asynchronous
execution, though I would then expect more variance in the results, so
perhaps I'm mistaken here.

It looks like my home machine reproduces the problem.  I'll set you up with
an account and send you the login info.

Thanks,
Danek
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH RFC] help: align description of 'base rev' with reality [issue5488]

[Augie Fackler]

On Tue, Feb 28, 2017 at 03:33:36PM +0100, Kim Alvefur wrote:
> # HG changeset patch
> # User Kim Alvefur 
> # Date 1488291548 -3600
> #  Tue Feb 28 15:19:08 2017 +0100
> # Node ID 397c0aa1cd18439d2ce90a68cd5737fb8dafab49
> # Parent  a185b903bda3c64678d1f9399bfa0be8d326dbff
> help: align description of 'base rev' with reality [issue5488]

Confirmed by reading the code this is correct. Very nice catch.

>
> The text about revlogs seems to be wrong about -1 being used to indicate
> the start of a delta chain. Attempt to correct this.
>
> diff -r a185b903bda3 -r 397c0aa1cd18 mercurial/help/internals/revlogs.txt
> --- a/mercurial/help/internals/revlogs.txtTue Feb 21 18:22:07 2017 +0100
> +++ b/mercurial/help/internals/revlogs.txtTue Feb 28 15:19:08 2017 +0100
> @@ -108,9 +108,9 @@ 12-15 (4 bytes)
>
>  16-19 (4 bytes)
> Base or previous revision this revision's delta was produced against.
> -   -1 means this revision holds full text (as opposed to a delta).
> -   For generaldelta repos, this is the previous revision in the delta
> -   chain. For non-generaldelta repos, this is the base or first
> +   This revision holds full text (as opposed to a delta) if it points to
> +   itself. For generaldelta repos, this is the previous revision in the
> +   delta chain. For non-generaldelta repos, this is the base or first
> revision in the delta chain.
>
>  20-23 (4 bytes)
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH evolve-ext] evolve: switch away from deprecated repo.opener

[Martin von Zweigbergk via Mercurial-devel]

# HG changeset patch
# User Martin von Zweigbergk 
# Date 1488908857 28800
#  Tue Mar 07 09:47:37 2017 -0800
# Node ID 5cf7a405f0e5c089a6da86583eed7c3f99c4ccde
# Parent  a4b09789117a64e4672ba6a2626bbaf965476e91
evolve: switch away from deprecated repo.opener

diff -r a4b09789117a -r 5cf7a405f0e5 hgext3rd/evolve/__init__.py
--- a/hgext3rd/evolve/__init__.py   Thu Mar 02 20:13:47 2017 +0100
+++ b/hgext3rd/evolve/__init__.py   Tue Mar 07 09:47:37 2017 -0800
@@ -1735,7 +1735,7 @@
 # XXX This is a terrible terrible hack, please get rid of it.
 lock = repo.wlock()
 try:
-repo.opener.write('graftstate', orig.hex() + '\n')
+repo.vfs.write('graftstate', orig.hex() + '\n')
 try:
 graftcmd = commands.table['graft'][0]
 ret = graftcmd(ui, repo, old_obsolete=True, **{'continue': 
True})
@@ -1949,7 +1949,7 @@
 tmpctx = repo[tmpid]
 obsolete.createmarkers(repo, [(bumped, (tmpctx,))])
 except MergeFailure:
-repo.opener.write('graftstate', bumped.hex() + '\n')
+repo.vfs.write('graftstate', bumped.hex() + '\n')
 repo.ui.write_err(_('evolution failed!\n'))
 msg = _("fix conflict and run 'hg evolve --continue'\n")
 repo.ui.write_err(msg)
@@ -3267,7 +3267,7 @@
 lock = repo.lock()
 if kwargs.get('old_obsolete'):
 if kwargs.get('continue'):
-obsoleted.extend(repo.opener.read('graftstate').splitlines())
+obsoleted.extend(repo.vfs.read('graftstate').splitlines())
 else:
 obsoleted.extend(revs)
 # convert obsolete target into revs to avoid alias joke
diff -r a4b09789117a -r 5cf7a405f0e5 hgext3rd/evolve/legacy.py
--- a/hgext3rd/evolve/legacy.py Thu Mar 02 20:13:47 2017 +0100
+++ b/hgext3rd/evolve/legacy.py Tue Mar 07 09:47:37 2017 -0800
@@ -51,7 +51,7 @@
 if 'debugc' in arg:
 break
 else:
-data = repo.opener.tryread('obsolete-relations')
+data = repo.vfs.tryread('obsolete-relations')
 if not data:
 data = repo.svfs.tryread('obsoletemarkers')
 if data:
@@ -90,7 +90,7 @@
 store = repo.obsstore
 ### very first format
 try:
-f = repo.opener('obsolete-relations')
+f = repo.vfs('obsolete-relations')
 try:
 some = True
 for line in f:
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] help: fix internals.changegroups

[Augie Fackler]

On Wed, Mar 01, 2017 at 06:38:20PM -0800, Siddharth Agarwal wrote:
> # HG changeset patch
> # User Kyle Lippincott 
> # Date 1488422254 28800
> #  Wed Mar 01 18:37:34 2017 -0800
> # Node ID f39bcd3a0f6d7c2e68948020c243ee53138db7bd
> # Parent  0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
> help: fix internals.changegroups

Queued this, thanks for documenting my (and others') mess.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 2 of 2] share: drop 'relshared' requirement as well

[Augie Fackler]

On Thu, Mar 02, 2017 at 08:11:18AM -0800, Ryan McElroy wrote:
> This series looks good to me.

Agreed, queued, thanks.

>
>
> On 3/2/17 7:35 AM, Yuya Nishihara wrote:
> > # HG changeset patch
> > # User Yuya Nishihara 
> > # Date 1488467511 -32400
> > #  Fri Mar 03 00:11:51 2017 +0900
> > # Node ID 271e966c8d27fbc904dab4c641fe265825384dd3
> > # Parent  8cdc39b46f210a702051b3a6d6f03e2112b41aba
> > share: drop 'relshared' requirement as well
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH V2] similar: compare between actual file contents for exact identity

[Augie Fackler]

On Fri, Mar 03, 2017 at 03:02:27AM +0900, FUJIWARA Katsunori wrote:
> # HG changeset patch
> # User FUJIWARA Katsunori 
> # Date 1488477426 -32400
> #  Fri Mar 03 02:57:06 2017 +0900
> # Node ID d7d47f54019fa900968245163e67ca6f02378995
> # Parent  0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
> similar: compare between actual file contents for exact identity

Sure, seems reasonable. Queued, thanks.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 6 of 6] manifest: rename matches to _matches

[Durham Goode]

On 3/6/17 10:31 PM, Martin von Zweigbergk wrote:

On Mon, Mar 6, 2017 at 5:40 PM, Durham Goode  wrote:



On 3/6/17 5:32 PM, Martin von Zweigbergk wrote:


On Fri, Mar 3, 2017 at 11:34 AM, Durham Goode  wrote:


# HG changeset patch
# User Durham Goode 
# Date 1488517595 28800
#  Thu Mar 02 21:06:35 2017 -0800
# Node ID 1bed8ee65389133923b9076909e04f66ef26f1b8
# Parent  207763e895c7d24885df22f5b9c0df5494d77daf
manifest: rename matches to _matches

Now that there are no external consumers of manifest.matches, let's
rename it to
_matches so it remains internal. This means alternative manifest
implementations
no longer have to implement it, and can instead implement efficient
versions of
diff() and filesnotin().



matches() still seems like a sensible method. Once "hg files" gains a
"-v" or "--flags" option so it can really replace "hg manifest" (or if
"hg manifest" started accepting a file pattern), it seems natural to
implement that using manifest.matches(). Would it be a problem to
queue patches 1-5 but not this one?



It's not a problem to queue 1-5 without 6, but given the performance
criticalness of the manifest, I'd lean towards not giving it functions that
could enable future bad algorithms.  If we need the ability to iterate over
the entire manifest in the future, maybe we could add a
manifest.walk(match=None) function that returns an iterator.


That already exists, but it walks the filenames only, not the
(filename,node,flags) tuples :-) But that's besides the point, I know;
we'd just need to find a different name.


The 'walk()'
part exposes caller to the fact that it is going to be an expensive
operation, and the iterator aspect means we don't have to allocate an entire
dict and we don't have to worry about people abusing it as a way of doing
naive set math using manifests.  That gets the manifest a little closer to
resembling a dirstate as well.


That makes sense. However, there's no such warning to users who do
*not* use a matcher. I think I took care of all those cases two years
ago, but if we want to make it less likely that someone does
set(mf.match(m)), then it seems even more important to prevent
set(mf).  So isn't it __iter__ and friends we should make explicit?


Yep, we should probably head in the direction of making all full 
iteration operations explicit.



I'd also like to be able to verify that this is going in the right
direction. Could you share the remainder of the series too (as a
link)? Thanks.


The only new commits on top of this so far are to change 
treemanifest.diff() to actually limit it's diff based on the visitdir() 
responses of the matcher.  You can see it here: 
https://bpaste.net/show/cba5f57820f1 (two patches, one to update tests, 
one to adjust the diff implementation. I'm working on migrating our 
internal treemanifest imlpementation to use this pattern now, so I can 
get perf numbers. Then I'll come back and do the same for 
treemanifest.filesnotin()

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 3 of 5 V2] localrepo: handle rename with hardlinks properly

[Augie Fackler]

On Thu, Mar 02, 2017 at 10:13:29PM -0800, Jun Wu wrote:
> # HG changeset patch
> # User Jun Wu 
> # Date 1488520170 28800
> #  Thu Mar 02 21:49:30 2017 -0800
> # Node ID 1cf153ec3faaef92c9ad3515372a6d8591195d6e
> # Parent  406f94842e3372f241a151a3ee6ea5f39c04758d
> # Available At https://bitbucket.org/quark-zju/hg-draft
> #  hg pull https://bitbucket.org/quark-zju/hg-draft -r 
> 1cf153ec3faa
> localrepo: handle rename with hardlinks properly

I've taken patches 1-3, and will try to find time to reflect on the
nuances of 4 and 5 soon (but will try and unblock other series first.)

>
> In "aftertrans", we rename "journal.*" to "undo.*". We expect "journal.*"
> files to disappear after renaming.
>
> However, if "journal.foo" and "undo.foo" refer to a same file (hardlink),
> rename may be a no-op, leaving both files on disk, according to Linux
> manpage [1]:
>
> If oldpath and newpath are existing hard links referring to the same
> file, then rename() does nothing, and returns  a  suc‐ cess status.
>
> The POSIX specification [2] is not very clear about what to do.
>
> To be safe, remove "undo.*" before the rename so "journal.*" cannot be left
> on disk.
>
> [1]: http://man7.org/linux/man-pages/man2/rename.2.html
> [2]: http://pubs.opengroup.org/onlinepubs/9699919799/
>
> diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
> --- a/mercurial/localrepo.py
> +++ b/mercurial/localrepo.py
> @@ -2000,4 +2000,12 @@ def aftertrans(files):
>  for vfs, src, dest in renamefiles:
>  try:
> +# if src and dest refer to a same file, vfs.rename is a 
> no-op,
> +# leaving both src and dest on disk. delete dest to make sure
> +# the rename couldn't be such a no-op.
> +vfs.unlink(dest)
> +except OSError as ex:
> +if ex.errno != errno.ENOENT:
> +raise
> +try:
>  vfs.rename(src, dest)
>  except OSError: # journal file does not yet exist
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 2 of 2 V2] dirstate: track otherparent files same as nonnormal

[Augie Fackler]

On Sun, Mar 05, 2017 at 04:51:01PM -0800, Durham Goode wrote:
> # HG changeset patch
> # User Durham Goode 
> # Date 1488761157 28800
> #  Sun Mar 05 16:45:57 2017 -0800
> # Node ID a465bc91a3c1a04c9c75a3cbd13fd5b777487de9
> # Parent  4a751a57ef0682dc3d7e46bdfa67a26b13cd9031
> dirstate: track otherparent files same as nonnormal

Very nice. Queued.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] hook: give exthooks tags for blocking time

[Augie Fackler]

On Mon, Mar 06, 2017 at 03:09:27AM -0800, Simon Farnsworth wrote:
> # HG changeset patch
> # User Simon Farnsworth 
> # Date 1488798555 28800
> #  Mon Mar 06 03:09:15 2017 -0800
> # Node ID 9bdc781849eb9bba369cf9698cb8a1f5aef2b966
> # Parent  b4cd912d7704cd976e1bee3a3c927e0e578ec88f
> hook: give exthooks tags for blocking time

Queued, thanks.

>
> The ui.system autogenerated tag isn't really useful - as they're named, let's
> use the name the user gave us.
>
> diff --git a/mercurial/hook.py b/mercurial/hook.py
> --- a/mercurial/hook.py
> +++ b/mercurial/hook.py
> @@ -142,7 +142,7 @@
>  cwd = repo.root
>  else:
>  cwd = pycompat.getcwd()
> -r = ui.system(cmd, environ=env, cwd=cwd)
> +r = ui.system(cmd, environ=env, cwd=cwd, blockedtag='exthook-%s' % 
> (name,))
>
>  duration = util.timer() - starttime
>  ui.log('exthook', 'exthook-%s: %s finished in %0.2f seconds\n',
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] filemerge: tag merge tool for blocked times

[Augie Fackler]

On Mon, Mar 06, 2017 at 03:19:56AM -0800, Simon Farnsworth wrote:
> # HG changeset patch
> # User Simon Farnsworth 
> # Date 1488799180 28800
> #  Mon Mar 06 03:19:40 2017 -0800
> # Node ID bca31954883ec7ffd16ee940bb84f12f60d286c8
> # Parent  b4cd912d7704cd976e1bee3a3c927e0e578ec88f
> filemerge: tag merge tool for blocked times

Queued, thanks.

>
> Merge tools can take a while - let's ensure that they're appropriately tagged
>
> diff --git a/mercurial/filemerge.py b/mercurial/filemerge.py
> --- a/mercurial/filemerge.py
> +++ b/mercurial/filemerge.py
> @@ -493,7 +493,7 @@
>  repo.ui.status(_('running merge tool %s for file %s\n') %
> (tool, fcd.path()))
>  repo.ui.debug('launching merge tool: %s\n' % cmd)
> -r = ui.system(cmd, cwd=repo.root, environ=env)
> +r = ui.system(cmd, cwd=repo.root, environ=env, blockedtag='mergetool')
>  repo.ui.debug('merge tool returned: %s\n' % r)
>  return True, r, False
>
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 7 of 7] hgk: set a blocked tag when the user invokes view

[Augie Fackler]

On Mon, Mar 06, 2017 at 03:28:08AM -0800, Simon Farnsworth wrote:
> # HG changeset patch
> # User Simon Farnsworth 
> # Date 1488799672 28800
> #  Mon Mar 06 03:27:52 2017 -0800
> # Node ID a24141da65e18e293bcd62f85f05050f01815942
> # Parent  5b8f7a33145a182a4d4985972a8e4425eb20908f
> hgk: set a blocked tag when the user invokes view

Queued, thanks.

>
> diff --git a/hgext/hgk.py b/hgext/hgk.py
> --- a/hgext/hgk.py
> +++ b/hgext/hgk.py
> @@ -345,4 +345,4 @@
>
>  cmd = ui.config("hgk", "path", "hgk") + " %s %s" % (optstr, " 
> ".join(etc))
>  ui.debug("running %s\n" % cmd)
> -ui.system(cmd)
> +ui.system(cmd, blockedtag='hgk_view')
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] revert: always display hunks reversed when reverting to parent

[Augie Fackler]

On Tue, Mar 07, 2017 at 12:18:14AM -0800, Martin von Zweigbergk via 
Mercurial-devel wrote:
> On Mon, Mar 6, 2017 at 5:25 AM, Denis Laxalde  wrote:
> > # HG changeset patch
> > # User Denis Laxalde 
> > # Date 1488805881 -3600
> > #  Mon Mar 06 14:11:21 2017 +0100
> > # Node ID 9f2b436197b02608e1546f57ac6426ec1427c7fd
> > # Parent  b4cd912d7704cd976e1bee3a3c927e0e578ec88f
> > # Available At http://hg.logilab.org/users/dlaxalde/hg
> > #  hg pull http://hg.logilab.org/users/dlaxalde/hg -r 
> > 9f2b436197b0
> > revert: always display hunks reversed when reverting to parent
> >
> > When reverting to the parent of working directory, operation is "discard" so
> > we want hunks to be presented in the same order as the diff (i.e. 
> > "reversed").
> > So we do not query the experimental.revertalternateinteractivemode option in
> > this case and always set "reversehunks" to True.
>
> It's too late in the evening for me to queue this now, but that makes
> sense to me.

I think this is obviously correct. Queued, thanks.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 8 of 8 V2] fileset: add a 'status(...)' predicate to control evaluation context

[Augie Fackler]

On Mon, Mar 06, 2017 at 10:36:55AM +0100, Pierre-Yves David wrote:
> # HG changeset patch
> # User Pierre-Yves David 
> # Date 1488546500 -3600
> #  Fri Mar 03 14:08:20 2017 +0100
> # Node ID 41ea8aee85ca16d652dfdb4afe37053b291702b4
> # Parent  9d6e733046b9aa7e2ded8c4207625fedcc2a8c04
> # EXP-Topic filesetrev-func
> # Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/
> #  hg pull 
> https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r 41ea8aee85ca
> fileset: add a 'status(...)' predicate to control evaluation context

These are queued, thanks.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

mercurial@31175: 7 new changesets

[Mercurial Commits]

7 new changesets in mercurial:

https://www.mercurial-scm.org/repo/hg/rev/48a8b2e5fe31
changeset:   31169:48a8b2e5fe31
user:Yuya Nishihara 
date:Sat Feb 25 16:26:58 2017 +0900
summary: templater: port formatnode filter from changeset_templater

https://www.mercurial-scm.org/repo/hg/rev/e64b70c96338
changeset:   31170:e64b70c96338
user:Yuya Nishihara 
date:Sat Feb 25 22:04:30 2017 +0900
summary: formatter: drop filters argument from maketemplater()

https://www.mercurial-scm.org/repo/hg/rev/1ec89cf0ea49
changeset:   31171:1ec89cf0ea49
user:Yuya Nishihara 
date:Sat Feb 25 16:38:26 2017 +0900
summary: templatekw: move defaulttmpl constant from changeset_templater

https://www.mercurial-scm.org/repo/hg/rev/16272d8c24f6
changeset:   31172:16272d8c24f6
user:Yuya Nishihara 
date:Sat Feb 25 17:00:07 2017 +0900
summary: formatter: add support for changeset templating

https://www.mercurial-scm.org/repo/hg/rev/052e4f1ffce9
changeset:   31173:052e4f1ffce9
user:Yuya Nishihara 
date:Sat Feb 25 17:08:42 2017 +0900
summary: branches: populate all template keywords in formatter

https://www.mercurial-scm.org/repo/hg/rev/842489d18118
changeset:   31174:842489d18118
user:Martin von Zweigbergk 
date:Mon Mar 06 23:19:57 2017 -0800
summary: tests: fix test-update-branches to remove non-conflicting file

https://www.mercurial-scm.org/repo/hg/rev/7433b3bc55ee
changeset:   31175:7433b3bc55ee
bookmark:@
tag: tip
user:Martin von Zweigbergk 
date:Mon Mar 06 23:21:27 2017 -0800
summary: update: for "noconflict" updates, print "conflicting changes" on 
conflict

-- 
Repository URL: https://www.mercurial-scm.org/repo/hg
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 6 of 6] patchbomb: add config knob to generate flags by template (issue5354)

[Augie Fackler]

On Tue, Mar 07, 2017 at 11:35:38PM +0900, Yuya Nishihara wrote:
> # HG changeset patch
> # User Yuya Nishihara 
> # Date 1488015660 -32400
> #  Sat Feb 25 18:41:00 2017 +0900
> # Node ID 90a019a9df1bfb1d65bd729f40cb0b72ff5a896d
> # Parent  471a972c189c7afa01da72df70463717e57032fe
> patchbomb: add config knob to generate flags by template (issue5354)

Queued these, many thanks for the nice feature.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: stable ordering of test output

[Pierre-Yves David]

On 03/07/2017 05:56 PM, Pierre-Yves David wrote:



On 03/07/2017 05:49 PM, Augie Fackler wrote:

On Fri, Mar 03, 2017 at 04:37:54PM -0800, Jun Wu wrote:

Excerpts from Danek Duvall's message of 2017-03-03 14:45:56 -0800:

I frequently get failures like this:

--- .../mercurial.hg/tests/test-bundle2-exchange.t
+++ .../mercurial.hg/tests/test-bundle2-exchange.t.err
@@ -1042,11 +1042,11 @@
   $ hg --config devel.legacy.exchange=bundle1 clone
ssh://user@dummy/bundle2onlyserver not-bundle2-ssh
   requesting all changes
   adding changesets
-  remote: abort: incompatible Mercurial client; bundle2 required
-  remote: (see
https://www.mercurial-scm.org/wiki/IncompatibleClient )
   transaction abort!
   rollback completed
   abort: stream ended unexpectedly (got 0 bytes, expected 4)
+  remote: abort: incompatible Mercurial client; bundle2 required
+  remote: (see
https://www.mercurial-scm.org/wiki/IncompatibleClient )
   [255]

   $ cat > bundle2onlyserver/.hg/hgrc << EOF

ERROR: test-bundle2-exchange.t output changed

It's usually fairly consistent, at least for a period of time, and
then it
goes away.  Presumably it's some sort of fairly stable timing issue,
and
possibly unique to the environment I'm running in (at least, I
assume that
the official tests aren't showing this).

I could patch the tests locally to reorder the lines, but if it's
really an
environmental issue, then that's guaranteed to work consistently
even for
me.

Is this (ever? frequently?) an issue for anyone else?


Yes. We have seen this on our OSX tests. I guess it's "select()"
returning
different things.


It's also a problem on the FreeBSD buildbot. I don't know enough about
the bundle2 code to understand how to fix it, but maybe we can figure
out a way to get marmoute an account on a machine taht would help him
diagnose? Danek, do you have a solaris machine you could get him
access to for testing purposes?


Yeah, I tried to setup the BSD on the gcc compile farm to debug this,
but the machine have such ancient everything else that I finally gave up
(after recompiling my own python and a couple of dependency). I did not
had time to spend on this since that last attempt. Having an account on
something showing the issue would help.

On the other hand, this is probably not so bundle2 specific. We have
some "select" logic to read stdout and stderr as soon as possible. This
is the main suspect as it is possible that this logic behave different
under linux and other unix (not too much effort have been put into it).
So there is not need of a deep knowledge of bundle2 to debug this if
someone else want to give it a shot.


As perr Augie request on IRC let me be more specific: I would have a 
look at the behavior of the logic in sshpeer.py using "doublepipe" class 
and the associated "util.poll".


It is responsible for reading the first stream that has data (of stderr 
and stdout). This kind of change in output seems to imply that either 
the server is flushing the stream in different order or that the "ready 
to read" detection is disabled or behaving differently.



The fix would be pretty straightforward while figuring out the root
cause
(what race condition it is) needs time.


I can't think of any particularly satisfying solution to this, other
than
perhaps separating the remote lines from the local lines, and comparing
each of those independently.  Would that make sense?

Thanks,
Danek

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel




--
Pierre-Yves David
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: stable ordering of test output

[Pierre-Yves David]

On 03/07/2017 05:49 PM, Augie Fackler wrote:

On Fri, Mar 03, 2017 at 04:37:54PM -0800, Jun Wu wrote:

Excerpts from Danek Duvall's message of 2017-03-03 14:45:56 -0800:

I frequently get failures like this:

--- .../mercurial.hg/tests/test-bundle2-exchange.t
+++ .../mercurial.hg/tests/test-bundle2-exchange.t.err
@@ -1042,11 +1042,11 @@
   $ hg --config devel.legacy.exchange=bundle1 clone 
ssh://user@dummy/bundle2onlyserver not-bundle2-ssh
   requesting all changes
   adding changesets
-  remote: abort: incompatible Mercurial client; bundle2 required
-  remote: (see https://www.mercurial-scm.org/wiki/IncompatibleClient )
   transaction abort!
   rollback completed
   abort: stream ended unexpectedly (got 0 bytes, expected 4)
+  remote: abort: incompatible Mercurial client; bundle2 required
+  remote: (see https://www.mercurial-scm.org/wiki/IncompatibleClient )
   [255]

   $ cat > bundle2onlyserver/.hg/hgrc << EOF

ERROR: test-bundle2-exchange.t output changed

It's usually fairly consistent, at least for a period of time, and then it
goes away.  Presumably it's some sort of fairly stable timing issue, and
possibly unique to the environment I'm running in (at least, I assume that
the official tests aren't showing this).

I could patch the tests locally to reorder the lines, but if it's really an
environmental issue, then that's guaranteed to work consistently even for
me.

Is this (ever? frequently?) an issue for anyone else?


Yes. We have seen this on our OSX tests. I guess it's "select()" returning
different things.


It's also a problem on the FreeBSD buildbot. I don't know enough about
the bundle2 code to understand how to fix it, but maybe we can figure
out a way to get marmoute an account on a machine taht would help him
diagnose? Danek, do you have a solaris machine you could get him
access to for testing purposes?


Yeah, I tried to setup the BSD on the gcc compile farm to debug this, 
but the machine have such ancient everything else that I finally gave up 
(after recompiling my own python and a couple of dependency). I did not 
had time to spend on this since that last attempt. Having an account on 
something showing the issue would help.


On the other hand, this is probably not so bundle2 specific. We have 
some "select" logic to read stdout and stderr as soon as possible. This 
is the main suspect as it is possible that this logic behave different 
under linux and other unix (not too much effort have been put into it). 
So there is not need of a deep knowledge of bundle2 to debug this if 
someone else want to give it a shot.



The fix would be pretty straightforward while figuring out the root cause
(what race condition it is) needs time.


I can't think of any particularly satisfying solution to this, other than
perhaps separating the remote lines from the local lines, and comparing
each of those independently.  Would that make sense?

Thanks,
Danek

___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel


--
Pierre-Yves David
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: stable ordering of test output

[Augie Fackler]

On Fri, Mar 03, 2017 at 04:37:54PM -0800, Jun Wu wrote:
> Excerpts from Danek Duvall's message of 2017-03-03 14:45:56 -0800:
> > I frequently get failures like this:
> >
> > --- .../mercurial.hg/tests/test-bundle2-exchange.t
> > +++ .../mercurial.hg/tests/test-bundle2-exchange.t.err
> > @@ -1042,11 +1042,11 @@
> >$ hg --config devel.legacy.exchange=bundle1 clone 
> > ssh://user@dummy/bundle2onlyserver not-bundle2-ssh
> >requesting all changes
> >adding changesets
> > -  remote: abort: incompatible Mercurial client; bundle2 required
> > -  remote: (see https://www.mercurial-scm.org/wiki/IncompatibleClient )
> >transaction abort!
> >rollback completed
> >abort: stream ended unexpectedly (got 0 bytes, expected 4)
> > +  remote: abort: incompatible Mercurial client; bundle2 required
> > +  remote: (see https://www.mercurial-scm.org/wiki/IncompatibleClient )
> >[255]
> >
> >$ cat > bundle2onlyserver/.hg/hgrc << EOF
> >
> > ERROR: test-bundle2-exchange.t output changed
> >
> > It's usually fairly consistent, at least for a period of time, and then it
> > goes away.  Presumably it's some sort of fairly stable timing issue, and
> > possibly unique to the environment I'm running in (at least, I assume that
> > the official tests aren't showing this).
> >
> > I could patch the tests locally to reorder the lines, but if it's really an
> > environmental issue, then that's guaranteed to work consistently even for
> > me.
> >
> > Is this (ever? frequently?) an issue for anyone else?
>
> Yes. We have seen this on our OSX tests. I guess it's "select()" returning
> different things.

It's also a problem on the FreeBSD buildbot. I don't know enough about
the bundle2 code to understand how to fix it, but maybe we can figure
out a way to get marmoute an account on a machine taht would help him
diagnose? Danek, do you have a solaris machine you could get him
access to for testing purposes?

>
> The fix would be pretty straightforward while figuring out the root cause
> (what race condition it is) needs time.
>
> > I can't think of any particularly satisfying solution to this, other than
> > perhaps separating the remote lines from the local lines, and comparing
> > each of those independently.  Would that make sense?
> >
> > Thanks,
> > Danek
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 11 of 11 V5] update: allow setting default update check to "noconflict"

[Augie Fackler]

On Wed, Mar 01, 2017 at 11:54:21AM -0800, Ryan McElroy wrote:
> I'm a very big +1 on this direction -- this is a huge usability improvment.
>
> Facebook currently adds a "--nocheck" flag and turns on "check" by default
> in our "tweakdefaults" extension, but this direction is strictly better than
> that and we will definitely switch over to this "noconflict" functionality
> once it gets into core. I even volunteer to do the work to remove the
> "nocheck" stuff from tweakdefaults and all that.

I feel obligated to point out that 4.1 added a --no-check flag to core
(but 4.2 will let you drop this entirely, hopefully, so probably don't
make your users change muscle memory now.)

>
> Thanks a ton for working on this Martin!
>
>
>
> On 2/27/17 4:31 PM, Martin von Zweigbergk via Mercurial-devel wrote:
> > # HG changeset patch
> > # User Martin von Zweigbergk 
> > # Date 1486973155 28800
> > #  Mon Feb 13 00:05:55 2017 -0800
> > # Node ID c88aa4bc36ee81b0837d2949501d6b4fcf825c38
> > # Parent  ae37f4578e3af6fd4cb7c29c9ab06ad0efd726da
> > update: allow setting default update check to "noconflict"
> >
> > The new value allows update (linear or not) as long as they don't
> > result in file merges.
> >
> > I'm hoping that this value can some day become the default.
> >
> > diff -r ae37f4578e3a -r c88aa4bc36ee mercurial/hg.py
> > --- a/mercurial/hg.py   Mon Feb 13 16:03:05 2017 -0800
> > +++ b/mercurial/hg.py   Mon Feb 13 00:05:55 2017 -0800
> > @@ -733,12 +733,13 @@
> >* none: don't check (merge working directory changes into 
> > destination)
> >* linear: check that update is linear before merging working 
> > directory
> >  changes into destination
> > + * noconflict: check that the update does not result in file merges
> >   This returns whether conflict is detected at updating or not.
> >   """
> >   if updatecheck is None:
> >   updatecheck = ui.config('experimental', 'updatecheck')
> > -if updatecheck not in ('abort', 'none', 'linear'):
> > +if updatecheck not in ('abort', 'none', 'linear', 'noconflict'):
> >   # If not configured, or invalid value configured
> >   updatecheck = 'linear'
> >   with repo.wlock():
> > diff -r ae37f4578e3a -r c88aa4bc36ee mercurial/merge.py
> > --- a/mercurial/merge.pyMon Feb 13 16:03:05 2017 -0800
> > +++ b/mercurial/merge.pyMon Feb 13 00:05:55 2017 -0800
> > @@ -1465,21 +1465,27 @@
> >   The table below shows all the behaviors of the update command
> >   given the -c and -C or no options, whether the working directory
> >   is dirty, whether a revision is specified, and the relationship of
> > -the parent rev to the target rev (linear or not). Match from top first.
> > +the parent rev to the target rev (linear or not). Match from top 
> > first. The
> > +-n option doesn't exist on the command line, but represents the
> > +experimental.updatecheck=noconflict option.
> >   This logic is tested by test-update-branches.t.
> > --c  -C  -m  dirty  rev  linear  |  result
> > - y   y   ** * * |(1)
> > - y   *   y* * * |(1)
> > - *   y   y* * * |(1)
> > - *   *   ** n n | x
> > - *   *   *n * * |ok
> > - n   n   ny * y |   merge
> > - n   n   ny y n |(2)
> > - n   n   yy * * |   merge
> > - n   y   ny * * |  discard
> > - y   n   ny * * |(3)
> > +-c  -C  -n  -m  dirty  rev  linear  |  result
> > + y   y   *   ** * * |(1)
> > + y   *   y   ** * * |(1)
> > + y   *   *   y* * * |(1)
> > + *   y   y   ** * * |(1)
> > + *   y   *   y* * * |(1)
> > + *   *   y   y* * * |(1)
> > + *   *   *   ** n n | x
> > + *   *   *   *n * * |ok
> > + n   n   n   ny * y |   merge
> > + n   n   n   ny y n |(2)
> > + n   n   n   yy * * |   merge
> > + n   n   y   ny * * |  merge if no conflict
> > + n   y   n   ny * * |  discard
> > + y   n   n   ny * * |(3)
> >   x = can't happen
> >   * = don't-care
> > @@ -1499,7 +1505,7 @@
> >   # updatecheck='abort' to better suppport some of these callers.
> >   if updatecheck is None:
> >   updatecheck = 'linear'
> > -assert updatecheck in ('none', 'linear')
> > +assert updatecheck in ('none', 'linear', 'noconflict')
> >   # If we're doing a partial update, we need to skip updating
> >   # the dirstate, so make a note of any partial-ness to the
> >   # update here.
> > @@ -1593,6 +1599,13 @@
> >   repo

[PATCH 6 of 6 py3] config: guard against setconfig specifying unicode values on py3

[Augie Fackler]

# HG changeset patch
# User Augie Fackler 
# Date 1488570176 18000
#  Fri Mar 03 14:42:56 2017 -0500
# Node ID 1bd41b5fdf2b19d4c3bf9fc47a81fc3b85cddf79
# Parent  4801067dee2c77ff4e720c931d8b19cf32515beb
config: guard against setconfig specifying unicode values on py3

This was leading to some difficult to trace problems because the
values were set in one place, but then blew up much later in the
program. Exploding violently with an assertion seems reasonable here.

diff --git a/mercurial/config.py b/mercurial/config.py
--- a/mercurial/config.py
+++ b/mercurial/config.py
@@ -13,6 +13,7 @@ import os
 from .i18n import _
 from . import (
 error,
+pycompat,
 util,
 )
 
@@ -69,6 +70,9 @@ class config(object):
 def items(self, section):
 return self._data.get(section, {}).items()
 def set(self, section, item, value, source=""):
+if pycompat.ispy3:
+assert not isinstance(value, str), (
+'config values may not be unicode strings on Python 3')
 if section not in self:
 self._data[section] = util.sortdict()
 self._data[section][item] = value
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 3 of 6 py3] extensions: tapdance to get reasonable import error formatting

[Augie Fackler]

# HG changeset patch
# User Augie Fackler 
# Date 1488568082 18000
#  Fri Mar 03 14:08:02 2017 -0500
# Node ID adf7419abb89aef332ac5eac48a0f1c9d3eab527
# Parent  3e82376d7d3f3e11d11f09fceb8b4c79233057a8
extensions: tapdance to get reasonable import error formatting

I'm not thrilled with this, but it seems to work.

diff --git a/mercurial/extensions.py b/mercurial/extensions.py
--- a/mercurial/extensions.py
+++ b/mercurial/extensions.py
@@ -18,6 +18,7 @@ from .i18n import (
 
 from . import (
 cmdutil,
+encoding,
 error,
 pycompat,
 util,
@@ -104,11 +105,19 @@ def _importext(name, path=None, reportfu
 mod = _importh(name)
 return mod
 
+def _forbytes(inst):
+"""Portably format an import error into a form suitable for
+%-formatting into bytestrings."""
+if pycompat.ispy3:
+return str(inst).encode('utf-8')
+return inst
+
 def _reportimporterror(ui, err, failed, next):
 # note: this ui.debug happens before --debug is processed,
 #   Use --config ui.debug=1 to see them.
-ui.debug('could not import %s (%s): trying %s\n'
- % (failed, err, next))
+msg = 'could not import %s (%s): trying %s\n' % (
+failed, _forbytes(err), next)
+ui.debug(encoding.tolocal(msg))
 if ui.debugflag:
 ui.traceback()
 
@@ -168,12 +177,13 @@ def loadall(ui):
 except KeyboardInterrupt:
 raise
 except Exception as inst:
+inst = _forbytes(inst)
 if path:
-ui.warn(_("*** failed to import extension %s from %s: %s\n")
-% (name, path, inst))
+fmt = _("*** failed to import extension %s from %s: %s\n")
+ui.warn(encoding.tolocal(fmt % (name, path, inst)))
 else:
-ui.warn(_("*** failed to import extension %s: %s\n")
-% (name, inst))
+fmt = _("*** failed to import extension %s: %s\n")
+ui.warn(encoding.tolocal(fmt % (name, inst)))
 ui.traceback()
 
 for name in _order[newindex:]:
diff --git a/tests/test-check-py3-commands.t b/tests/test-check-py3-commands.t
--- a/tests/test-check-py3-commands.t
+++ b/tests/test-check-py3-commands.t
@@ -12,3 +12,12 @@ The full traceback is hidden to have a s
   warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   debuginstall
   TypeError: Can't convert 'bytes' object to str implicitly
+
+  $ cat > included-hgrc < [extensions]
+  > babar = imaginary_elephant
+  > EOF
+  $ cat >> $HGRCPATH < %include $TESTTMP/included-hgrc
+  > EOF
+  $ $PYTHON3 `which hg` version --config ui.debug=1
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 5 of 6 py3] dispatch: enforce bytes when converting boolean flags to config items

[Augie Fackler]

# HG changeset patch
# User Augie Fackler 
# Date 1488570207 18000
#  Fri Mar 03 14:43:27 2017 -0500
# Node ID 4801067dee2c77ff4e720c931d8b19cf32515beb
# Parent  a6e8bb19707e0c7505ccfdf44f7e1b19a0f65d48
dispatch: enforce bytes when converting boolean flags to config items

This fixes --verbose on Python 3.

diff --git a/mercurial/dispatch.py b/mercurial/dispatch.py
--- a/mercurial/dispatch.py
+++ b/mercurial/dispatch.py
@@ -744,6 +744,8 @@ def _dispatch(req):
 if options['verbose'] or options['debug'] or options['quiet']:
 for opt in ('verbose', 'debug', 'quiet'):
 val = str(bool(options[opt]))
+if pycompat.ispy3:
+val = val.encode('latin1')
 for ui_ in uis:
 ui_.setconfig('ui', opt, val, '--' + opt)
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 4 of 6 py3] py3: add a test that proves %include and missing-extension code works

[Augie Fackler]

# HG changeset patch
# User Augie Fackler 
# Date 1488567128 18000
#  Fri Mar 03 13:52:08 2017 -0500
# Node ID a6e8bb19707e0c7505ccfdf44f7e1b19a0f65d48
# Parent  adf7419abb89aef332ac5eac48a0f1c9d3eab527
py3: add a test that proves %include and missing-extension code works

diff --git a/tests/test-check-py3-commands.t b/tests/test-check-py3-commands.t
--- a/tests/test-check-py3-commands.t
+++ b/tests/test-check-py3-commands.t
@@ -20,4 +20,6 @@ The full traceback is hidden to have a s
   $ cat >> $HGRCPATH < %include $TESTTMP/included-hgrc
   > EOF
-  $ $PYTHON3 `which hg` version --config ui.debug=1
+  $ $PYTHON3 `which hg` version | tail -1
+  *** failed to import extension babar from imaginary_elephant: *: 
'imaginary_elephant' (glob)
+  warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 2 of 6 py3] extensions: use [0:1] slice on config path instead of [0]

[Augie Fackler]

# HG changeset patch
# User Augie Fackler 
# Date 1488565930 18000
#  Fri Mar 03 13:32:10 2017 -0500
# Node ID 3e82376d7d3f3e11d11f09fceb8b4c79233057a8
# Parent  48c957d67d0f40a7134aeca72b98c33eb1546f17
extensions: use [0:1] slice on config path instead of [0]

This behaves the same in Python 2 and Python 3, even though the path
is a bytes.

diff --git a/mercurial/extensions.py b/mercurial/extensions.py
--- a/mercurial/extensions.py
+++ b/mercurial/extensions.py
@@ -160,7 +160,7 @@ def loadall(ui):
 newindex = len(_order)
 for (name, path) in result:
 if path:
-if path[0] == '!':
+if path[0:1] == '!':
 _disabledextensions[name] = path[1:]
 continue
 try:
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 1 of 6 py3] extensions: use inspect module instead of func_code.co_argcount

[Augie Fackler]

# HG changeset patch
# User Augie Fackler 
# Date 1488565641 18000
#  Fri Mar 03 13:27:21 2017 -0500
# Node ID 48c957d67d0f40a7134aeca72b98c33eb1546f17
# Parent  150cd51257221fad5ccba5794e7a21837afba479
extensions: use inspect module instead of func_code.co_argcount

Fixes the extsetup argspec check on Python 3.

diff --git a/mercurial/extensions.py b/mercurial/extensions.py
--- a/mercurial/extensions.py
+++ b/mercurial/extensions.py
@@ -8,6 +8,7 @@
 from __future__ import absolute_import
 
 import imp
+import inspect
 import os
 
 from .i18n import (
@@ -150,7 +151,7 @@ def _runextsetup(name, ui):
 try:
 extsetup(ui)
 except TypeError:
-if extsetup.func_code.co_argcount != 0:
+if inspect.getargspec(extsetup).args:
 raise
 extsetup() # old extsetup with no ui argument
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 6 of 6] patchbomb: add config knob to generate flags by template (issue5354)

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1488015660 -32400
#  Sat Feb 25 18:41:00 2017 +0900
# Node ID 90a019a9df1bfb1d65bd729f40cb0b72ff5a896d
# Parent  471a972c189c7afa01da72df70463717e57032fe
patchbomb: add config knob to generate flags by template (issue5354)

This can be used to flag patches by branch or topic automatically. Flags
optionally given by --flag option are exported as {flags} template keyword,
so you can add --flag V2.

diff --git a/hgext/patchbomb.py b/hgext/patchbomb.py
--- a/hgext/patchbomb.py
+++ b/hgext/patchbomb.py
@@ -60,6 +60,14 @@ overwritten by command line flags like -
   intro=never  # never include an introduction message
   intro=always # always include an introduction message
 
+You can specify a template for flags to be added in subject prefixes. Flags
+specified by --flag option are exported as ``{flags}`` keyword::
+
+  [patchbomb]
+  flagtemplate = "{separate(' ',
+ifeq(branch, 'default', '', branch|upper),
+flags)}"
+
 You can set patchbomb to always ask for confirmation by setting
 ``patchbomb.confirm`` to true.
 '''
@@ -77,11 +85,13 @@ from mercurial import (
 commands,
 encoding,
 error,
+formatter,
 hg,
 mail,
 node as nodemod,
 patch,
 scmutil,
+templater,
 util,
 )
 stringio = util.stringio
@@ -135,9 +145,22 @@ def introwanted(ui, opts, number):
 intro = 1 < number
 return intro
 
+def _formatflags(ui, repo, rev, flags):
+"""build flag string optionally by template"""
+tmpl = ui.config('patchbomb', 'flagtemplate')
+if not tmpl:
+return ' '.join(flags)
+out = util.stringio()
+opts = {'template': templater.unquotestring(tmpl)}
+with formatter.templateformatter(ui, out, 'patchbombflag', opts) as fm:
+fm.startitem()
+fm.context(ctx=repo[rev])
+fm.write('flags', '%s', fm.formatlist(flags, name='flag'))
+return out.getvalue()
+
 def _formatprefix(ui, repo, rev, flags, idx, total, numbered):
 """build prefix to patch subject"""
-flag = ' '.join(flags)
+flag = _formatflags(ui, repo, rev, flags)
 if flag:
 flag = ' ' + flag
 
diff --git a/tests/test-extension.t b/tests/test-extension.t
--- a/tests/test-extension.t
+++ b/tests/test-extension.t
@@ -1113,6 +1113,14 @@ Disabled extensions:
 intro=never  # never include an introduction message
 intro=always # always include an introduction message
   
+  You can specify a template for flags to be added in subject prefixes. Flags
+  specified by --flag option are exported as "{flags}" keyword:
+  
+[patchbomb]
+flagtemplate = "{separate(' ',
+  ifeq(branch, 'default', '', branch|upper),
+  flags)}"
+  
   You can set patchbomb to always ask for confirmation by setting
   "patchbomb.confirm" to true.
   
diff --git a/tests/test-patchbomb.t b/tests/test-patchbomb.t
--- a/tests/test-patchbomb.t
+++ b/tests/test-patchbomb.t
@@ -2371,6 +2371,128 @@ test multi-address parsing:
   
   
 
+test flag template:
+  $ echo foo > intro.text
+  $ hg email --date '1970-1-1 0:1' -n -f quux -t foo -r 0:1 \
+  > --desc intro.text --subject test \
+  > --config patchbomb.flagtemplate='R{rev}'
+  this patch series consists of 2 patches.
+  
+  Cc: 
+  
+  displaying [PATCH 0 of 2 R1] test ...
+  Content-Type: text/plain; charset="us-ascii"
+  MIME-Version: 1.0
+  Content-Transfer-Encoding: 7bit
+  Subject: [PATCH 0 of 2 R1] test
+  Message-Id:  (glob)
+  User-Agent: Mercurial-patchbomb/* (glob)
+  Date: Thu, 01 Jan 1970 00:01:00 +
+  From: quux
+  To: foo
+  
+  foo
+  
+  displaying [PATCH 1 of 2 R0] a ...
+  Content-Type: text/plain; charset="us-ascii"
+  MIME-Version: 1.0
+  Content-Transfer-Encoding: 7bit
+  Subject: [PATCH 1 of 2 R0] a
+  X-Mercurial-Node: 8580ff50825a50c8f716709acdf8de0deddcd6ab
+  X-Mercurial-Series-Index: 1
+  X-Mercurial-Series-Total: 2
+  Message-Id: <8580ff50825a50c8f716.61@*> (glob)
+  X-Mercurial-Series-Id: <8580ff50825a50c8f716.61@*> (glob)
+  In-Reply-To:  (glob)
+  References:  (glob)
+  User-Agent: Mercurial-patchbomb/* (glob)
+  Date: Thu, 01 Jan 1970 00:01:01 +
+  From: quux
+  To: foo
+  
+  # HG changeset patch
+  # User test
+  # Date 1 0
+  #  Thu Jan 01 00:00:01 1970 +
+  # Node ID 8580ff50825a50c8f716709acdf8de0deddcd6ab
+  # Parent  
+  a
+  
+  diff -r  -r 8580ff50825a a
+  --- /dev/nullThu Jan 01 00:00:00 1970 +
+  +++ b/a  Thu Jan 01 00:00:01 1970 +
+  @@ -0,0 +1,1 @@
+  +a
+  
+  displaying [PATCH 2 of 2 R1] b ...
+  Content-Type: text/plain; charset="us-ascii"
+  MIME-Version: 1.0
+  Content-Transfer-Encoding: 7bit
+  Subject: [PATCH 2 of 2 R1] b
+  X-Mercurial-Node: 97d72e5f12c7e84f85064aa72e5a297142c36ed9
+  X-Mercurial-Series-Index: 2
+  X-Mercurial-Series-Total: 2
+  Message-Id: <97d72e5f12c7e84f8506.62@*> (glob)
+  X-Mercurial-Series

[PATCH 5 of 6] patchbomb: pass around ui and revs that are needed for flag template

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1488015334 -32400
#  Sat Feb 25 18:35:34 2017 +0900
# Node ID 471a972c189c7afa01da72df70463717e57032fe
# Parent  619df871ab3248552a7ac982c3916a7632639acd
patchbomb: pass around ui and revs that are needed for flag template

See the next patch for why.

diff --git a/hgext/patchbomb.py b/hgext/patchbomb.py
--- a/hgext/patchbomb.py
+++ b/hgext/patchbomb.py
@@ -135,7 +135,7 @@ def introwanted(ui, opts, number):
 intro = 1 < number
 return intro
 
-def _formatprefix(flags, idx, total, numbered):
+def _formatprefix(ui, repo, rev, flags, idx, total, numbered):
 """build prefix to patch subject"""
 flag = ' '.join(flags)
 if flag:
@@ -147,7 +147,7 @@ def _formatprefix(flags, idx, total, num
 tlen = len(str(total))
 return '[PATCH %0*d of %d%s]' % (tlen, idx, total, flag)
 
-def makepatch(ui, repo, patchlines, opts, _charsets, idx, total, numbered,
+def makepatch(ui, repo, rev, patchlines, opts, _charsets, idx, total, numbered,
   patchname=None):
 
 desc = []
@@ -214,7 +214,8 @@ def makepatch(ui, repo, patchlines, opts
 else:
 msg = mail.mimetextpatch(body, display=opts.get('test'))
 
-prefix = _formatprefix(opts.get('flag'), idx, total, numbered)
+prefix = _formatprefix(ui, repo, rev, opts.get('flag'), idx, total,
+   numbered)
 subj = desc[0].strip().rstrip('. ')
 if not numbered:
 subj = ' '.join([prefix, opts.get('subject') or subj])
@@ -311,14 +312,16 @@ def _getbundlemsgs(repo, sender, bundle,
 msg['Subject'] = mail.headencode(ui, subj, _charsets, opts.get('test'))
 return [(msg, subj, None)]
 
-def _makeintro(repo, sender, patches, **opts):
+def _makeintro(repo, sender, revs, patches, **opts):
 """make an introduction email, asking the user for content if needed
 
 email is returned as (subject, body, cumulative-diffstat)"""
 ui = repo.ui
 _charsets = mail._charsets(ui)
 
-prefix = _formatprefix(opts.get('flag'), 0, len(patches), numbered=True)
+# use the last revision which is likely to be a bookmarked head
+prefix = _formatprefix(ui, repo, revs.last(), opts.get('flag'),
+   0, len(patches), numbered=True)
 subj = (opts.get('subject') or
 prompt(ui, '(optional) Subject: ', rest=prefix, default=''))
 if not subj:
@@ -357,7 +360,7 @@ def _getpatchmsgs(repo, sender, revs, pa
 
 # build the intro message, or skip it if the user declines
 if introwanted(ui, opts, len(patches)):
-msg = _makeintro(repo, sender, patches, **opts)
+msg = _makeintro(repo, sender, revs, patches, **opts)
 if msg:
 msgs.append(msg)
 
@@ -366,10 +369,11 @@ def _getpatchmsgs(repo, sender, revs, pa
 
 # now generate the actual patch messages
 name = None
-for i, p in enumerate(patches):
+assert len(revs) == len(patches)
+for i, (r, p) in enumerate(zip(revs, patches)):
 if patchnames:
 name = patchnames[i]
-msg = makepatch(ui, repo, p, opts, _charsets, i + 1,
+msg = makepatch(ui, repo, r, p, opts, _charsets, i + 1,
 len(patches), numbered, name)
 msgs.append(msg)
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 3 of 6] patchbomb: drop internal option for pbranch extension (API)

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1488014717 -32400
#  Sat Feb 25 18:25:17 2017 +0900
# Node ID 3ce342fc93179cfec1b85722f79d9861cabb52de
# Parent  55baa01cf8615087cdec38e7cae79e09fa9acdd9
patchbomb: drop internal option for pbranch extension (API)

I want to move _getpatches() to _getpatchmsgs() to make sure each patch text
is tied with the corresponding revision number. This helps adding templater
support.

IIRC, the pbranch extension doesn't work with the recent Mercurial versions,
so the removal of this option wouldn't hurt.

diff --git a/hgext/patchbomb.py b/hgext/patchbomb.py
--- a/hgext/patchbomb.py
+++ b/hgext/patchbomb.py
@@ -514,14 +514,12 @@ def email(ui, repo, *revs, **opts):
 mbox = opts.get('mbox')
 outgoing = opts.get('outgoing')
 rev = opts.get('rev')
-# internal option used by pbranches
-patches = opts.get('patches')
 
 if not (opts.get('test') or mbox):
 # really sending
 mail.validateconfig(ui)
 
-if not (revs or rev or outgoing or bundle or patches):
+if not (revs or rev or outgoing or bundle):
 raise error.Abort(_('specify at least one changeset with -r or -o'))
 
 if outgoing and bundle:
@@ -593,10 +591,7 @@ def email(ui, repo, *revs, **opts):
   ui.config('patchbomb', 'from') or
   prompt(ui, 'From', ui.username()))
 
-if patches:
-msgs = _getpatchmsgs(repo, sender, patches, opts.get('patchnames'),
- **opts)
-elif bundle:
+if bundle:
 bundledata = _getbundle(repo, dest, **opts)
 bundleopts = opts.copy()
 bundleopts.pop('bundle', None)  # already processed
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 4 of 6] patchbomb: build patch texts by _getpatchmsgs()

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1488014884 -32400
#  Sat Feb 25 18:28:04 2017 +0900
# Node ID 619df871ab3248552a7ac982c3916a7632639acd
# Parent  3ce342fc93179cfec1b85722f79d9861cabb52de
patchbomb: build patch texts by _getpatchmsgs()

Now _getpatchmsgs() knows revision numbers, which allows us to generate flags
by applying a template to changectx objects.

diff --git a/hgext/patchbomb.py b/hgext/patchbomb.py
--- a/hgext/patchbomb.py
+++ b/hgext/patchbomb.py
@@ -340,7 +340,7 @@ def _makeintro(repo, sender, patches, **
  opts.get('test'))
 return (msg, subj, diffstat)
 
-def _getpatchmsgs(repo, sender, patches, patchnames=None, **opts):
+def _getpatchmsgs(repo, sender, revs, patchnames=None, **opts):
 """return a list of emails from a list of patches
 
 This involves introduction message creation if necessary.
@@ -349,6 +349,7 @@ def _getpatchmsgs(repo, sender, patches,
 """
 ui = repo.ui
 _charsets = mail._charsets(ui)
+patches = list(_getpatches(repo, revs, **opts))
 msgs = []
 
 ui.write(_('this patch series consists of %d patches.\n\n')
@@ -597,8 +598,7 @@ def email(ui, repo, *revs, **opts):
 bundleopts.pop('bundle', None)  # already processed
 msgs = _getbundlemsgs(repo, sender, bundledata, **bundleopts)
 else:
-_patches = list(_getpatches(repo, revs, **opts))
-msgs = _getpatchmsgs(repo, sender, _patches, **opts)
+msgs = _getpatchmsgs(repo, sender, revs, **opts)
 
 showaddrs = []
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 2 of 6] patchbomb: factor out function that builds a prefix string to patch subject

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1488014201 -32400
#  Sat Feb 25 18:16:41 2017 +0900
# Node ID 55baa01cf8615087cdec38e7cae79e09fa9acdd9
# Parent  f56b72c28430d0275457b9b7500df05fdb3d58df
patchbomb: factor out function that builds a prefix string to patch subject

I'll add templating support.

diff --git a/hgext/patchbomb.py b/hgext/patchbomb.py
--- a/hgext/patchbomb.py
+++ b/hgext/patchbomb.py
@@ -135,6 +135,18 @@ def introwanted(ui, opts, number):
 intro = 1 < number
 return intro
 
+def _formatprefix(flags, idx, total, numbered):
+"""build prefix to patch subject"""
+flag = ' '.join(flags)
+if flag:
+flag = ' ' + flag
+
+if not numbered:
+return '[PATCH%s]' % flag
+else:
+tlen = len(str(total))
+return '[PATCH %0*d of %d%s]' % (tlen, idx, total, flag)
+
 def makepatch(ui, repo, patchlines, opts, _charsets, idx, total, numbered,
   patchname=None):
 
@@ -202,16 +214,12 @@ def makepatch(ui, repo, patchlines, opts
 else:
 msg = mail.mimetextpatch(body, display=opts.get('test'))
 
-flag = ' '.join(opts.get('flag'))
-if flag:
-flag = ' ' + flag
-
+prefix = _formatprefix(opts.get('flag'), idx, total, numbered)
 subj = desc[0].strip().rstrip('. ')
 if not numbered:
-subj = '[PATCH%s] %s' % (flag, opts.get('subject') or subj)
+subj = ' '.join([prefix, opts.get('subject') or subj])
 else:
-tlen = len(str(total))
-subj = '[PATCH %0*d of %d%s] %s' % (tlen, idx, total, flag, subj)
+subj = ' '.join([prefix, subj])
 msg['Subject'] = mail.headencode(ui, subj, _charsets, opts.get('test'))
 msg['X-Mercurial-Node'] = node
 msg['X-Mercurial-Series-Index'] = '%i' % idx
@@ -309,13 +317,8 @@ def _makeintro(repo, sender, patches, **
 email is returned as (subject, body, cumulative-diffstat)"""
 ui = repo.ui
 _charsets = mail._charsets(ui)
-tlen = len(str(len(patches)))
 
-flag = opts.get('flag') or ''
-if flag:
-flag = ' ' + ' '.join(flag)
-prefix = '[PATCH %0*d of %d%s]' % (tlen, 0, len(patches), flag)
-
+prefix = _formatprefix(opts.get('flag'), 0, len(patches), numbered=True)
 subj = (opts.get('subject') or
 prompt(ui, '(optional) Subject: ', rest=prefix, default=''))
 if not subj:
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 1 of 6] formatter: add argument to change output file of non-plain formatter

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1421571884 -32400
#  Sun Jan 18 18:04:44 2015 +0900
# Node ID f56b72c28430d0275457b9b7500df05fdb3d58df
# Parent  052e4f1ffce964ffae6333aec4883ef2b4080e4f
formatter: add argument to change output file of non-plain formatter

This allows us to build data not written to the console. That would be
doable by ui.pushbuffer()/popbuffer(), but changing the file object seems
cleaner.

diff --git a/mercurial/formatter.py b/mercurial/formatter.py
--- a/mercurial/formatter.py
+++ b/mercurial/formatter.py
@@ -261,24 +261,26 @@ class plainformatter(baseformatter):
 pass
 
 class debugformatter(baseformatter):
-def __init__(self, ui, topic, opts):
+def __init__(self, ui, out, topic, opts):
 baseformatter.__init__(self, ui, topic, opts, _nullconverter)
-self._ui.write("%s = [\n" % self._topic)
+self._out = out
+self._out.write("%s = [\n" % self._topic)
 def _showitem(self):
-self._ui.write("" + repr(self._item) + ",\n")
+self._out.write("" + repr(self._item) + ",\n")
 def end(self):
 baseformatter.end(self)
-self._ui.write("]\n")
+self._out.write("]\n")
 
 class pickleformatter(baseformatter):
-def __init__(self, ui, topic, opts):
+def __init__(self, ui, out, topic, opts):
 baseformatter.__init__(self, ui, topic, opts, _nullconverter)
+self._out = out
 self._data = []
 def _showitem(self):
 self._data.append(self._item)
 def end(self):
 baseformatter.end(self)
-self._ui.write(pickle.dumps(self._data))
+self._out.write(pickle.dumps(self._data))
 
 def _jsonifyobj(v):
 if isinstance(v, dict):
@@ -299,28 +301,29 @@ def _jsonifyobj(v):
 return '"%s"' % encoding.jsonescape(v)
 
 class jsonformatter(baseformatter):
-def __init__(self, ui, topic, opts):
+def __init__(self, ui, out, topic, opts):
 baseformatter.__init__(self, ui, topic, opts, _nullconverter)
-self._ui.write("[")
+self._out = out
+self._out.write("[")
 self._ui._first = True
 def _showitem(self):
 if self._ui._first:
 self._ui._first = False
 else:
-self._ui.write(",")
+self._out.write(",")
 
-self._ui.write("\n {\n")
+self._out.write("\n {\n")
 first = True
 for k, v in sorted(self._item.items()):
 if first:
 first = False
 else:
-self._ui.write(",\n")
-self._ui.write('  "%s": %s' % (k, _jsonifyobj(v)))
-self._ui.write("\n }")
+self._out.write(",\n")
+self._out.write('  "%s": %s' % (k, _jsonifyobj(v)))
+self._out.write("\n }")
 def end(self):
 baseformatter.end(self)
-self._ui.write("\n]\n")
+self._out.write("\n]\n")
 
 class _templateconverter(object):
 '''convert non-primitive data types to be processed by templater'''
@@ -346,8 +349,9 @@ class _templateconverter(object):
   lambda d: fmt % d[name])
 
 class templateformatter(baseformatter):
-def __init__(self, ui, topic, opts):
+def __init__(self, ui, out, topic, opts):
 baseformatter.__init__(self, ui, topic, opts, _templateconverter)
+self._out = out
 self._topic = topic
 self._t = gettemplater(ui, topic, opts.get('template', ''),
cache=templatekw.defaulttempl)
@@ -371,7 +375,7 @@ class templateformatter(baseformatter):
 else:
 props = self._item
 g = self._t(self._topic, ui=self._ui, cache=self._cache, **props)
-self._ui.write(templater.stringify(g))
+self._out.write(templater.stringify(g))
 
 def lookuptemplate(ui, topic, tmpl):
 # looks like a literal template?
@@ -423,17 +427,17 @@ def maketemplater(ui, topic, tmpl, cache
 def formatter(ui, topic, opts):
 template = opts.get("template", "")
 if template == "json":
-return jsonformatter(ui, topic, opts)
+return jsonformatter(ui, ui, topic, opts)
 elif template == "pickle":
-return pickleformatter(ui, topic, opts)
+return pickleformatter(ui, ui, topic, opts)
 elif template == "debug":
-return debugformatter(ui, topic, opts)
+return debugformatter(ui, ui, topic, opts)
 elif template != "":
-return templateformatter(ui, topic, opts)
+return templateformatter(ui, ui, topic, opts)
 # developer config: ui.formatdebug
 elif ui.configbool('ui', 'formatdebug'):
-return debugformatter(ui, topic, opts)
+return debugformatter(ui, ui, topic, opts)
 # deprecated config: ui.formatjson
 elif ui.configbool('ui', 'formatjson'):
-return jsonformatter(ui, topic, opts)
+return jsonformatter(ui, ui, topic, opts)
 return plainformatter(ui, topic, opts)
__

Re: [PATCH 8 of 8 V2] fileset: add a 'status(...)' predicate to control evaluation context

[Yuya Nishihara]

On Mon, 06 Mar 2017 10:36:55 +0100, Pierre-Yves David wrote:
> # HG changeset patch
> # User Pierre-Yves David 
> # Date 1488546500 -3600
> #  Fri Mar 03 14:08:20 2017 +0100
> # Node ID 41ea8aee85ca16d652dfdb4afe37053b291702b4
> # Parent  9d6e733046b9aa7e2ded8c4207625fedcc2a8c04
> # EXP-Topic filesetrev-func
> # Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/
> #  hg pull 
> https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r 41ea8aee85ca
> fileset: add a 'status(...)' predicate to control evaluation context

Looks good, thanks. I've marked these as 2nd Review Requested since I was too
involved to this series.

> +@predicate('status(base, rev, pattern)')
> +def status(mctx, x):
> +"""``status(base, rev, revspec)``
> +
> +Evaluate predicate using status change between ``base`` and
> +``rev``. Examples:
> +
> +- ``status(3, 7, added())`` - matches files added from "3" to "7"
> +"""
> +repo = mctx.ctx.repo()
> +# i18n: "status" is a keyword
> +b, r, x = getargs(x, 3, 3, _("status takes three arguments"))
> +# i18n: "status" is a keyword
> +baseerr = _("first argument to status must be a revision")
> +baserevspec = getstring(b, baseerr)
> +if not baserevspec:
> +raise error.ParseError(baseerr)

Nit: need to insert "# i18n: "status" is a keyword" comment here.

> +reverr = _("second argument to status must be a revision")
> +revspec = getstring(r, reverr)
> +if not revspec:
> +raise error.ParseError(reverr)
> +basenode, node = scmutil.revpair(repo, [baserevspec, revspec])
> +basectx = repo[basenode]
> +ctx = repo[node]
> +return getset(mctx.switch(ctx, _buildstatus(ctx, x, basectx=basectx)), x)
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 4 of 6 py3] dispatch: allow testedwith to be bytes or str

[Yuya Nishihara]

On Mon, 06 Mar 2017 18:23:08 -0500, Augie Fackler wrote:
> # HG changeset patch
> # User Augie Fackler 
> # Date 1488565765 18000
> #  Fri Mar 03 13:29:25 2017 -0500
> # Node ID 00547c2e1a71fc5d3efd13e5c766fda8bc398e6a
> # Parent  cf5be2a3d1804666dff9a4c99a33354dab7cc322
> dispatch: allow testedwith to be bytes or str
> 
> diff --git a/mercurial/dispatch.py b/mercurial/dispatch.py
> --- a/mercurial/dispatch.py
> +++ b/mercurial/dispatch.py
> @@ -857,6 +857,8 @@ def _exceptionwarning(ui):
>  if ui.config('ui', 'supportcontact', None) is None:
>  for name, mod in extensions.extensions():
>  testedwith = getattr(mod, 'testedwith', '')
> +if pycompat.ispy3 and isinstance(testedwith, str):
> +testedwith = testedwith.encode(u'utf-8')
>  report = getattr(mod, 'buglink', _('the extension author.'))
>  if not testedwith.strip():
>  # We found an untested extension. It's likely the culprit.
> @@ -877,7 +879,7 @@ def _exceptionwarning(ui):
>  worst = name, nearest, report
>  if worst[0] is not None:
>  name, testedwith, report = worst
> -if not isinstance(testedwith, str):
> +if not isinstance(testedwith, (bytes, str)):
>  testedwith = '.'.join([str(c) for c in testedwith])

Perhaps this can be isinstance(testedwith, bytes), but dropping str now
doesn't make sense as we do str(c). So queued with no change.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 6 of 6 py3] schemes: move re construction to module-level and python3-ify

[Yuya Nishihara]

On Mon, 06 Mar 2017 18:23:10 -0500, Augie Fackler wrote:
> # HG changeset patch
> # User Augie Fackler 
> # Date 1488565530 18000
> #  Fri Mar 03 13:25:30 2017 -0500
> # Node ID e36144b27c88bf5a0f2f520d2f2352c290ea4f15
> # Parent  c28e0b997db472d0286927234bf06cc9848b6ff7
> schemes: move re construction to module-level and python3-ify
> 
> This makes the schemes extension load correctly in Python 3.
> 
> diff --git a/hgext/schemes.py b/hgext/schemes.py
> --- a/hgext/schemes.py
> +++ b/hgext/schemes.py
> @@ -63,6 +63,7 @@ command = cmdutil.command(cmdtable)
>  # leave the attribute unspecified.
>  testedwith = 'ships-with-hg-core'
>  
> +_partre = re.compile(r'\{(\d+)\}'.encode(u'latin1'))

Maybe this could be br'' ?
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 1 of 6 py3] config: add sanity assert that files are opened as binary

[Yuya Nishihara]

On Mon, 06 Mar 2017 18:23:05 -0500, Augie Fackler wrote:
> # HG changeset patch
> # User Augie Fackler 
> # Date 1488563711 18000
> #  Fri Mar 03 12:55:11 2017 -0500
> # Node ID 428615d799a9115249f3aab8d6481e384e879f6a
> # Parent  41a9edc5d00f2b5744e1edaf2edaa10084099f53
> config: add sanity assert that files are opened as binary

Fixed minor nits and queued the series, thanks.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 3 of 6 py3] ui: fix ui.traceback on Python 3

[Yuya Nishihara]

On Mon, 06 Mar 2017 18:23:07 -0500, Augie Fackler wrote:
> # HG changeset patch
> # User Augie Fackler 
> # Date 1488568154 18000
> #  Fri Mar 03 14:09:14 2017 -0500
> # Node ID cf5be2a3d1804666dff9a4c99a33354dab7cc322
> # Parent  2e19b1921f0ba7f3550c11f7f3c85e88e732e3a0
> ui: fix ui.traceback on Python 3
> 
> diff --git a/mercurial/ui.py b/mercurial/ui.py
> --- a/mercurial/ui.py
> +++ b/mercurial/ui.py
> @@ -1336,7 +1336,11 @@ class ui(object):
> ''.join(exconly))
>  else:
>  output = traceback.format_exception(exc[0], exc[1], exc[2])
> -self.write_err(''.join(output))
> +data = r''.join(output)
> +if pycompat.ispy3:
> +  enc = encoding.encoding.decode('latin1', errors='replace')
> +  data = data.encode(enc, errors='replace')

Fixed indent and some bytes-vs-unicode issues as follows:

   if pycompat.ispy3:
   enc = pycompat.sysstr(encoding.encoding)
   data = data.encode(enc, errors=r'replace')
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 2 of 2] update: for "noconflict" updates, print "conflicting changes" on conflict

[Yuya Nishihara]

On Mon, 06 Mar 2017 23:50:38 -0800, Martin von Zweigbergk via Mercurial-devel 
wrote:
> # HG changeset patch
> # User Martin von Zweigbergk 
> # Date 1488871287 28800
> #  Mon Mar 06 23:21:27 2017 -0800
> # Node ID 426835372fed3048a413eb210ed527886912cbfc
> # Parent  76aa7c5b624792a316c6e028b77a59aee95e5a3b
> update: for "noconflict" updates, print "conflicting changes" on conflict

Thanks, queued these.

(I only read the first paragraph of Ryan's comment, so missed this one.)
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH stable] fsmonitor: match watchman and local encoding

[Olivier Trempe]

On Mon, Mar 6, 2017 at 8:14 PM, Siddharth Agarwal 
wrote:

> On 3/6/17 09:50, Olivier Trempe wrote:
>
>> # HG changeset patch
>> # User Olivier Trempe 
>> # Date 1488810111 18000
>> #  Mon Mar 06 09:21:51 2017 -0500
>> # Branch stable
>> # Node ID c9d3f8d1a57346228f5c3bb749acdff90d37e194
>> # Parent  6b00c3ecd15b26587de8cca6fab811069cba3b2f
>> fsmonitor: match watchman and local encoding
>>
>> watchman's paths encoding is os dependant. For example, on Windows, it's
>> always utf-8. This causes paths comparison mismatch when paths contain
>> non ascii
>> characters.
>>
>
> I really doubt this is correct unixes, where Watchman returns bytes as
> they are on disk, which matches exactly with what Mercurial wants.
>
> (On Windows Watchman indeed always returns UTF-8.)
>
> This is what I meant by "os dependent". You get a different behavior on a
different os. I can rewrite it to be more Windows specific.

>
>> diff -r 6b00c3ecd15b -r c9d3f8d1a573 hgext/fsmonitor/__init__.py
>> --- a/hgext/fsmonitor/__init__.py   Thu Mar 02 20:19:45 2017 -0500
>> +++ b/hgext/fsmonitor/__init__.py   Mon Mar 06 09:21:51 2017 -0500
>> @@ -99,6 +99,7 @@
>>   from mercurial import (
>>   context,
>>   encoding,
>> +error,
>>   extensions,
>>   localrepo,
>>   merge,
>> @@ -110,6 +111,7 @@
>>   from mercurial import match as matchmod
>> from . import (
>> +pywatchman,
>>   state,
>>   watchmanclient,
>>   )
>> @@ -159,6 +161,20 @@
>>   sha1.update('\0')
>>   return sha1.hexdigest()
>>   +def _watchmanencodingtolocal(path):
>> +"""Fix path to match watchman and local encoding
>> +
>> +watchman's paths encoding is os dependant. For example, on Windows,
>> it's
>>
>
> "dependent"
>
> +always utf-8. This converts watchman encoded paths to local encoding
>> to
>> +avoid paths comparison mismatch.
>> +"""
>> +try:
>> +decoded = pywatchman.encoding.decode_local(path)
>> +except UnicodeDecodeError as e:
>> +raise error.Abort(e, hint='watchman encoding error')
>>
>
> Could you elaborate a bit on when the exception can happen?
>

This is a defensive implementation. I did not encounter this exception
myself. However, even though watchman and pywatchman are maintained
together, there is no guarantee that the user will run the watchman build
matching the pywatchman version distributed with mercurial. Given that
getting the watchman encoding relies on pywatchman's encoding module, you
could get an error if something changes on either side. I just didn't want
to let a possible unhandled exception propagate up to the user.

>
> +
>> +return decoded.encode(encoding.encoding, 'replace')
>> +
>>   def overridewalk(orig, self, match, subrepos, unknown, ignored,
>> full=True):
>>   '''Replacement for dirstate.walk, hooking into Watchman.
>>   @@ -302,7 +318,7 @@
>>   # Watchman tracks files.  We use this property to reconcile deletes
>>   # for name case changes.
>>   for entry in result['files']:
>> -fname = entry['name']
>> +fname = _watchmanencodingtolocal(entry['name'])
>>
>
> Adding a non-trivial function call here is likely going to regress
> performance. Have you measured the impact?


10 calls to the function result in a ~0.3 seconds impact.
I can optimize it by triggering the conversion only if the encoding
returned by pywatchman is different than the local encoding. If they are
the same, the impact is close to null. If not, the impact is ~0.24 seconds.

>
>
- Siddharth
>
>   if switch_slashes:
>>   fname = fname.replace('\\', '/')
>>   if normalize:
>> ___
>> Mercurial-devel mailing list
>> Mercurial-devel@mercurial-scm.org
>> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>>
>
>
>
This is a very nasty bug on windows. In common cases, you don't get
modified files when calling status. No so bad. However, I lost
modifications while doing history editing operations. Files containing
non-ascii characters in their path were silently discarded from a rebased
changeset.

Thanks for your comments. I will submit a new patch when I get feedback
from you.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] revert: always display hunks reversed when reverting to parent

[Denis Laxalde]

Martin von Zweigbergk a écrit :

Relatedly, did you ever resend you patch from last fall that stopped
even reading from revertalternateinteractivemode or was that blocked
by some discussion taking place?


No, I didn't resend it and there was no clear conclusion to the
discussion as far as I can tell.


I still like my idea of using a
different verb for the use case of copying the state from another
commit as opposed to dropping changes since another commit, but it
seems few (if any) others did.


In fact, I do like this over any other proposals (which, if I remember
correctly were either to use another command -- like graft -- with a new
option or have a way to reverse hunks interactively).
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: RFC: bitmap storage for precursors and phases

[Pierre-Yves David]

On 02/18/2017 04:51 AM, Jun Wu wrote:

Excerpts from Stanislau Hlebik's message of 2017-02-17 16:06:33 +:

This is implementation of two caches (nonpublic + precursor) using
serialized sorted lists and sets
https://bitbucket.org/stashlebik/hg/commits/99879579ac2848a2567810b677d8344150a7b319?at=hiddenbitmaps_lists


I had a quick look. Here are my suggestions:

1. Prefer caching at a lower-level, so they get more widely used.


+1 on that,



   Practically, do not change localrepo.py, change phasecache.py and
   obsolete.py instead.

   I happened to do some refactoring in phasecache [1]. I'll send a V2 soon,
   since the change seems related.

   Note: phasecache is some C code so you need some data to prove the change
   is worthy.

2. The "Python set" serialization is already in repoview.py. Reuse it.

   Maybe move "_writehiddencache" to scmutil, or a new simple class.
   "tryreadcache" could be changed and then moved.

   Then the functions could be reused in obsolete.getrevs.

[1]: 
https://www.mercurial-scm.org/pipermail/mercurial-devel/2017-February/092817.html
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel



--
Pierre-Yves David
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: RFC: bitmap storage for precursors and phases

[Pierre-Yves David]

On 02/14/2017 02:04 AM, Sean Farley wrote:

Jun Wu  writes:


In general, I think this is a good direction. Some random thoughts:

  - general purposed

I think the bitmap is not always a cache, so it should only have
operations like set/unset/readfromdisk/writetodisk. Practically, I won't
couple cache invalidation with the bitmap implementation.

In additional, I'll try to avoid using Python-only types in the
interface. So once we decide to rewrite part of the implementation in
native C, we won't have trouble.

See "revset" below for a possibility that bitmap is used as a non-set.

  - revset

This is a possibility that probably won't happen any time soon.

The revset currently uses Python set for maintaining its state. For huge
sets, Python sets may not be a good option. And various operations could
benefit from an always-topologically-sorted set, which is the bitmap.

  - mmap

My intuition is that bitmaps fit better with mmap which can reduce the
reading loading cost. I think "vfs.mmapread" could be a thing, and
various places can benefit from it - Gabor recently showed interest in
loading revlog data by mmap, I had patches that uses mmap to read revlog
index.

In additional, not directly related to this series, I'm a big fan of
single direction data flow. But the current code base does not seem to do a
good job in this area. As we are adding more caching layers to the code
base, it'd be nice if we have some tiny framework maintaining the dependency
of all kinds of data, to be able to understand the data flow easily, and
just to be more confident about loading orders. I think people more
experienced on architecture may want to share some ideas here.


I was thinking about a more high-level approach (please feel free to
pick apart):

r = repo.filtered("bitmap1")
r2 = r.filtered("bitmap2")

So that r2 would be an intersection of bitmap1 and bitmap2 (haven't
thought about a union nor the inverse).



This double filtering idea is interresting. maybe we could have the 
'repoview' API understant repo.filtered("foo+bar") as a combination of 
filtering of foo+bar. The smart part of repoview (eg: filter hierarchy 
for cache inheritance, cache key, etc) should be able to automatically 
compute what do to for a combinaison.


Exposing the bitmap at that level seems strange. I think it is better to 
have the internal implementation of the filtering rely on a bitmat than 
to have the repository/repoview API to expose bitmap directly.


Cheers,

--
Pierre-Yves David
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] localrepo: avoid unnecessary conversion from node to rev

[Pierre-Yves David]

On 02/11/2017 07:44 PM, Gregory Szorc wrote:




On Feb 3, 2017, at 05:16, Yuya Nishihara  wrote:


On Thu, 2 Feb 2017 15:06:46 +, Jun Wu wrote:
This patch looks good to me. See inline comment about how to make it faster.
That could probably be fixed in flight.

Excerpts from Stanislau Hlebik's message of 2017-02-02 02:57:24 -0800:

# HG changeset patch
# User Stanislau Hlebik 
# Date 1486032998 28800
#  Thu Feb 02 02:56:38 2017 -0800
# Node ID 13a528b72173b1228f6eeb0ffc2346e7b78d1d78
# Parent  abf029200e198878a4576a87e095bd8d77d9cea9
localrepo: avoid unnecessary conversion from node to rev

changelog.heads() first calls headrevs then converts them to nodes.
localrepo.heads() then sorts them using self.changelog.rev function and makes
useless conversion back to revs. Instead let's call changelog.headrevs() from
localrepo.heads(), sort the output and then convert to nodes. Because headrevs
does not support start parameter this optimization only works if start is None.

diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -1852,6 +1852,10 @@
  listsubrepos)

def heads(self, start=None):
+if start is None:
+headrevs = sorted(self.changelog.headrevs(), reverse=True)


headrevs() is already sorted in both C and Python implementations, so
"sorted(..., reverse=True)" could be replaced by "reversed(...)".


Yeah.


+return [self.changelog.node(rev) for rev in headrevs]


The function call "self.changelog" wasn't instant because of repoview, but I
don't remember if that's still true. Pierre-Yves?


Pierre-Yves made it substantially faster a few releases ago. But there is still 
enough overhead for it to be problematic.


I'm a bit late to the party, but I confirm that the overhead is 
significant enough that you do not want it to be in loops.



FWIW, just doing a plain attribute lookup in a simple loop like this can 
introduce measurable overhead. We go so far as to alias list.append in a local 
to avoid this in some cases!


--
Pierre-Yves David
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH] revert: always display hunks reversed when reverting to parent

[Martin von Zweigbergk via Mercurial-devel]

On Mon, Mar 6, 2017 at 5:25 AM, Denis Laxalde  wrote:
> # HG changeset patch
> # User Denis Laxalde 
> # Date 1488805881 -3600
> #  Mon Mar 06 14:11:21 2017 +0100
> # Node ID 9f2b436197b02608e1546f57ac6426ec1427c7fd
> # Parent  b4cd912d7704cd976e1bee3a3c927e0e578ec88f
> # Available At http://hg.logilab.org/users/dlaxalde/hg
> #  hg pull http://hg.logilab.org/users/dlaxalde/hg -r 9f2b436197b0
> revert: always display hunks reversed when reverting to parent
>
> When reverting to the parent of working directory, operation is "discard" so
> we want hunks to be presented in the same order as the diff (i.e. "reversed").
> So we do not query the experimental.revertalternateinteractivemode option in
> this case and always set "reversehunks" to True.

It's too late in the evening for me to queue this now, but that makes
sense to me.

Relatedly, did you ever resend you patch from last fall that stopped
even reading from revertalternateinteractivemode or was that blocked
by some discussion taking place? I still like my idea of using a
different verb for the use case of copying the state from another
commit as opposed to dropping changes since another commit, but it
seems few (if any) others did. So, given that, and that the majority
seems to want the default of revertalternateinteractivemode to change
to False, I wonder if we shouldn't do that soon.

>
> diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py
> --- a/mercurial/cmdutil.py
> +++ b/mercurial/cmdutil.py
> @@ -3258,15 +3258,18 @@ def _performrevert(repo, parents, ctx, a
>  diffopts = patch.difffeatureopts(repo.ui, whitespace=True)
>  diffopts.nodates = True
>  diffopts.git = True
> -reversehunks = repo.ui.configbool('experimental',
> -  'revertalternateinteractivemode',
> -  True)
> +operation = 'discard'
> +reversehunks = True
> +if node != parent:
> +operation = 'revert'
> +reversehunks = repo.ui.configbool('experimental',
> +  
> 'revertalternateinteractivemode',
> +  True)
>  if reversehunks:
>  diff = patch.diff(repo, ctx.node(), None, m, opts=diffopts)
>  else:
>  diff = patch.diff(repo, None, ctx.node(), m, opts=diffopts)
>  originalchunks = patch.parsepatch(diff)
> -operation = 'discard' if node == parent else 'revert'
>
>  try:
>
> diff --git a/tests/test-revert-interactive.t b/tests/test-revert-interactive.t
> --- a/tests/test-revert-interactive.t
> +++ b/tests/test-revert-interactive.t
> @@ -380,29 +380,29 @@ Check the experimental config to invert
>3 hunks, 3 lines changed
>examine changes to 'folder1/g'? [Ynesfdaq?] y
>
> -  @@ -1,5 +1,4 @@
> -  -firstline
> +  @@ -1,4 +1,5 @@
> +  +firstline
> c
> 1
> 2
> 3
>discard change 1/3 to 'folder1/g'? [Ynesfdaq?] y
>
> -  @@ -2,7 +1,7 @@
> +  @@ -1,7 +2,7 @@
> c
> 1
> 2
> 3
> -  - 3
> -  +4
> +  -4
> +  + 3
> 5
> d
>discard change 2/3 to 'folder1/g'? [Ynesfdaq?] y
>
> -  @@ -7,3 +6,2 @@
> +  @@ -6,2 +7,3 @@
> 5
> d
> -  -lastline
> +  +lastline
>discard change 3/3 to 'folder1/g'? [Ynesfdaq?] n
>
>$ hg diff --nodates
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel