[PATCH V2] posix: update os.popen with subprocess (issue4746)
# HG changeset patch # User Rishabh Madan # Date 1488526994 -19800 # Fri Mar 03 13:13:14 2017 +0530 # Node ID a134680eec2421b86d7f240b0b06947c1d777502 # Parent 71f692f1f678d86ffb4f95a3621aacfdaeb96b05 posix: update os.popen with subprocess (issue4746) While importing a patch, we use a patch tool, say 'patch --merge', and the patch doesn't merge cleanly, 'patch --merge' will return with exit status 1, but 'hg import' displays 'abort: patch command failed: exited with status 256'. The previous version of this patch replaced deprecated os.popen with subprocess.Popen for mode='r'. But as discussed it would break popen for mode='w'. This patch implements conditional popen for both read/write mode. diff -r 71f692f1f678 -r a134680eec24 mercurial/posix.py --- a/mercurial/posix.pyWed Mar 01 20:22:04 2017 +0100 +++ b/mercurial/posix.pyFri Mar 03 13:13:14 2017 +0530 @@ -16,6 +16,7 @@ import re import select import stat +import subprocess import sys import tempfile import unicodedata @@ -419,7 +420,12 @@ return cmd def popen(command, mode='r'): -return os.popen(command, mode) +if mode == 'r': +return subprocess.Popen(command, shell=True, +stdout=subprocess.PIPE).stdout +else: +return subprocess.Popen(command, shell=True, +stdin=subprocess.PIPE).stdin def testpid(pid): '''return False if pid dead, True if running or not sure''' ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] util: add a getfstype method
I went ahead implementing a test gated by a new hghave check. I'll drop this
from patchwork.
Excerpts from Jun Wu's message of 2017-03-02 16:58:30 -0800:
> Excerpts from Augie Fackler's message of 2017-03-02 19:58:14 -0500:
> > On Thu, Mar 02, 2017 at 11:22:16AM -0800, Jun Wu wrote:
> > > # HG changeset patch
> > > # User Jun Wu
> > > # Date 1488482129 28800
> > > # Thu Mar 02 11:15:29 2017 -0800
> > > # Node ID 1e0decacf511722da03e0297bd846b2379953ae6
> > > # Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
> > > # Available At https://bitbucket.org/quark-zju/hg-draft
> > > # hg pull https://bitbucket.org/quark-zju/hg-draft -r
> > > 1e0decacf511
> > > util: add a getfstype method
> >
> > I'd really like to see this used in context before taking it, or at a
> > minimum a sketch of the full path forward (including some testing notions.)
>
> The test is a bit hard to write - we cannot mount filesystem from userspace
> (in a portable way). So I guess the best thing we can do is probably testing
> equality for two paths that we are sure that they are on a same filesystem.
>
> I'll try to get it used in vfs.
>
> >
> > Thanks!
> >
> > >
> > > We have been very conservative about things like hardlink, flock etc.
> > > because of some buggy (network) filesystem implementations. That's sad
> > > because there are a lot of good (local) filesystem implementations where
> > > optimizations like hardlinks could be just used safely.
> > >
> > > This patch adds a "getfstype" method as a first step to solve the problem.
> > > It only supports Linux for now, as Linux is widely used and could be
> > > supported relatively cleanly. We can add support for other platforms
> > > later.
> > >
> > > diff --git a/mercurial/posix.py b/mercurial/posix.py
> > > --- a/mercurial/posix.py
> > > +++ b/mercurial/posix.py
> > > @@ -658,2 +658,33 @@ def bindunixsocket(sock, path):
> > > os.fchdir(bakwdfd)
> > > os.close(bakwdfd)
> > > +
> > > +if pycompat.sysplatform.startswith('linux'):
> > > +# for Linux, reading /etc/mtab (symlink to /proc/self/mounts) is a
> > > reliable
> > > +# way to get the current filesystem mount information
> > > +def _fstypetable():
> > > +result = {}
> > > +try:
> > > +with open('/etc/mtab', 'r') as f:
> > > +for line in f.read().splitlines():
> > > +name, path, fs, ops, freq, passno = line.split(' ',
> > > 5)
> > > +result[path] = fs
> > > +except OSError:
> > > +# /etc/mtab is not guaranteed available
> > > +pass
> > > +return result
> > > +else:
> > > +# unknown platform
> > > +def _fstypetable():
> > > +return {}
> > > +
> > > +def getfstype(path):
> > > +"""Given a path, return filesystem type or None (best-effort)"""
> > > +table = _fstypetable()
> > > +while True:
> > > +if path in table:
> > > +return table[path]
> > > +nextpath = os.path.dirname(path)
> > > +if nextpath == path:
> > > +return None
> > > +else:
> > > +path = nextpath
> > > diff --git a/mercurial/util.py b/mercurial/util.py
> > > --- a/mercurial/util.py
> > > +++ b/mercurial/util.py
> > > @@ -93,7 +93,8 @@ expandglobs = platform.expandglobs
> > > explainexit = platform.explainexit
> > > findexe = platform.findexe
> > > +getfstype = platform.getfstype
> > > gethgcmd = platform.gethgcmd
> > > +getpid = os.getpid
> > > getuser = platform.getuser
> > > -getpid = os.getpid
> > > groupmembers = platform.groupmembers
> > > groupname = platform.groupname
> > > diff --git a/mercurial/windows.py b/mercurial/windows.py
> > > --- a/mercurial/windows.py
> > > +++ b/mercurial/windows.py
> > > @@ -478,2 +478,7 @@ def readpipe(pipe):
> > > def bindunixsocket(sock, path):
> > > raise NotImplementedError('unsupported platform')
> > > +
> > > +def getfstype(path):
> > > +"""Given a path, return filesystem type or None (best-effort)"""
> > > +# not implemented for Windows
> > > +return None
> > > ___
> > > Mercurial-devel mailing list
> > > Mercurial-devel@mercurial-scm.org
> > > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 3 of 5 V2] localrepo: handle rename with hardlinks properly
# HG changeset patch # User Jun Wu # Date 1488520170 28800 # Thu Mar 02 21:49:30 2017 -0800 # Node ID 1cf153ec3faaef92c9ad3515372a6d8591195d6e # Parent 406f94842e3372f241a151a3ee6ea5f39c04758d # Available At https://bitbucket.org/quark-zju/hg-draft # hg pull https://bitbucket.org/quark-zju/hg-draft -r 1cf153ec3faa localrepo: handle rename with hardlinks properly In "aftertrans", we rename "journal.*" to "undo.*". We expect "journal.*" files to disappear after renaming. However, if "journal.foo" and "undo.foo" refer to a same file (hardlink), rename may be a no-op, leaving both files on disk, according to Linux manpage [1]: If oldpath and newpath are existing hard links referring to the same file, then rename() does nothing, and returns a suc‐ cess status. The POSIX specification [2] is not very clear about what to do. To be safe, remove "undo.*" before the rename so "journal.*" cannot be left on disk. [1]: http://man7.org/linux/man-pages/man2/rename.2.html [2]: http://pubs.opengroup.org/onlinepubs/9699919799/ diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py --- a/mercurial/localrepo.py +++ b/mercurial/localrepo.py @@ -2000,4 +2000,12 @@ def aftertrans(files): for vfs, src, dest in renamefiles: try: +# if src and dest refer to a same file, vfs.rename is a no-op, +# leaving both src and dest on disk. delete dest to make sure +# the rename couldn't be such a no-op. +vfs.unlink(dest) +except OSError as ex: +if ex.errno != errno.ENOENT: +raise +try: vfs.rename(src, dest) except OSError: # journal file does not yet exist ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 1 of 5 V2] dirstate: try to use hardlink to backup dirstate
# HG changeset patch # User Jun Wu # Date 1488419961 28800 # Wed Mar 01 17:59:21 2017 -0800 # Node ID 44cbc7533adfb66e687e4b28312b166f32b01ce0 # Parent b4cd912d7704cd976e1bee3a3c927e0e578ec88f # Available At https://bitbucket.org/quark-zju/hg-draft # hg pull https://bitbucket.org/quark-zju/hg-draft -r 44cbc7533adf dirstate: try to use hardlink to backup dirstate This should be more efficient once util.copyfile has real hardlink support. diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py --- a/mercurial/dirstate.py +++ b/mercurial/dirstate.py @@ -1225,6 +1225,12 @@ class dirstate(object): tr.registertmp(filename, location='plain') -self._opener.write(prefix + self._filename + suffix, - self._opener.tryread(filename)) +backupname = prefix + self._filename + suffix +assert backupname != filename +if self._opener.exists(backupname): +self._opener.unlink(backupname) +# hardlink backup is okay because _writedirstate is always called +# with an "atomictemp=True" file. +util.copyfile(self._opener.join(filename), + self._opener.join(backupname), hardlink=True) def restorebackup(self, tr, suffix='', prefix=''): diff --git a/tests/test-largefiles-small-disk.t b/tests/test-largefiles-small-disk.t --- a/tests/test-largefiles-small-disk.t +++ b/tests/test-largefiles-small-disk.t @@ -6,5 +6,9 @@ Test how largefiles abort in case the di > # > # this makes the original largefiles code abort: + > _origcopyfileobj = shutil.copyfileobj > def copyfileobj(fsrc, fdst, length=16*1024): + > # allow journal files (used by transaction) to be written + > if 'journal.' in fdst.name: + > return _origcopyfileobj(fsrc, fdst, length) > fdst.write(fsrc.read(4)) > raise IOError(errno.ENOSPC, os.strerror(errno.ENOSPC)) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 2 of 5 V2] dirstate: avoid unnecessary load+dump during backup
# HG changeset patch # User Jun Wu # Date 1488421266 28800 # Wed Mar 01 18:21:06 2017 -0800 # Node ID 406f94842e3372f241a151a3ee6ea5f39c04758d # Parent 44cbc7533adfb66e687e4b28312b166f32b01ce0 # Available At https://bitbucket.org/quark-zju/hg-draft # hg pull https://bitbucket.org/quark-zju/hg-draft -r 406f94842e33 dirstate: avoid unnecessary load+dump during backup Previously, dirstate.savebackup unconditionally dumps the dirstate map to disk. It may require loading dirstate first to be able to dump it. Those operations could be expensive if the dirstate is big, and could be avoided if we know the dirstate file is up-to-date. This patch avoids the read and write if the dirstate is clean. In that case, we just do a plain copy without any serialization. This should make commands which use transactions but do not touch dirstate faster. For example, "hg bookmark -r REV NAME". diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py --- a/mercurial/dirstate.py +++ b/mercurial/dirstate.py @@ -1210,6 +1210,7 @@ class dirstate(object): # because the latter omits writing out if transaction is running. # output file will be used to create backup of dirstate at this point. -self._writedirstate(self._opener(filename, "w", atomictemp=True, - checkambig=True)) +if self._dirty or not self._opener.exists(filename): +self._writedirstate(self._opener(filename, "w", atomictemp=True, + checkambig=True)) if tr: ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 5 of 5 V2] util: restore copyfile's hardlink support for some filesystems
# HG changeset patch
# User Jun Wu
# Date 1488521537 28800
# Thu Mar 02 22:12:17 2017 -0800
# Node ID 76c7d930d73ee58b7f38c1a6ce74e264e0613b0f
# Parent d79c818940ff7e29c76ff5e985b920885aa4e7c1
# Available At https://bitbucket.org/quark-zju/hg-draft
# hg pull https://bitbucket.org/quark-zju/hg-draft -r 76c7d930d73e
util: restore copyfile's hardlink support for some filesystems
We know Windows CIFS server implementation has issues with hardlinks. But
local filesystems like ext4 should be safe. So re-enable hardlinks if we
detect such safe filesystems.
This patch removes the global variable "allowhardlinks" added by "util: add
allowhardlinks module variable". Third party extensions wanting to enable
hardlink support unconditionally can wrap the "_ishardlinksupportgenuine"
function instead.
A new hghave check "hardlink-genuine" was added and a
"test-hardlinks-genuine" was copied from "test-hardlinks" to test the real
hardlink support.
diff --git a/mercurial/util.py b/mercurial/util.py
--- a/mercurial/util.py
+++ b/mercurial/util.py
@@ -1058,8 +1058,23 @@ def checksignature(func):
return check
-# Hardlinks are problematic on CIFS, do not allow hardlinks
-# until we find a way to work around it cleanly (issue4546).
-# This is a variable so extensions can opt-in to using them.
-allowhardlinks = False
+# a whilelist of known filesystems that hardlink works reliably
+_genuinehardlinkfslist = set([
+'btrfs',
+'ext3',
+'ext4',
+'tmpfs',
+'xfs',
+])
+
+def _ishardlinksupportgenuine(path):
+'''
+return True if path lives on a filesystem with good hardlink support
+return False if unsure
+
+some filesystems report they support hardlink, but using hardlinks on them
+is problematic. see issue4546 for an example where Windows CIFS
+implementation is broken.
+'''
+return getfstype(path) in _genuinehardlinkfslist
def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
@@ -1079,5 +1094,5 @@ def copyfile(src, dest, hardlink=False,
oldstat = checkambig and filestat(dest)
unlink(dest)
-if allowhardlinks and hardlink:
+if hardlink and _ishardlinksupportgenuine(dest):
try:
oslink(src, dest)
diff --git a/tests/hghave.py b/tests/hghave.py
--- a/tests/hghave.py
+++ b/tests/hghave.py
@@ -347,4 +347,9 @@ def has_hardlink():
os.unlink(fn)
+@check("hardlink-genuine", "hardlinks known to be safe to use")
+def has_hardlink_genuine():
+from mercurial import util
+return util._ishardlinksupportgenuine('.')
+
@check("rmcwd", "can remove current working directory")
def has_rmcwd():
diff --git a/tests/test-hardlinks.t b/tests/test-hardlinks-genuine.t
copy from tests/test-hardlinks.t
copy to tests/test-hardlinks-genuine.t
--- a/tests/test-hardlinks.t
+++ b/tests/test-hardlinks-genuine.t
@@ -1,3 +1,9 @@
#require hardlink
+#require hardlink-genuine
+
+This test is similar to test-hardlinks.t, but will only run on some filesystems
+that we are sure to have known good hardlink supports (see issue4546 for an
+example where the filesystem claims hardlink support but is actually
+problematic).
$ cat > nlinks.py
[PATCH 4 of 5 V2] util: add a getfstype method
# HG changeset patch
# User Jun Wu
# Date 1488521308 28800
# Thu Mar 02 22:08:28 2017 -0800
# Node ID d79c818940ff7e29c76ff5e985b920885aa4e7c1
# Parent 1cf153ec3faaef92c9ad3515372a6d8591195d6e
# Available At https://bitbucket.org/quark-zju/hg-draft
# hg pull https://bitbucket.org/quark-zju/hg-draft -r d79c818940ff
util: add a getfstype method
We have been very conservative about things like hardlink, flock etc.
because of some buggy (network) filesystem implementations. That's sad
because there are a lot of good (local) filesystem implementations where
optimizations like hardlinks could be just used safely.
This patch adds a "getfstype" method as a first step to solve the problem.
It only supports Linux for now, as Linux is widely used and could be
supported relatively cleanly. We can add support for other platforms later.
diff --git a/mercurial/posix.py b/mercurial/posix.py
--- a/mercurial/posix.py
+++ b/mercurial/posix.py
@@ -658,2 +658,40 @@ def bindunixsocket(sock, path):
os.fchdir(bakwdfd)
os.close(bakwdfd)
+
+if pycompat.sysplatform.startswith('linux'):
+# for Linux, reading /etc/mtab (symlink to /proc/self/mounts) is a sane way
+# to get the current filesystem mount information
+def _getfstypetable():
+result = {}
+try:
+with open('/etc/mtab', 'r') as f:
+for line in f.read().splitlines():
+name, path, fs, ops, freq, passno = line.split(' ', 5)
+result[path] = fs
+except OSError:
+# /etc/mtab is not guaranteed available
+pass
+return result
+else:
+# unknown platform
+def _getfstypetable():
+return {}
+
+# cache _getfstypetable() to avoid repetitive expensive queries, it assumes
+# mount table does not change during the lifetime of the process, which is
+# reasonable for short-lived process
+_fstypetable = None
+
+def getfstype(path):
+"""Given a path, return filesystem type or None (best-effort)"""
+global _fstypetable
+if _fstypetable is None:
+_fstypetable = _getfstypetable()
+nextpath = os.path.abspath(path)
+while True:
+path = nextpath
+if path in _fstypetable:
+return _fstypetable[path]
+nextpath = os.path.dirname(path)
+if nextpath == path:
+return None
diff --git a/mercurial/util.py b/mercurial/util.py
--- a/mercurial/util.py
+++ b/mercurial/util.py
@@ -93,4 +93,5 @@ expandglobs = platform.expandglobs
explainexit = platform.explainexit
findexe = platform.findexe
+getfstype = platform.getfstype
gethgcmd = platform.gethgcmd
getuser = platform.getuser
diff --git a/mercurial/windows.py b/mercurial/windows.py
--- a/mercurial/windows.py
+++ b/mercurial/windows.py
@@ -478,2 +478,7 @@ def readpipe(pipe):
def bindunixsocket(sock, path):
raise NotImplementedError('unsupported platform')
+
+def getfstype(path):
+"""Given a path, return filesystem type or None (best-effort)"""
+# not implemented for Windows
+return None
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
mercurial@31134: 5 new changesets
5 new changesets in mercurial: https://www.mercurial-scm.org/repo/hg/rev/6582b3716ae0 changeset: 31130:6582b3716ae0 user:Gregory Szorc date:Wed Feb 15 16:42:17 2017 -0800 summary: minirst: detect bullet lists using asterisks https://www.mercurial-scm.org/repo/hg/rev/50a49ead4db4 changeset: 31131:50a49ead4db4 user:Gregory Szorc date:Wed Feb 15 11:47:14 2017 -0800 summary: minirst: dynamically compile admonitions regexp https://www.mercurial-scm.org/repo/hg/rev/bbdd712e9adb changeset: 31132:bbdd712e9adb user:Gregory Szorc date:Wed Feb 15 11:49:12 2017 -0800 summary: minirst: support passing admonitions into findadmonitions() and parse() https://www.mercurial-scm.org/repo/hg/rev/23080c03a604 changeset: 31133:23080c03a604 user:Dan Villiom Podlaski Christiansen date:Mon Feb 13 14:05:24 2017 +0100 summary: share: add --relative flag to store a relative path to the source https://www.mercurial-scm.org/repo/hg/rev/c22253c4c1b8 changeset: 31134:c22253c4c1b8 bookmark:@ tag: tip user:Durham Goode date:Wed Mar 01 19:51:05 2017 -0800 summary: revert: remove set(mf) because it's O(manifest) -- Repository URL: https://www.mercurial-scm.org/repo/hg ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH V2] util: add allowhardlinks module variable
On Thu, Mar 02, 2017 at 10:12:51AM -0800, Durham Goode wrote: > # HG changeset patch > # User Durham Goode > # Date 1488478360 28800 > # Thu Mar 02 10:12:40 2017 -0800 > # Node ID 32c17aa5fc546a112b355e734fb71b740172cf09 > # Parent a8458fe51a9d155f1daeaffdcf503e674d4d4588 > util: add allowhardlinks module variable Queued, thanks. > > To enable extensions to enable hardlinks for certain environments, let's move > the 'if False' to be an 'if allowhardlinks' and let extensions modify the > allowhardlinks variable. > > Tests on linux ext4 pass with it set to True and to False. > > diff --git a/mercurial/util.py b/mercurial/util.py > --- a/mercurial/util.py > +++ b/mercurial/util.py > @@ -1056,6 +1056,11 @@ def checksignature(func): > > return check > > +# Hardlinks are problematic on CIFS, do not allow hardlinks > +# until we find a way to work around it cleanly (issue4546). > +# This is a variable so extensions can opt-in to using them. > +allowhardlinks = False > + > def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False): > '''copy a file, preserving mode and optionally other stat info like > atime/mtime > @@ -1072,9 +1077,7 @@ def copyfile(src, dest, hardlink=False, > if checkambig: > oldstat = checkambig and filestat(dest) > unlink(dest) > -# hardlinks are problematic on CIFS, quietly ignore this flag > -# until we find a way to work around it cleanly (issue4546) > -if False and hardlink: > +if allowhardlinks and hardlink: > try: > oslink(src, dest) > return > ___ > Mercurial-devel mailing list > Mercurial-devel@mercurial-scm.org > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] util: add a getfstype method
Excerpts from Augie Fackler's message of 2017-03-02 19:58:14 -0500:
> On Thu, Mar 02, 2017 at 11:22:16AM -0800, Jun Wu wrote:
> > # HG changeset patch
> > # User Jun Wu
> > # Date 1488482129 28800
> > # Thu Mar 02 11:15:29 2017 -0800
> > # Node ID 1e0decacf511722da03e0297bd846b2379953ae6
> > # Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
> > # Available At https://bitbucket.org/quark-zju/hg-draft
> > # hg pull https://bitbucket.org/quark-zju/hg-draft -r
> > 1e0decacf511
> > util: add a getfstype method
>
> I'd really like to see this used in context before taking it, or at a
> minimum a sketch of the full path forward (including some testing notions.)
The test is a bit hard to write - we cannot mount filesystem from userspace
(in a portable way). So I guess the best thing we can do is probably testing
equality for two paths that we are sure that they are on a same filesystem.
I'll try to get it used in vfs.
>
> Thanks!
>
> >
> > We have been very conservative about things like hardlink, flock etc.
> > because of some buggy (network) filesystem implementations. That's sad
> > because there are a lot of good (local) filesystem implementations where
> > optimizations like hardlinks could be just used safely.
> >
> > This patch adds a "getfstype" method as a first step to solve the problem.
> > It only supports Linux for now, as Linux is widely used and could be
> > supported relatively cleanly. We can add support for other platforms later.
> >
> > diff --git a/mercurial/posix.py b/mercurial/posix.py
> > --- a/mercurial/posix.py
> > +++ b/mercurial/posix.py
> > @@ -658,2 +658,33 @@ def bindunixsocket(sock, path):
> > os.fchdir(bakwdfd)
> > os.close(bakwdfd)
> > +
> > +if pycompat.sysplatform.startswith('linux'):
> > +# for Linux, reading /etc/mtab (symlink to /proc/self/mounts) is a
> > reliable
> > +# way to get the current filesystem mount information
> > +def _fstypetable():
> > +result = {}
> > +try:
> > +with open('/etc/mtab', 'r') as f:
> > +for line in f.read().splitlines():
> > +name, path, fs, ops, freq, passno = line.split(' ', 5)
> > +result[path] = fs
> > +except OSError:
> > +# /etc/mtab is not guaranteed available
> > +pass
> > +return result
> > +else:
> > +# unknown platform
> > +def _fstypetable():
> > +return {}
> > +
> > +def getfstype(path):
> > +"""Given a path, return filesystem type or None (best-effort)"""
> > +table = _fstypetable()
> > +while True:
> > +if path in table:
> > +return table[path]
> > +nextpath = os.path.dirname(path)
> > +if nextpath == path:
> > +return None
> > +else:
> > +path = nextpath
> > diff --git a/mercurial/util.py b/mercurial/util.py
> > --- a/mercurial/util.py
> > +++ b/mercurial/util.py
> > @@ -93,7 +93,8 @@ expandglobs = platform.expandglobs
> > explainexit = platform.explainexit
> > findexe = platform.findexe
> > +getfstype = platform.getfstype
> > gethgcmd = platform.gethgcmd
> > +getpid = os.getpid
> > getuser = platform.getuser
> > -getpid = os.getpid
> > groupmembers = platform.groupmembers
> > groupname = platform.groupname
> > diff --git a/mercurial/windows.py b/mercurial/windows.py
> > --- a/mercurial/windows.py
> > +++ b/mercurial/windows.py
> > @@ -478,2 +478,7 @@ def readpipe(pipe):
> > def bindunixsocket(sock, path):
> > raise NotImplementedError('unsupported platform')
> > +
> > +def getfstype(path):
> > +"""Given a path, return filesystem type or None (best-effort)"""
> > +# not implemented for Windows
> > +return None
> > ___
> > Mercurial-devel mailing list
> > Mercurial-devel@mercurial-scm.org
> > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] util: add a getfstype method
On Thu, Mar 02, 2017 at 11:22:16AM -0800, Jun Wu wrote:
> # HG changeset patch
> # User Jun Wu
> # Date 1488482129 28800
> # Thu Mar 02 11:15:29 2017 -0800
> # Node ID 1e0decacf511722da03e0297bd846b2379953ae6
> # Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
> # Available At https://bitbucket.org/quark-zju/hg-draft
> # hg pull https://bitbucket.org/quark-zju/hg-draft -r
> 1e0decacf511
> util: add a getfstype method
I'd really like to see this used in context before taking it, or at a
minimum a sketch of the full path forward (including some testing notions.)
Thanks!
>
> We have been very conservative about things like hardlink, flock etc.
> because of some buggy (network) filesystem implementations. That's sad
> because there are a lot of good (local) filesystem implementations where
> optimizations like hardlinks could be just used safely.
>
> This patch adds a "getfstype" method as a first step to solve the problem.
> It only supports Linux for now, as Linux is widely used and could be
> supported relatively cleanly. We can add support for other platforms later.
>
> diff --git a/mercurial/posix.py b/mercurial/posix.py
> --- a/mercurial/posix.py
> +++ b/mercurial/posix.py
> @@ -658,2 +658,33 @@ def bindunixsocket(sock, path):
> os.fchdir(bakwdfd)
> os.close(bakwdfd)
> +
> +if pycompat.sysplatform.startswith('linux'):
> +# for Linux, reading /etc/mtab (symlink to /proc/self/mounts) is a
> reliable
> +# way to get the current filesystem mount information
> +def _fstypetable():
> +result = {}
> +try:
> +with open('/etc/mtab', 'r') as f:
> +for line in f.read().splitlines():
> +name, path, fs, ops, freq, passno = line.split(' ', 5)
> +result[path] = fs
> +except OSError:
> +# /etc/mtab is not guaranteed available
> +pass
> +return result
> +else:
> +# unknown platform
> +def _fstypetable():
> +return {}
> +
> +def getfstype(path):
> +"""Given a path, return filesystem type or None (best-effort)"""
> +table = _fstypetable()
> +while True:
> +if path in table:
> +return table[path]
> +nextpath = os.path.dirname(path)
> +if nextpath == path:
> +return None
> +else:
> +path = nextpath
> diff --git a/mercurial/util.py b/mercurial/util.py
> --- a/mercurial/util.py
> +++ b/mercurial/util.py
> @@ -93,7 +93,8 @@ expandglobs = platform.expandglobs
> explainexit = platform.explainexit
> findexe = platform.findexe
> +getfstype = platform.getfstype
> gethgcmd = platform.gethgcmd
> +getpid = os.getpid
> getuser = platform.getuser
> -getpid = os.getpid
> groupmembers = platform.groupmembers
> groupname = platform.groupname
> diff --git a/mercurial/windows.py b/mercurial/windows.py
> --- a/mercurial/windows.py
> +++ b/mercurial/windows.py
> @@ -478,2 +478,7 @@ def readpipe(pipe):
> def bindunixsocket(sock, path):
> raise NotImplementedError('unsupported platform')
> +
> +def getfstype(path):
> +"""Given a path, return filesystem type or None (best-effort)"""
> +# not implemented for Windows
> +return None
> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] posix: update os.popen with subprocess (issue4746)
Excerpts from Rishabh Madan's message of 2017-03-03 04:31:07 +0530: > # HG changeset patch > # User Rishabh Madan > # Date 1488495530 -19800 > # Fri Mar 03 04:28:50 2017 +0530 > # Node ID dde3cd0607652020b3b2e495e8a688383cf916ba > # Parent 71f692f1f678d86ffb4f95a3621aacfdaeb96b05 > posix: update os.popen with subprocess (issue4746) > > While importing a patch, we use a patch tool, say 'patch --merge', and the > patch doesn't merge cleanly, 'patch --merge' will return with exit status 1, > but 'hg import' displays 'abort: patch command failed: exited with status > 256'. > This patch intends to replace the deprecated os.popen with subprocess.Popen > to return the correct output. > > diff -r 71f692f1f678 -r dde3cd060765 mercurial/posix.py > --- a/mercurial/posix.pyWed Mar 01 20:22:04 2017 +0100 > +++ b/mercurial/posix.pyFri Mar 03 04:28:50 2017 +0530 > @@ -16,6 +16,7 @@ > import re > import select > import stat > +import subprocess > import sys > import tempfile > import unicodedata > @@ -419,7 +420,7 @@ > return cmd > > def popen(command, mode='r'): > -return os.popen(command, mode) > +return subprocess.Popen(command, shell=True, > stdout=subprocess.PIPE).stdout I think this will break popen(mode='w') > > def testpid(pid): > '''return False if pid dead, True if running or not sure''' ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 1 of 3] help: clarify revision / chunk behavior
This series looks fine to me, except for some minor issues I commented individually. Thanks for improving the documentation! In general, I think Patch 1 could be made more friendly to new people. Patch 2 is good. And Patch 3 looks a bit biased on some topics - probably fine - I can improve it on topics like linkrevs and map data structures in the future. Excerpts from Gregory Szorc's message of 2017-02-27 12:54:00 -0800: > # HG changeset patch > # User Gregory Szorc > # Date 1488226671 28800 > # Mon Feb 27 12:17:51 2017 -0800 > # Node ID ded4aedfaffbabce6c083f660fc5fb287f0c > # Parent abb92b3d370e116b29eba4d2e3154e9691c8edbb > help: clarify revision / chunk behavior > > Try to make it easier to understand the differences between the logical > and physical model of revlog storage. > > diff --git a/mercurial/help/internals/revlogs.txt > b/mercurial/help/internals/revlogs.txt > --- a/mercurial/help/internals/revlogs.txt > +++ b/mercurial/help/internals/revlogs.txt > @@ -2,17 +2,18 @@ Revision logs - or *revlogs* - are an ap > storing discrete entries, or *revisions*. They are the primary storage > mechanism of repository data. > > +A revlog revision logically consists of 2 parts: metadata and a content > +blob. Metadata includes the hash of the revision's content, sizes, and > +links to its *parent* entries. The collective metadata is referred > +to as the *index* and the revision content is the *data*. > + > Revlogs effectively model a directed acyclic graph (DAG). Each node > has edges to 1 or 2 *parent* nodes. Each node contains metadata and > the raw value for that node. > > -Revlogs consist of entries which have metadata and revision data. > -Metadata includes the hash of the revision's content, sizes, and > -links to its *parent* entries. The collective metadata is referred > -to as the *index* and the revision data is the *data*. > - > -Revision data is stored as a series of compressed deltas against previous > -revisions. > +The revision data physically stored in a revlog entry is referred to as > +a *chunk*. A *chunk* is either the raw fulltext of a revision or a delta > +against a previous fulltext. In both cases, a *chunk* may be compressed. > > Revlogs are written in an append-only fashion. We never need to rewrite > a file to insert nor do we need to remove data. Rolling back in-progress > @@ -87,7 +88,7 @@ 0-3 (4 bytes) (rev 0 only) > Revlog header > > 0-5 (6 bytes) > - Absolute offset of revision data from beginning of revlog. > + Absolute offset of revision chunk from beginning of revlog. > > 6-7 (2 bytes) > Bit flags impacting revision behavior. The following bit offsets define: > @@ -100,15 +101,15 @@ 6-7 (2 bytes) > 2: REVIDX_EXTSTORED revision data is stored externally. > > 8-11 (4 bytes) > - Compressed length of revision data / chunk as stored in revlog. > + Compressed length of revision chunk as stored in revlog. > > 12-15 (4 bytes) > Uncompressed length of revision data. This is the size of the full > - revision data, not the size of the chunk post decompression. > + revision data (as opposed to the delta/chunk). > > 16-19 (4 bytes) > Base or previous revision this revision's delta was produced against. > - -1 means this revision holds full text (as opposed to a delta). > + -1 means this chunk holds full text (as opposed to a delta). > For generaldelta repos, this is the previous revision in the delta > chain. For non-generaldelta repos, this is the base or first > revision in the delta chain. > @@ -185,16 +186,16 @@ The actual layout of revlog files on dis > *store format*. Typically, a ``.i`` file represents the index revlog > (possibly containing inline data) and a ``.d`` file holds the revision data. > > -Revision Entries > - > +Revision Chunks > +=== > > -Revision entries consist of an optional 1 byte header followed by an > -encoding of the revision data. The headers are as follows: > +Chunks in revision entries consist of an optional 1 byte header followed > +by an encoding of the chunk data. The headers are as follows: > > \0 (0x00) > - Revision data is the entirety of the entry, including this header. > + Chunk data is the entirety of the entry, including this header. > u (0x75) > - Raw revision data follows. > + Raw chunk data follows. > x (0x78) > zlib (RFC 1950) data. > ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH] posix: update os.popen with subprocess (issue4746)
# HG changeset patch # User Rishabh Madan # Date 1488495530 -19800 # Fri Mar 03 04:28:50 2017 +0530 # Node ID dde3cd0607652020b3b2e495e8a688383cf916ba # Parent 71f692f1f678d86ffb4f95a3621aacfdaeb96b05 posix: update os.popen with subprocess (issue4746) While importing a patch, we use a patch tool, say 'patch --merge', and the patch doesn't merge cleanly, 'patch --merge' will return with exit status 1, but 'hg import' displays 'abort: patch command failed: exited with status 256'. This patch intends to replace the deprecated os.popen with subprocess.Popen to return the correct output. diff -r 71f692f1f678 -r dde3cd060765 mercurial/posix.py --- a/mercurial/posix.pyWed Mar 01 20:22:04 2017 +0100 +++ b/mercurial/posix.pyFri Mar 03 04:28:50 2017 +0530 @@ -16,6 +16,7 @@ import re import select import stat +import subprocess import sys import tempfile import unicodedata @@ -419,7 +420,7 @@ return cmd def popen(command, mode='r'): -return os.popen(command, mode) +return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout def testpid(pid): '''return False if pid dead, True if running or not sure''' ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 3 of 3] help: document known deficiencies with revlog format
Excerpts from Gregory Szorc's message of 2017-02-27 12:54:02 -0800: > # HG changeset patch > # User Gregory Szorc > # Date 1488228075 28800 > # Mon Feb 27 12:41:15 2017 -0800 > # Node ID 4ecfe89fa8c1477ab54fa8ef271589a8b20c5497 > # Parent 33cdab923127f930357032db7eb6c4c9441e73ae > help: document known deficiencies with revlog format > > I've been doing a lot of work to properly integrate zstd into revlogs. > Related to this, I've been hacking on multi-threaded APIs for > python-zstandard with the intent that they be usable by Mercurial > to facilitate multi-threaded decompression of delta chains. The > performance of the python-zstandard APIs is *very* promising > (10x read speedup over zlib). However, the existing revlog format > won't be able to take full advantage of the speed potential because > of its current design. > > In other news, a SHA-1 collision now exists and people are starting > to think more seriously about our migration strategy. > > The common theme for these items is the existing revlog format. > As good as it is, it has some limitations. > > The patch introduces documentation on known revlog deficiencies > and areas for improvement. I think I captured most of the low-level > problems. Some higher-level problems (such as filelog data for > copies/renames) or linkrev adjustment aren't captured. Someone > whose thought more about those can contribute write-ups of those. > > I think documenting these deficiencies in the spec (as opposed to > say the wiki) is important because the problems reside with the > specification. Putting the deficiencies in the spec maximizes their > visibility and increases the chances that mistakes won't be repeated. > > diff --git a/mercurial/help/internals/revlogs.txt > b/mercurial/help/internals/revlogs.txt > --- a/mercurial/help/internals/revlogs.txt > +++ b/mercurial/help/internals/revlogs.txt > @@ -217,3 +217,183 @@ 1. Hash the parent nodes > 2. Hash the fulltext of the revision > > The 20 byte node ids of the parents are fed into the hasher in ascending > order. > + > +Deficiencies and Areas for Improvement > +== > + > +This section documents known problems with the revlog format and ways it can > +be improved. > + > +4 byte header is shared with first revision > + The first 4 bytes of the revlog are shared with the *absolute offset* > + field of the first revision's index entry. This is a clever optimization. > + But it requires one-off handling when parsing index entries. > + > + Dropping the optimization and having full separation between the revlog > + header and index data would make parsing simpler at the cost of 4 extra > + bytes per revlog. I guess there is some consideration about alignment / performance. For revlog version, I think it's unlikely to have mixed versions of revlogs so it's possible to use different directory names (or repo requirement) to denote the version of revlog used by the repo, like ".hg/storev2/". So we don't pay the possible alignment / offset calculation cost. > +Integers stored as big-endian > + Integers in revlog index entries are stored as big-endian. Most systems > + that Mercurial runs on are little-endian. > + > + Having a bit ordering mismatch between index entries and the running > + machine means that integers must be converted before they can be > + used. This means that a revlog implementation in a systems language on > + little-endian machines can't simply cast a memory address as a struct to > + read index entries. This adds overhead in the form of data conversion > + and extra allocations. > + > + Changing revlog data to little-endian would facilitate 0-copy operations > + on most machines that Mercurial runs on and would make things faster. > + > +Delta chain base revision not explicitly recorded for generaldelta > + For generaldelta revlogs, the index entry stores the revision for the > + previous revision in the delta chain. To find the first or base revision > + of the delta chain, one must walk index entries. > + > + Revlogs limit the linear distance between chunk data in delta chains. > + This requires resolving the base revision for a delta chain during > + insertions. Insertion-heavy operations like changegroup application > + thus require resolving several delta chain bases. This can lead to a > + performance bottleneck resolving delta chains. That's why the > + Python layer implements an LRU cache of delta chain bases. See > + 92ac2baaea86 for more. > + > + Adding the delta chain base revision to index entries should be considered > + as a way to mitigate this performance issue. Note: existing performance > + issues with delta chain computation could be the result of delta chain > + traversal being implemented in Python. It is possible an efficient > + implementation in C would make delta chain base resolution "fast enough." > + > +Raw chunk size not explicitly recorded > + M
Re: [PATCH 1 of 3] help: clarify revision / chunk behavior
Excerpts from Gregory Szorc's message of 2017-02-27 12:54:00 -0800: > # HG changeset patch > # User Gregory Szorc > # Date 1488226671 28800 > # Mon Feb 27 12:17:51 2017 -0800 > # Node ID ded4aedfaffbabce6c083f660fc5fb287f0c > # Parent abb92b3d370e116b29eba4d2e3154e9691c8edbb > help: clarify revision / chunk behavior > > Try to make it easier to understand the differences between the logical > and physical model of revlog storage. > > diff --git a/mercurial/help/internals/revlogs.txt > b/mercurial/help/internals/revlogs.txt > --- a/mercurial/help/internals/revlogs.txt > +++ b/mercurial/help/internals/revlogs.txt > @@ -2,17 +2,18 @@ Revision logs - or *revlogs* - are an ap > storing discrete entries, or *revisions*. They are the primary storage > mechanism of repository data. > > +A revlog revision logically consists of 2 parts: metadata and a content "revision" is undefined to a new person reading here. How about moving it to the paragraph below, and replacing it with "node" (or, make it clear that a "node" is a "revision") ? > +blob. Metadata includes the hash of the revision's content, sizes, and > +links to its *parent* entries. The collective metadata is referred > +to as the *index* and the revision content is the *data*. > + > Revlogs effectively model a directed acyclic graph (DAG). Each node > has edges to 1 or 2 *parent* nodes. Each node contains metadata and > the raw value for that node. > > -Revlogs consist of entries which have metadata and revision data. > -Metadata includes the hash of the revision's content, sizes, and > -links to its *parent* entries. The collective metadata is referred > -to as the *index* and the revision data is the *data*. Actually I think the old version is good enough and in a better order - first introduce the DAG concept, then explain details. > - > -Revision data is stored as a series of compressed deltas against previous > -revisions. I'd keep the above sentence - it's concise and does not hurt. > +The revision data physically stored in a revlog entry is referred to as "entry" vs "revision" vs "node" could confuse new people. > +a *chunk*. A *chunk* is either the raw fulltext of a revision or a delta > +against a previous fulltext. In both cases, a *chunk* may be compressed. I'd say "against another revision". "previous" may imply rev-1. "fulltext" may imply that delta base cannot be a delta. > Revlogs are written in an append-only fashion. We never need to rewrite > a file to insert nor do we need to remove data. Rolling back in-progress > @@ -87,7 +88,7 @@ 0-3 (4 bytes) (rev 0 only) > Revlog header > > 0-5 (6 bytes) > - Absolute offset of revision data from beginning of revlog. > + Absolute offset of revision chunk from beginning of revlog. > > 6-7 (2 bytes) > Bit flags impacting revision behavior. The following bit offsets define: > @@ -100,15 +101,15 @@ 6-7 (2 bytes) > 2: REVIDX_EXTSTORED revision data is stored externally. > > 8-11 (4 bytes) > - Compressed length of revision data / chunk as stored in revlog. > + Compressed length of revision chunk as stored in revlog. > > 12-15 (4 bytes) > Uncompressed length of revision data. This is the size of the full > - revision data, not the size of the chunk post decompression. > + revision data (as opposed to the delta/chunk). > > 16-19 (4 bytes) > Base or previous revision this revision's delta was produced against. > - -1 means this revision holds full text (as opposed to a delta). > + -1 means this chunk holds full text (as opposed to a delta). > For generaldelta repos, this is the previous revision in the delta > chain. For non-generaldelta repos, this is the base or first > revision in the delta chain. > @@ -185,16 +186,16 @@ The actual layout of revlog files on dis > *store format*. Typically, a ``.i`` file represents the index revlog > (possibly containing inline data) and a ``.d`` file holds the revision data. > > -Revision Entries > - > +Revision Chunks > +=== > > -Revision entries consist of an optional 1 byte header followed by an > -encoding of the revision data. The headers are as follows: > +Chunks in revision entries consist of an optional 1 byte header followed > +by an encoding of the chunk data. The headers are as follows: > > \0 (0x00) > - Revision data is the entirety of the entry, including this header. > + Chunk data is the entirety of the entry, including this header. > u (0x75) > - Raw revision data follows. > + Raw chunk data follows. > x (0x78) > zlib (RFC 1950) data. > These changes look good to me. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 3 of 4 V2] chgcache: implement the background preloading thread
Excerpts from Yuya Nishihara's message of 2017-03-03 00:51:12 +0900: > I hope so, but googling "fork thread" or "fork thread python" made me feel > bad. I haven't read them carefully, but they seemed to say "don't do it." Good catch! I had a look at CPython source code. The most related function is PyOS_AfterFork(void), which will be executed by the child. It does not handle all lock issues (like malloc() in libc). I think we can use os.fork() in master and avoid threading to solve the issue: worker masterforked master | send(repopath) -> | | fork() -> | | preload() | replace master's socket | connect() --> | This solves other problems - like parsers.c has some slow functions that may block accept(). ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] transaction: enable hardlink backups for non-windows systems
On 3/2/17 11:55 AM, Augie Fackler wrote: On Mar 2, 2017, at 12:37 PM, Durham Goode wrote: On 3/2/17 8:09 AM, Ryan McElroy wrote: On 3/2/17 7:46 AM, Yuya Nishihara wrote: On Wed, 1 Mar 2017 17:55:25 -0800, Jun Wu wrote: According to https://urldefense.proofpoint.com/v2/url?u=https-3A__bz.mercurial-2Dscm.org_show-5Fbug.cgi-3Fid-3D4546-3A&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=vjW-XcGbq_x7AbgPtmJ8lHDdDLYljlyx2xvUxSdf4IE&s=6scd8qQkJR-0MXuZo-p47gjTcBa4U3m-ZywPbWfxOKI&e= Testing with mercurial 3.2 and 3.3 to a Linux samba server worked without issue in the few tests I've done. This suggests the most likely cause is an NTFS/Windows based file server? So I think at least we can have a whitelist (instead of a blacklist) that allows Linux to use real hardlinks. Just ignore OS X or Windows for now. She said a Linux samba server had no problem, not a Linux client with Windows server. Sounds like just changing this is a non-starter then. How about using a config option, so people like us can get the perf benefits in places where we are not concerned about the possible downsides (because we prevent clones onto NFS/CIFS shares, for example)? Would anyone object to turning this into an option that defaults to off? Might be hard to get a config option down to this level, since there's no ui object around here, but we could probably move the 'False' value to a module level variable so extensions could at least set it to True. I worry about such a config knob (see my previous statements about people actually choosing to turn on SVN_I_LOVE_CORRUPTED_WORKING_COPIES_SO_DISABLE_SLEEP_FOR_TIMESTAMPS and then being surprised when they get the promised corruption) in core. I’d welcome a change like this that would let an out-of-tree extension with extensive warnings make the desired changes though. Earlier this morning I sent a patch "util: add allowhardlinks module variable" V2 to do just this. Let me know if you think it needs more scary comments. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] transaction: enable hardlink backups for non-windows systems
> On Mar 2, 2017, at 12:37 PM, Durham Goode wrote: > > On 3/2/17 8:09 AM, Ryan McElroy wrote: >> >> >> On 3/2/17 7:46 AM, Yuya Nishihara wrote: >>> On Wed, 1 Mar 2017 17:55:25 -0800, Jun Wu wrote: According to https://urldefense.proofpoint.com/v2/url?u=https-3A__bz.mercurial-2Dscm.org_show-5Fbug.cgi-3Fid-3D4546-3A&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=vjW-XcGbq_x7AbgPtmJ8lHDdDLYljlyx2xvUxSdf4IE&s=6scd8qQkJR-0MXuZo-p47gjTcBa4U3m-ZywPbWfxOKI&e= Testing with mercurial 3.2 and 3.3 to a Linux samba server worked without issue in the few tests I've done. This suggests the most likely cause is an NTFS/Windows based file server? So I think at least we can have a whitelist (instead of a blacklist) that allows Linux to use real hardlinks. Just ignore OS X or Windows for now. >>> She said a Linux samba server had no problem, not a Linux client with >>> Windows >>> server. >> >> Sounds like just changing this is a non-starter then. How about using a >> config option, so people like us can get the perf benefits in places >> where we are not concerned about the possible downsides (because we >> prevent clones onto NFS/CIFS shares, for example)? Would anyone object >> to turning this into an option that defaults to off? > > Might be hard to get a config option down to this level, since there's no ui > object around here, but we could probably move the 'False' value to a module > level variable so extensions could at least set it to True. I worry about such a config knob (see my previous statements about people actually choosing to turn on SVN_I_LOVE_CORRUPTED_WORKING_COPIES_SO_DISABLE_SLEEP_FOR_TIMESTAMPS and then being surprised when they get the promised corruption) in core. I’d welcome a change like this that would let an out-of-tree extension with extensive warnings make the desired changes though. > ___ > Mercurial-devel mailing list > Mercurial-devel@mercurial-scm.org > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] util: add a getfstype method
Excerpts from Jun Wu's message of 2017-03-02 11:22:16 -0800:
> # HG changeset patch
> # User Jun Wu
> # Date 1488482129 28800
> # Thu Mar 02 11:15:29 2017 -0800
> # Node ID 1e0decacf511722da03e0297bd846b2379953ae6
> # Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
> # Available At https://bitbucket.org/quark-zju/hg-draft
> # hg pull https://bitbucket.org/quark-zju/hg-draft -r
> 1e0decacf511
> util: add a getfstype method
>
> We have been very conservative about things like hardlink, flock etc.
> because of some buggy (network) filesystem implementations. That's sad
> because there are a lot of good (local) filesystem implementations where
> optimizations like hardlinks could be just used safely.
>
> This patch adds a "getfstype" method as a first step to solve the problem.
> It only supports Linux for now, as Linux is widely used and could be
> supported relatively cleanly. We can add support for other platforms later.
>
> diff --git a/mercurial/posix.py b/mercurial/posix.py
> --- a/mercurial/posix.py
> +++ b/mercurial/posix.py
> @@ -658,2 +658,33 @@ def bindunixsocket(sock, path):
> os.fchdir(bakwdfd)
> os.close(bakwdfd)
> +
> +if pycompat.sysplatform.startswith('linux'):
> +# for Linux, reading /etc/mtab (symlink to /proc/self/mounts) is a
> reliable
> +# way to get the current filesystem mount information
> +def _fstypetable():
> +result = {}
> +try:
> +with open('/etc/mtab', 'r') as f:
> +for line in f.read().splitlines():
> +name, path, fs, ops, freq, passno = line.split(' ', 5)
> +result[path] = fs
> +except OSError:
> +# /etc/mtab is not guaranteed available
> +pass
> +return result
> +else:
> +# unknown platform
> +def _fstypetable():
> +return {}
> +
> +def getfstype(path):
> +"""Given a path, return filesystem type or None (best-effort)"""
> +table = _fstypetable()
> +while True:
> +if path in table:
> +return table[path]
> +nextpath = os.path.dirname(path)
> +if nextpath == path:
> +return None
> +else:
> +path = nextpath
> diff --git a/mercurial/util.py b/mercurial/util.py
> --- a/mercurial/util.py
> +++ b/mercurial/util.py
> @@ -93,7 +93,8 @@ expandglobs = platform.expandglobs
> explainexit = platform.explainexit
> findexe = platform.findexe
> +getfstype = platform.getfstype
> gethgcmd = platform.gethgcmd
> +getpid = os.getpid
> getuser = platform.getuser
> -getpid = os.getpid
Sorry. This is unintentional. I thought those names were sorted
alphabetically.
> groupmembers = platform.groupmembers
> groupname = platform.groupname
> diff --git a/mercurial/windows.py b/mercurial/windows.py
> --- a/mercurial/windows.py
> +++ b/mercurial/windows.py
> @@ -478,2 +478,7 @@ def readpipe(pipe):
> def bindunixsocket(sock, path):
> raise NotImplementedError('unsupported platform')
> +
> +def getfstype(path):
> +"""Given a path, return filesystem type or None (best-effort)"""
> +# not implemented for Windows
> +return None
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH] util: add a getfstype method
# HG changeset patch
# User Jun Wu
# Date 1488482129 28800
# Thu Mar 02 11:15:29 2017 -0800
# Node ID 1e0decacf511722da03e0297bd846b2379953ae6
# Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
# Available At https://bitbucket.org/quark-zju/hg-draft
# hg pull https://bitbucket.org/quark-zju/hg-draft -r 1e0decacf511
util: add a getfstype method
We have been very conservative about things like hardlink, flock etc.
because of some buggy (network) filesystem implementations. That's sad
because there are a lot of good (local) filesystem implementations where
optimizations like hardlinks could be just used safely.
This patch adds a "getfstype" method as a first step to solve the problem.
It only supports Linux for now, as Linux is widely used and could be
supported relatively cleanly. We can add support for other platforms later.
diff --git a/mercurial/posix.py b/mercurial/posix.py
--- a/mercurial/posix.py
+++ b/mercurial/posix.py
@@ -658,2 +658,33 @@ def bindunixsocket(sock, path):
os.fchdir(bakwdfd)
os.close(bakwdfd)
+
+if pycompat.sysplatform.startswith('linux'):
+# for Linux, reading /etc/mtab (symlink to /proc/self/mounts) is a reliable
+# way to get the current filesystem mount information
+def _fstypetable():
+result = {}
+try:
+with open('/etc/mtab', 'r') as f:
+for line in f.read().splitlines():
+name, path, fs, ops, freq, passno = line.split(' ', 5)
+result[path] = fs
+except OSError:
+# /etc/mtab is not guaranteed available
+pass
+return result
+else:
+# unknown platform
+def _fstypetable():
+return {}
+
+def getfstype(path):
+"""Given a path, return filesystem type or None (best-effort)"""
+table = _fstypetable()
+while True:
+if path in table:
+return table[path]
+nextpath = os.path.dirname(path)
+if nextpath == path:
+return None
+else:
+path = nextpath
diff --git a/mercurial/util.py b/mercurial/util.py
--- a/mercurial/util.py
+++ b/mercurial/util.py
@@ -93,7 +93,8 @@ expandglobs = platform.expandglobs
explainexit = platform.explainexit
findexe = platform.findexe
+getfstype = platform.getfstype
gethgcmd = platform.gethgcmd
+getpid = os.getpid
getuser = platform.getuser
-getpid = os.getpid
groupmembers = platform.groupmembers
groupname = platform.groupname
diff --git a/mercurial/windows.py b/mercurial/windows.py
--- a/mercurial/windows.py
+++ b/mercurial/windows.py
@@ -478,2 +478,7 @@ def readpipe(pipe):
def bindunixsocket(sock, path):
raise NotImplementedError('unsupported platform')
+
+def getfstype(path):
+"""Given a path, return filesystem type or None (best-effort)"""
+# not implemented for Windows
+return None
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] revert: remove set(mf) because it's O(repo)
On Thu, Mar 2, 2017 at 10:21 AM, Durham Goode wrote: > > > On 3/2/17 9:57 AM, Martin von Zweigbergk wrote: >> >> On Wed, Mar 1, 2017 at 8:01 PM, Durham Goode wrote: >>> >>> # HG changeset patch >>> # User Durham Goode >>> # Date 1488426665 28800 >>> # Wed Mar 01 19:51:05 2017 -0800 >>> # Node ID a8458fe51a9d155f1daeaffdcf503e674d4d4588 >>> # Parent b787c41767339158927232ec7a9092196e887453 >>> revert: remove set(mf) because it's O(repo) >> >> >> O(repo) sounds misleading to me, because it sounds like number of >> revisions matter too. I'd change it to O(manifest) >> >>> >>> The revert code had a 'set(manifest)' line in it, which has a runtime >>> equivalent >>> to the size of the working copy. With alternative manifest >>> implementations, like >>> treemanifest, this can be extra expensive. Let's rewrite it to be >>> O(changes) >>> instead of O(working copy size). >> >> >> Thanks! This was one of the remaining few pieces I never got around to >> fixing two years ago (the other place is copies.py). Now that I see >> your patch, I wonder why I didn't see that simple solution. >> >>> >>> diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py >>> --- a/mercurial/cmdutil.py >>> +++ b/mercurial/cmdutil.py >>> @@ -2977,10 +2977,12 @@ def revert(ui, repo, ctx, parents, *pats >>> modadded = set() >>> >>> # split between files known in target manifest and the others >> >> >> Does it make sense to leave this comment here? I'm not sure what its >> scope is meant to be even before this patch. > > > Me either, which is why I left it... We can probably remove it though. Want > me to resend, or you want to just drop this line during queueing, since I > don't think I'm changing the bits below. I'll drop it in flight. I'm also changing the O(repo) to be O(manifest) in the subject line. Thanks! > >>> -smf = set(mf) >>> >>> # determine the exact nature of the deleted changesets >>> -deladded = _deleted - smf >>> +deladded = set(_deleted) >>> +for path in _deleted: >>> +if path in mf: >>> +deladded.remove(path) >> >> >> Would it be better to not add it to the set in the first place? Like so: >> >>> -deladded = _deleted - smf >>> +deladded = set() >>> +for path in _deleted: >>> +if path not in mf: >>> +deladded.add(path) >> >> >> I can't see it making any difference in practice, so pick whichever >> seems simpler (i.e. feel free to ignore). > > > The set constructor does a smart allocation when doing a set copy, so we > avoid internal set-growth reallocations by starting big and going small. My > assumption is that this stuff is going to be pretty small regardless, so it > probably doesn't matter really. > > See line 660 > http://svn.python.org/view/python/trunk/Objects/setobject.c?view=markup > >>> deleted = _deleted - deladded >>> >>> # We need to account for the state of the file in the dirstate, >>> @@ -3024,7 +3026,10 @@ def revert(ui, repo, ctx, parents, *pats >>> # in case of merge, files that are actually added can be >>> reported as >>> # modified, we need to post process the result >>> if p2 != nullid: >>> -mergeadd = dsmodified - smf >>> +mergeadd = set(dsmodified) >>> +for path in dsmodified: >>> +if path in mf: >>> +mergeadd.remove(path) >> >> >> Same here. >> >>> dsadded |= mergeadd >>> dsmodified -= mergeadd >>> >>> ___ >>> Mercurial-devel mailing list >>> Mercurial-devel@mercurial-scm.org >>> >>> https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=nuarHzhP1wi1T9iURRCj1A&m=uZTCRgAVm8p5eoX9JtG7NXyHYRWAtyDglOxpwq5zkdk&s=PC6LNzvhXaSgoieJBJpN2q_sfLvi5N6IBK0Iv9VZcCg&e= ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] revert: remove set(mf) because it's O(repo)
On 3/2/17 9:57 AM, Martin von Zweigbergk wrote: On Wed, Mar 1, 2017 at 8:01 PM, Durham Goode wrote: # HG changeset patch # User Durham Goode # Date 1488426665 28800 # Wed Mar 01 19:51:05 2017 -0800 # Node ID a8458fe51a9d155f1daeaffdcf503e674d4d4588 # Parent b787c41767339158927232ec7a9092196e887453 revert: remove set(mf) because it's O(repo) O(repo) sounds misleading to me, because it sounds like number of revisions matter too. I'd change it to O(manifest) The revert code had a 'set(manifest)' line in it, which has a runtime equivalent to the size of the working copy. With alternative manifest implementations, like treemanifest, this can be extra expensive. Let's rewrite it to be O(changes) instead of O(working copy size). Thanks! This was one of the remaining few pieces I never got around to fixing two years ago (the other place is copies.py). Now that I see your patch, I wonder why I didn't see that simple solution. diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py --- a/mercurial/cmdutil.py +++ b/mercurial/cmdutil.py @@ -2977,10 +2977,12 @@ def revert(ui, repo, ctx, parents, *pats modadded = set() # split between files known in target manifest and the others Does it make sense to leave this comment here? I'm not sure what its scope is meant to be even before this patch. Me either, which is why I left it... We can probably remove it though. Want me to resend, or you want to just drop this line during queueing, since I don't think I'm changing the bits below. -smf = set(mf) # determine the exact nature of the deleted changesets -deladded = _deleted - smf +deladded = set(_deleted) +for path in _deleted: +if path in mf: +deladded.remove(path) Would it be better to not add it to the set in the first place? Like so: -deladded = _deleted - smf +deladded = set() +for path in _deleted: +if path not in mf: +deladded.add(path) I can't see it making any difference in practice, so pick whichever seems simpler (i.e. feel free to ignore). The set constructor does a smart allocation when doing a set copy, so we avoid internal set-growth reallocations by starting big and going small. My assumption is that this stuff is going to be pretty small regardless, so it probably doesn't matter really. See line 660 http://svn.python.org/view/python/trunk/Objects/setobject.c?view=markup deleted = _deleted - deladded # We need to account for the state of the file in the dirstate, @@ -3024,7 +3026,10 @@ def revert(ui, repo, ctx, parents, *pats # in case of merge, files that are actually added can be reported as # modified, we need to post process the result if p2 != nullid: -mergeadd = dsmodified - smf +mergeadd = set(dsmodified) +for path in dsmodified: +if path in mf: +mergeadd.remove(path) Same here. dsadded |= mergeadd dsmodified -= mergeadd ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=nuarHzhP1wi1T9iURRCj1A&m=uZTCRgAVm8p5eoX9JtG7NXyHYRWAtyDglOxpwq5zkdk&s=PC6LNzvhXaSgoieJBJpN2q_sfLvi5N6IBK0Iv9VZcCg&e= ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] util: add allowhardlinks module variable
On 3/2/17 10:12 AM, Durham Goode wrote: # HG changeset patch # User Durham Goode # Date 1488476378 28800 # Thu Mar 02 09:39:38 2017 -0800 # Node ID d52b0a72423f65b4f774d5b646474cdafd4a5cc0 # Parent a8458fe51a9d155f1daeaffdcf503e674d4d4588 util: add allowhardlinks module variable To enable extensions to enable hardlinks for certain environments, let's move the 'if False' to be an 'if allowhardlinks' and let extensions modify the allowhardlinks variable. Tests on linux ext4 pass with it set to True and to False. Sorry for the immediate V2. I had uncommited working copy changes that I had forgotten to amend, so I had to send V2. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH V2] util: add allowhardlinks module variable
# HG changeset patch # User Durham Goode # Date 1488478360 28800 # Thu Mar 02 10:12:40 2017 -0800 # Node ID 32c17aa5fc546a112b355e734fb71b740172cf09 # Parent a8458fe51a9d155f1daeaffdcf503e674d4d4588 util: add allowhardlinks module variable To enable extensions to enable hardlinks for certain environments, let's move the 'if False' to be an 'if allowhardlinks' and let extensions modify the allowhardlinks variable. Tests on linux ext4 pass with it set to True and to False. diff --git a/mercurial/util.py b/mercurial/util.py --- a/mercurial/util.py +++ b/mercurial/util.py @@ -1056,6 +1056,11 @@ def checksignature(func): return check +# Hardlinks are problematic on CIFS, do not allow hardlinks +# until we find a way to work around it cleanly (issue4546). +# This is a variable so extensions can opt-in to using them. +allowhardlinks = False + def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False): '''copy a file, preserving mode and optionally other stat info like atime/mtime @@ -1072,9 +1077,7 @@ def copyfile(src, dest, hardlink=False, if checkambig: oldstat = checkambig and filestat(dest) unlink(dest) -# hardlinks are problematic on CIFS, quietly ignore this flag -# until we find a way to work around it cleanly (issue4546) -if False and hardlink: +if allowhardlinks and hardlink: try: oslink(src, dest) return ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH] util: add allowhardlinks module variable
# HG changeset patch # User Durham Goode # Date 1488476378 28800 # Thu Mar 02 09:39:38 2017 -0800 # Node ID d52b0a72423f65b4f774d5b646474cdafd4a5cc0 # Parent a8458fe51a9d155f1daeaffdcf503e674d4d4588 util: add allowhardlinks module variable To enable extensions to enable hardlinks for certain environments, let's move the 'if False' to be an 'if allowhardlinks' and let extensions modify the allowhardlinks variable. Tests on linux ext4 pass with it set to True and to False. diff --git a/mercurial/util.py b/mercurial/util.py --- a/mercurial/util.py +++ b/mercurial/util.py @@ -1056,6 +1056,10 @@ def checksignature(func): return check +# hardlinks are problematic on CIFS, quietly ignore this flag +# until we find a way to work around it cleanly (issue4546) +allowhardlinks = False + def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False): '''copy a file, preserving mode and optionally other stat info like atime/mtime @@ -1072,9 +1076,7 @@ def copyfile(src, dest, hardlink=False, if checkambig: oldstat = checkambig and filestat(dest) unlink(dest) -# hardlinks are problematic on CIFS, quietly ignore this flag -# until we find a way to work around it cleanly (issue4546) -if False and hardlink: +if allowhardlinks and hardlink: try: oslink(src, dest) return ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH V2] similar: compare between actual file contents for exact identity
# HG changeset patch
# User FUJIWARA Katsunori
# Date 1488477426 -32400
# Fri Mar 03 02:57:06 2017 +0900
# Node ID d7d47f54019fa900968245163e67ca6f02378995
# Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
similar: compare between actual file contents for exact identity
Before this patch, similarity detection logic (for addremove and
automv) depends entirely on SHA-1 digesting. But this causes incorrect
rename detection, if:
- removing file A and adding file B occur at same committing, and
- SHA-1 hash values of file A and B are same
This may prevent security experts from managing sample files for
SHAttered issue in Mercurial repository, for example.
https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html
https://shattered.it/
Hash collision itself isn't so serious for core repository
functionality of Mercurial, described by mpm as below, though.
https://www.mercurial-scm.org/wiki/mpm/SHA1
This patch compares between actual file contents after hash comparison
for exact identity.
Even after this patch, SHA-1 is still used, because it is reasonable
enough to quickly detect existence of "(almost) same" file.
- replacing SHA-1 causes decreasing performance, and
- replacement of it has ambiguity, yet
Getting content of removed file (= rfctx.data()) at each exact
comparison should be cheap enough, even though getting content of
added one costs much.
=== == =
filefctx data() reads from
=== == =
removed filectxin-memory revlog data
added workingfilectx storage
=== == =
diff --git a/mercurial/similar.py b/mercurial/similar.py
--- a/mercurial/similar.py
+++ b/mercurial/similar.py
@@ -35,9 +35,13 @@ def _findexactmatches(repo, added, remov
for i, fctx in enumerate(added):
repo.ui.progress(_('searching for exact renames'), i + len(removed),
total=numfiles, unit=_('files'))
-h = hashlib.sha1(fctx.data()).digest()
+adata = fctx.data()
+h = hashlib.sha1(adata).digest()
if h in hashes:
-yield (hashes[h], fctx)
+rfctx = hashes[h]
+# compare between actual file contents for exact identity
+if adata == rfctx.data():
+yield (rfctx, fctx)
# Done
repo.ui.progress(_('searching for exact renames'), None)
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] revert: remove set(mf) because it's O(repo)
On Wed, Mar 1, 2017 at 8:01 PM, Durham Goode wrote: > # HG changeset patch > # User Durham Goode > # Date 1488426665 28800 > # Wed Mar 01 19:51:05 2017 -0800 > # Node ID a8458fe51a9d155f1daeaffdcf503e674d4d4588 > # Parent b787c41767339158927232ec7a9092196e887453 > revert: remove set(mf) because it's O(repo) O(repo) sounds misleading to me, because it sounds like number of revisions matter too. I'd change it to O(manifest) > > The revert code had a 'set(manifest)' line in it, which has a runtime > equivalent > to the size of the working copy. With alternative manifest implementations, > like > treemanifest, this can be extra expensive. Let's rewrite it to be O(changes) > instead of O(working copy size). Thanks! This was one of the remaining few pieces I never got around to fixing two years ago (the other place is copies.py). Now that I see your patch, I wonder why I didn't see that simple solution. > > diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py > --- a/mercurial/cmdutil.py > +++ b/mercurial/cmdutil.py > @@ -2977,10 +2977,12 @@ def revert(ui, repo, ctx, parents, *pats > modadded = set() > > # split between files known in target manifest and the others Does it make sense to leave this comment here? I'm not sure what its scope is meant to be even before this patch. > -smf = set(mf) > > # determine the exact nature of the deleted changesets > -deladded = _deleted - smf > +deladded = set(_deleted) > +for path in _deleted: > +if path in mf: > +deladded.remove(path) Would it be better to not add it to the set in the first place? Like so: > -deladded = _deleted - smf > +deladded = set() > +for path in _deleted: > +if path not in mf: > +deladded.add(path) I can't see it making any difference in practice, so pick whichever seems simpler (i.e. feel free to ignore). > deleted = _deleted - deladded > > # We need to account for the state of the file in the dirstate, > @@ -3024,7 +3026,10 @@ def revert(ui, repo, ctx, parents, *pats > # in case of merge, files that are actually added can be reported as > # modified, we need to post process the result > if p2 != nullid: > -mergeadd = dsmodified - smf > +mergeadd = set(dsmodified) > +for path in dsmodified: > +if path in mf: > +mergeadd.remove(path) Same here. > dsadded |= mergeadd > dsmodified -= mergeadd > > ___ > Mercurial-devel mailing list > Mercurial-devel@mercurial-scm.org > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: Ideas and Mentors for GSoC 2017.
We are accepted as a sub-org under Python Software Foundation and we are listed on their wiki page http://python-gsoc.org/#ideas. Now we need mentors and new project ideas. I guess we already have 2-3 new contributors who are interested in doing GSoC with Mercurial. (Sorry for using Comic Sans in the previous mail) On Fri, Feb 17, 2017 at 12:55 AM, Pulkit Goyal <7895pul...@gmail.com> wrote: > Hi everyone, > > Just like last year, this year also we are going to take part in GSoC > under Python Software Foundation. Though we are still not listed at their > wiki page because the page is frozen until Google confirms participation of > Python. I am handling GSoC work this year of Mercurial ;). > > We need ideas and mentors to participate. I have created a ideas page > where I have enlisted leftover ideas from previous year. > https://www.mercurial-scm.org/wiki/SummerOfCode/Ideas2017 > > Those who want to be mentors can reply to this mail, we want at least > three mentors like last year. > > Let's cook some new ideas. You can either discuss the idea here before > adding there or can just add that to wiki page (however first one is > preferred). > > Regards > Pulkit Goyal > ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] transaction: enable hardlink backups for non-windows systems
On 3/2/17 8:09 AM, Ryan McElroy wrote: On 3/2/17 7:46 AM, Yuya Nishihara wrote: On Wed, 1 Mar 2017 17:55:25 -0800, Jun Wu wrote: According to https://urldefense.proofpoint.com/v2/url?u=https-3A__bz.mercurial-2Dscm.org_show-5Fbug.cgi-3Fid-3D4546-3A&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=vjW-XcGbq_x7AbgPtmJ8lHDdDLYljlyx2xvUxSdf4IE&s=6scd8qQkJR-0MXuZo-p47gjTcBa4U3m-ZywPbWfxOKI&e= Testing with mercurial 3.2 and 3.3 to a Linux samba server worked without issue in the few tests I've done. This suggests the most likely cause is an NTFS/Windows based file server? So I think at least we can have a whitelist (instead of a blacklist) that allows Linux to use real hardlinks. Just ignore OS X or Windows for now. She said a Linux samba server had no problem, not a Linux client with Windows server. Sounds like just changing this is a non-starter then. How about using a config option, so people like us can get the perf benefits in places where we are not concerned about the possible downsides (because we prevent clones onto NFS/CIFS shares, for example)? Would anyone object to turning this into an option that defaults to off? Might be hard to get a config option down to this level, since there's no ui object around here, but we could probably move the 'False' value to a module level variable so extensions could at least set it to True. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
mercurial@31129: new changeset
New changeset in mercurial: https://www.mercurial-scm.org/repo/hg/rev/71f692f1f678 changeset: 31129:71f692f1f678 bookmark:@ tag: tip user:Pierre-Yves David date:Wed Mar 01 20:22:04 2017 +0100 summary: color: update the help table -- Repository URL: https://www.mercurial-scm.org/repo/hg ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 1 of 5] transaction: introduce scope
FWIW, I thought this approach was elegant and it would have benefits
even on systems without good hardlink support.
On 3/1/17 8:48 PM, Jun Wu wrote:
I'm going to drop this series from patchwork. The new dirstate patches is a
better fix.
Excerpts from Jun Wu's message of 2017-02-27 09:35:25 -0800:
# HG changeset patch
# User Jun Wu
# Date 1488185788 28800
# Mon Feb 27 00:56:28 2017 -0800
# Node ID 5dac612ec9a87553fcd329100846bcb01bae9d80
# Parent b4cb86ab4c719eb615a4308eafd8b1386a511eeb
# Available At
https://urldefense.proofpoint.com/v2/url?u=https-3A__bitbucket.org_quark-2Dzju_hg-2Ddraft&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=PUI_VSItx4f9h9UN0MwUI5ZWzNcTzVxIX5kISKO7ndY&s=dpXH9Ao6YM_Ho7SmRBoNsXmlruqUd2Lo6JlEEetKfRI&e=
# hg pull
https://urldefense.proofpoint.com/v2/url?u=https-3A__bitbucket.org_quark-2Dzju_hg-2Ddraft&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=PUI_VSItx4f9h9UN0MwUI5ZWzNcTzVxIX5kISKO7ndY&s=dpXH9Ao6YM_Ho7SmRBoNsXmlruqUd2Lo6JlEEetKfRI&e=
-r 5dac612ec9a8
transaction: introduce scope
Previously, transaction tries to cover everything: bookmarks, dirstate,
phaseroots, and branch. It backs them up unconditionally.
That could be inefficient if we still back up the dirstate in a giant repo,
where the transaction is only intended to change bookmarks.
This patch introduces "scope" to transactions, which is a set of things that
the transaction should cover. For bookmark-only update, it could avoid
specifying the "dirstate" scope to be more efficient.
diff --git a/mercurial/transaction.py b/mercurial/transaction.py
--- a/mercurial/transaction.py
+++ b/mercurial/transaction.py
@@ -102,5 +102,6 @@ def _playback(journal, report, opener, v
class transaction(object):
def __init__(self, report, opener, vfsmap, journalname, undoname=None,
- after=None, createmode=None, validator=None, releasefn=None):
+ after=None, createmode=None, validator=None, releasefn=None,
+ scope=None):
"""Begin a new transaction
@@ -111,4 +112,5 @@ class transaction(object):
* `createmode`: the mode of the journal file that will be created
* `releasefn`: called after releasing (with transaction and result)
+* `scope`: a set-like, nested transaction's scope must be a subset
"""
self.count = 1
@@ -171,4 +173,7 @@ class transaction(object):
self._abortcallback = {}
+# a set indicating what's covered
+self._scope = scope or frozenset()
+
def __del__(self):
if self.journal:
@@ -342,5 +347,10 @@ class transaction(object):
@active
-def nest(self):
+def nest(self, scope=None):
+scope = scope or frozenset()
+if not scope.issubset(self._scope):
+raise error.ProgrammingError(
+'nested transaction has a superset scope (%s > %s)'
+% (scope, self._scope))
self.count += 1
self.usages += 1
@@ -555,4 +565,7 @@ class transaction(object):
self.releasefn(self, False) # notify failure of transaction
+def scope(self):
+return self._scope
+
def rollback(opener, vfsmap, file, report):
"""Rolls back the transaction contained in the given file
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=PUI_VSItx4f9h9UN0MwUI5ZWzNcTzVxIX5kISKO7ndY&s=-x5cILk1FvncdPrM6RVANdj0aCMXVZ76OahlF5Gh9pM&e=
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 5 of 5] localrepo: deprecated 'repo.opener' (API)
This series looks good to me. I'd suggest changing the past-tense
"deprecated" to the present-tense "deprecate" in the titles in-flight.
On 3/1/17 6:58 PM, Pierre-Yves David wrote:
# HG changeset patch
# User Pierre-Yves David
# Date 1470398170 -7200
# Fri Aug 05 13:56:10 2016 +0200
# Node ID aacf8b01b81483b4815a974f17d0ff5d214c4d3d
# Parent 437a39859c33901ea29cd22341d93be752e4acc0
# EXP-Topic vfs.cleanup
localrepo: deprecated 'repo.opener' (API)
The "new" 'repo.vfs' attribute have been around for almost 5 years. I think we
can deprecate the old form now ;-)
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -266,7 +266,6 @@ class localrepository(object):
self.nofsauditor = pathutil.pathauditor(self.root, self._checknested,
realfs=False)
self.vfs = scmutil.vfs(self.path)
-self.opener = self.vfs
self.baseui = baseui
self.ui = baseui.copy()
self.ui.copy = baseui.copy # prevent copying repo configuration
@@ -379,6 +378,11 @@ class localrepository(object):
self.ui.deprecwarn("use 'repo.wvfs' instead of 'repo.wopener'", '4.2')
return self.wvfs
+@property
+def opener(self):
+self.ui.deprecwarn("use 'repo.vfs' instead of 'repo.opener'", '4.2')
+return self.vfs
+
def close(self):
self._writecaches()
diff --git a/mercurial/statichttprepo.py b/mercurial/statichttprepo.py
--- a/mercurial/statichttprepo.py
+++ b/mercurial/statichttprepo.py
@@ -123,7 +123,6 @@ class statichttprepository(localrepo.loc
vfsclass = build_opener(ui, authinfo)
self.vfs = vfsclass(self.path)
-self.opener = self.vfs
self._phasedefaults = []
self.names = namespaces.namespaces()
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=pGe4-AWiOAPibuWIRfURjzzHLk5Xnfg7vTq3aCWDwyM&s=W8xRG1nGSGlZascbfPeFdqDPaF3a3XY5lj-hewgtrNw&e=
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 1 of 5] localrepo: add some comment about role of various vfs object
I'm a big +1 on adding documentation -- thanks for doing this!
On 3/1/17 6:58 PM, Pierre-Yves David wrote:
# HG changeset patch
# User Pierre-Yves David
# Date 1470397745 -7200
# Fri Aug 05 13:49:05 2016 +0200
# Node ID b34202587b00d949dd77ff97399e50d6340779a6
# Parent 3f8f53190d6afed0aca6c6527236edad28ce785c
# EXP-Topic vfs.cleanup
localrepo: add some comment about role of various vfs object
This should make things clearer for most people.
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -253,7 +253,12 @@ class localrepository(object):
def __init__(self, baseui, path, create=False):
self.requirements = set()
+# vfs to access the working copy
self.wvfs = scmutil.vfs(path, expandpath=True, realpath=True)
+# vfs to access the content of the repository
This comment doesn't help me a lot -- it requires that I understand that
the "repository" means "things under the .hg directory", which is
non-obvious to newcomers to the code. I think it would be more clear to say:
# vfs to access repo files in .hg directory (excluding .hg/store)
+self.vfs = None
+# vfs to access the store part of the repository
Consider, perhaps:
# vfs to access repo history under .hg/store
+self.svfs = None
self.wopener = self.wvfs
self.root = self.wvfs.base
self.path = self.wvfs.join(".hg")
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=0Wa7QvUc5RMmEg0g8AsgL0S7_VRXgwF_-YNIWt3wfP8&s=TNGKczgSgGUR0zek-5nxBhh7u1P8sfbYWw5-do7ANyU&e=
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 2 of 2] share: drop 'relshared' requirement as well
This series looks good to me.
On 3/2/17 7:35 AM, Yuya Nishihara wrote:
# HG changeset patch
# User Yuya Nishihara
# Date 1488467511 -32400
# Fri Mar 03 00:11:51 2017 +0900
# Node ID 271e966c8d27fbc904dab4c641fe265825384dd3
# Parent 8cdc39b46f210a702051b3a6d6f03e2112b41aba
share: drop 'relshared' requirement as well
diff --git a/hgext/share.py b/hgext/share.py
--- a/hgext/share.py
+++ b/hgext/share.py
@@ -117,6 +117,7 @@ def unshare(ui, repo):
util.rename(sharefile, sharefile + '.old')
repo.requirements.discard('shared')
+repo.requirements.discard('relshared')
repo._writerequirements()
finally:
destlock and destlock.release()
diff --git a/tests/test-share.t b/tests/test-share.t
--- a/tests/test-share.t
+++ b/tests/test-share.t
@@ -393,6 +393,21 @@ renames and changes of PWD
[255]
$ hg -R thatdir/rel root
$TESTTMP/thatdir/rel
+
+test unshare relshared repo
+
+ $ cd thatdir/rel
+ $ hg unshare
+ $ test -d .hg/store
+ $ test -f .hg/sharedpath
+ [1]
+ $ grep shared .hg/requires
+ [1]
+ $ hg unshare
+ abort: this is not a shared repo
+ [255]
+ $ cd ../..
+
$ rm -r thatdir
Explicitly kill daemons to let the test exit on Windows
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=iTIb_qgLjcw2hyg08rbDA97Qwo7_RtmmA858ptHNeg8&s=9dasQD1Osl4smdqc-hq8G2LYzwd_L52M5VoJ_Kih1UY&e=
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] transaction: enable hardlink backups for non-windows systems
On 3/2/17 7:46 AM, Yuya Nishihara wrote: On Wed, 1 Mar 2017 17:55:25 -0800, Jun Wu wrote: According to https://urldefense.proofpoint.com/v2/url?u=https-3A__bz.mercurial-2Dscm.org_show-5Fbug.cgi-3Fid-3D4546-3A&d=DwIGaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=Jw8rundaE7TbmqBYd1txIQ&m=vjW-XcGbq_x7AbgPtmJ8lHDdDLYljlyx2xvUxSdf4IE&s=6scd8qQkJR-0MXuZo-p47gjTcBa4U3m-ZywPbWfxOKI&e= Testing with mercurial 3.2 and 3.3 to a Linux samba server worked without issue in the few tests I've done. This suggests the most likely cause is an NTFS/Windows based file server? So I think at least we can have a whitelist (instead of a blacklist) that allows Linux to use real hardlinks. Just ignore OS X or Windows for now. She said a Linux samba server had no problem, not a Linux client with Windows server. Sounds like just changing this is a non-starter then. How about using a config option, so people like us can get the perf benefits in places where we are not concerned about the possible downsides (because we prevent clones onto NFS/CIFS shares, for example)? Would anyone object to turning this into an option that defaults to off? ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] config: add extension point for extracting all included files
On Wed, 1 Mar 2017 15:51:44 +0100, Mathias De Maré wrote: > On 11-02-17 10:52, Yuya Nishihara wrote: > > On Tue, 7 Feb 2017 15:31:02 +, De Mare, Mathias (Nokia - BE) wrote: > On Feb 3, 2017, at 09:56, Mathias De Maré > >>> wrote: > # HG changeset patch > # User Mathias De Mare # Date > >>> 1486132049 > -3600 > # Fri Feb 03 15:27:29 2017 +0100 > # Node ID ea2e95febfeff39377c878fa05cc853832bc3b2a > # Parent eb78ec9e97b70310e2944f72c29463bedfc21442 > config: add extension point for extracting all included files > >>> Hmm... for those of us who haven't been paying close attention, can you > >>> say > >>> more about why this is desirable? Generally we don't add an extension > >>> point > >>> unless at least an in-tree extension uses it. (It's possible there are > >>> historical > >>> exceptions to this.) > >> Pierre-Yves and myself spent some time working on an extension that would > >> allow users to share their configuration with a server and allow a server > >> to send configuration suggestions to a client. The work so far is > >> available at https://bitbucket.org/Mathiasdm/hg-configexpress/overview > >> One of the possible suggestions to clients would be to add specific > >> includes, and this extension point allows checking if these includes are > >> present. > > Can it be done by using/extending config._source dict? > > > > I don't think hooking config method is a good idea because config object > > may be used to parse template maps, .hgsub, etc. > config._source can't be used as it is (as it stores the absolute paths, > while config includes could be include with a relative path). > > I guess one possible alternative would be to create a config._relsource > containing the relative paths, but that would require signature changes > to some of the config methods. > Another option would be for hg-configexpress to implement parsing of > configfiles itself (only the 'includes' part), but that seems a bit > redundant. > Finally, it would be possible to add 2 additional keyword arguments to > the 'include' function that's called inside config.config.parse, so it > would be something like: include(inc, remap=remap, sections=sections, > base=base, expanded=expanded) Internal API changes should be okay. I don't know which would be better, but both config._relsource and include() sound good to me. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] transaction: enable hardlink backups for non-windows systems
On Wed, 1 Mar 2017 17:55:25 -0800, Jun Wu wrote: > According to https://bz.mercurial-scm.org/show_bug.cgi?id=4546: > > Testing with mercurial 3.2 and 3.3 to a Linux samba server worked without > issue in the few tests I've done. This suggests the most likely cause is > an NTFS/Windows based file server? > > So I think at least we can have a whitelist (instead of a blacklist) that > allows Linux to use real hardlinks. Just ignore OS X or Windows for now. She said a Linux samba server had no problem, not a Linux client with Windows server. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 3 of 4 V2] chgcache: implement the background preloading thread
On Wed, 1 Mar 2017 08:38:03 -0800, Jun Wu wrote:
> Excerpts from Yuya Nishihara's message of 2017-03-01 21:56:08 +0900:
> > On Wed, 22 Feb 2017 18:16:10 -0800, Jun Wu wrote:
> > > # HG changeset patch
> > > # User Jun Wu
> > > # Date 1487813979 28800
> > > # Wed Feb 22 17:39:39 2017 -0800
> > > # Node ID 28571825744fb4f4b424385f55afa9484532ef43
> > > # Parent 630457e88fa0485bce7822345ea640b5cdcb9b8e
> > > # Available At https://bitbucket.org/quark-zju/hg-draft
> > > # hg pull https://bitbucket.org/quark-zju/hg-draft -r
> > > 28571825744f
> > > chgcache: implement the background preloading thread
> > >
> > > The background preloading thread is meant to used by chg master, and it
> > > sets
> > > up the IPC channel.
> >
> > > +def _preloadloop(self, interval=0.5):
> > > +while not self._shouldexit.is_set():
> > > +try:
> > > +atimestr, path = self._ipc.recv().split(' ', 1)
> > > +atime = float(atimestr)
> > > +except Exception: # ignore errors
> > > +pass
> > > +else:
> > > +if path in _repocaches and _repocaches[path][0] >= atime:
> > > +# this repocache is up-to-date, no need to update
> > > +continue
> > > +now = time.time()
> > > +# TODO update repocache properly
> >
> > Another concern just came up. Do you know what will happen if the process
> > is fork()ed while the preloader thread is actively loading cache?
> >
> > Only the main thread (which called fork()) would be copied, so the preloader
> > would stop. That should be okay, but the other resources such as files and
> > mutexes (if any) would be copied.
>
> Right. It's possible to leak some fds, where I think it's probably fine. I
> guess Python will deal with mutexes created by Python code.
I hope so, but googling "fork thread" or "fork thread python" made me feel
bad. I haven't read them carefully, but they seemed to say "don't do it."
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 2 of 2] share: drop 'relshared' requirement as well
# HG changeset patch
# User Yuya Nishihara
# Date 1488467511 -32400
# Fri Mar 03 00:11:51 2017 +0900
# Node ID 271e966c8d27fbc904dab4c641fe265825384dd3
# Parent 8cdc39b46f210a702051b3a6d6f03e2112b41aba
share: drop 'relshared' requirement as well
diff --git a/hgext/share.py b/hgext/share.py
--- a/hgext/share.py
+++ b/hgext/share.py
@@ -117,6 +117,7 @@ def unshare(ui, repo):
util.rename(sharefile, sharefile + '.old')
repo.requirements.discard('shared')
+repo.requirements.discard('relshared')
repo._writerequirements()
finally:
destlock and destlock.release()
diff --git a/tests/test-share.t b/tests/test-share.t
--- a/tests/test-share.t
+++ b/tests/test-share.t
@@ -393,6 +393,21 @@ renames and changes of PWD
[255]
$ hg -R thatdir/rel root
$TESTTMP/thatdir/rel
+
+test unshare relshared repo
+
+ $ cd thatdir/rel
+ $ hg unshare
+ $ test -d .hg/store
+ $ test -f .hg/sharedpath
+ [1]
+ $ grep shared .hg/requires
+ [1]
+ $ hg unshare
+ abort: this is not a shared repo
+ [255]
+ $ cd ../..
+
$ rm -r thatdir
Explicitly kill daemons to let the test exit on Windows
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 1 of 2] share: fix typo to drop 'shared' requirement on unshare
# HG changeset patch
# User Yuya Nishihara
# Date 1488467478 -32400
# Fri Mar 03 00:11:18 2017 +0900
# Node ID 8cdc39b46f210a702051b3a6d6f03e2112b41aba
# Parent 23080c03a604737b7d32cb789b7526bbec125e08
share: fix typo to drop 'shared' requirement on unshare
This must be a typo and it seems correct to drop the requirement since the
repo is no longer a shared repository.
diff --git a/hgext/share.py b/hgext/share.py
--- a/hgext/share.py
+++ b/hgext/share.py
@@ -116,7 +116,7 @@ def unshare(ui, repo):
sharefile = repo.join('sharedpath')
util.rename(sharefile, sharefile + '.old')
-repo.requirements.discard('sharedpath')
+repo.requirements.discard('shared')
repo._writerequirements()
finally:
destlock and destlock.release()
diff --git a/tests/test-share.t b/tests/test-share.t
--- a/tests/test-share.t
+++ b/tests/test-share.t
@@ -114,6 +114,8 @@ test unshare command
$ test -d .hg/store
$ test -f .hg/sharedpath
[1]
+ $ grep shared .hg/requires
+ [1]
$ hg unshare
abort: this is not a shared repo
[255]
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 8 of 8 V2] hgweb: add a "followlines" link to hover-box in annotate
On Wed, 1 Mar 2017 17:36:48 +0100, Denis Laxalde wrote: > Yuya Nishihara a écrit : > > On Tue, 28 Feb 2017 16:00:44 -0800, Gregory Szorc wrote: > >> Also, the page it links to showing the revisions with links to a line range > >> filtered diff is good but not great. I imagine developers will complain > >> that it requires too many clicks to go to what they want (the actual line > >> content). IMO it would be *really* useful if there were a single page > >> showing multiple changesets and line range data for either/and a) raw line > >> content b) annotate view c) diff view. That way, developers could scroll > >> and see all versions of that line range. If you were looking at say the > >> history of a function, this would be *extremely* useful compared to loading > >> N pages. This improvement could be done as a follow-up and doesn't need to > >> block this series. > > > > When the page gets more dynamically rendered, will we still need the mdiff > > filtering function? I guess filtering will be handled by js, and we'll need > > to label all chunks instead of filtering them on server side. > > > > I think the mdiff and patch.diff() APIs are the most controversial part in > > this series. If we can go without them, that would be nice. > > > > Maybe filtering can occur in webutil.diffs only, but I seems to me that > this would imply parsing diff headers (@@ +start,count -start,count @@). > I think that filtering in mdiff/patch.diff is cleaner but I agree that > the API is not nice. Also, working on mdiff/patch.diff side will > probably be necessary when implementing the command-line version of this > feature. One idea is to attach a context to each chunk so we can filter them later or embed in attribute. For example, patch.diff() yields text patch.diffchunks() yields (text, some_line_information) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] share: add --relative flag to store a relative path to the source
On Tue, 14 Feb 2017 16:53:47 +0100, Dan Villiom Podlaski Christiansen wrote:
> # HG changeset patch
> # User Dan Villiom Podlaski Christiansen
> # Date 1486991124 -3600
> # Mon Feb 13 14:05:24 2017 +0100
> # Node ID e6d4c5b2de24b6151d709696891eed665000d363
> # Parent a0e3d808690d57d1c9dff840e0b8ee099526397b
> share: add --relative flag to store a relative path to the source
> --- a/mercurial/hg.py
> +++ b/mercurial/hg.py
> @@ -195,7 +195,8 @@ def defaultdest(source):
> return ''
> return os.path.basename(os.path.normpath(path))
>
> -def share(ui, source, dest=None, update=True, bookmarks=True,
> defaultpath=None):
> +def share(ui, source, dest=None, update=True, bookmarks=True,
> defaultpath=None,
> + relative=False):
> '''create a shared repository'''
>
> if not islocal(source):
> @@ -235,7 +236,16 @@ def share(ui, source, dest=None, update=
> if inst.errno != errno.ENOENT:
> raise
>
> -requirements += 'shared\n'
> +if relative:
> +try:
> +sharedpath = os.path.relpath(sharedpath, destvfs.base)
> +requirements += 'relshared\n'
> +except IOError as e:
> +raise error.Abort(_('cannot calculate relative path'),
> + hint=e.message)
BaseException.message is deprecated. Replaced with str(e) in flight.
Do you know when IOError would be raised? I suspect it might be OSError.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] share: add --relative flag to store a relative path to the source
On Tue, 14 Feb 2017 16:53:47 +0100, Dan Villiom Podlaski Christiansen wrote: > # HG changeset patch > # User Dan Villiom Podlaski Christiansen > # Date 1486991124 -3600 > # Mon Feb 13 14:05:24 2017 +0100 > # Node ID e6d4c5b2de24b6151d709696891eed665000d363 > # Parent a0e3d808690d57d1c9dff840e0b8ee099526397b > share: add --relative flag to store a relative path to the source Sorry for really late review. This looks good and I think is useful. Also, there were no negative comments for weeks, so queued, thanks. > Currently, Mercurial interprets paths in `.hg/shared` relative to > $PWD. I suspect this is very much unintentional, and you have to > manually edit `.hg/shared` in order to trigger this behaviour. > > However, on the off chance that someone might rely on it, I added a > new capability called 'relshared'. In addition, this makes earlier > versions of Mercurial fail with a graceful error. Yep, new capability would be needed to kick out old Mercurial versions. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 1 of 4] minirst: detect bullet lists using asterisks
On Wed, 15 Feb 2017 17:58:58 -0800, Gregory Szorc wrote: > # HG changeset patch > # User Gregory Szorc > # Date 1487205737 28800 > # Wed Feb 15 16:42:17 2017 -0800 > # Node ID dd90d5f7dc1908d9b69e6a4b8165a73757d1c84b > # Parent afaf3c2b129c8940387fd9928ae4fdc28259d13c > minirst: detect bullet lists using asterisks The first 3 patches look nice even if we wouldn't take the 4th RFC path. Queued these, thanks. > +* This is the first bullet > +* This is the second bullet It > + has 2 lines > +* This is the third bullet > -- > > html format: > @@ -276,6 +290,14 @@ Line blocks are also a form of list: >This is the first line. The line continues here. >This is the second line. > > + > +Bullet lists are also detected: > + > + > + This is the first bullet > + This is the second bullet It has 2 lines > + This is the third bullet > + This should be . Fixed in flight. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 2 of 4] minirst: dynamically compile admonitions regexp
On Wed, 15 Feb 2017 17:58:59 -0800, Gregory Szorc wrote: > # HG changeset patch > # User Gregory Szorc > # Date 1487188034 28800 > # Wed Feb 15 11:47:14 2017 -0800 > # Node ID 197ba3e5885366038d453b9b22fa2910a0792988 > # Parent dd90d5f7dc1908d9b69e6a4b8165a73757d1c84b > minirst: dynamically compile admonitions regexp > +_admonitions = set([ > +'admonition', > +'attention', > +'caution', > +'danger', > +'error', > +'hint', > +'important', > +'note', > +'tip', > +'warning', > +]) Maybe this could be combined with _admonitiontitles. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] color: update the help table
On Thu, 02 Mar 2017 13:57:22 +0100, Pierre-Yves David wrote: > # HG changeset patch > # User Pierre-Yves David > # Date 1488396124 -3600 > # Wed Mar 01 20:22:04 2017 +0100 > # Node ID f854d56365922dce4fdf9545a165d85ce331d084 > # Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa > # Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ > # hg pull > https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r f854d5636592 > # EXP-Topic color > color: update the help table Queued this, thanks. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH] color: update the help table
# HG changeset patch
# User Pierre-Yves David
# Date 1488396124 -3600
# Wed Mar 01 20:22:04 2017 +0100
# Node ID f854d56365922dce4fdf9545a165d85ce331d084
# Parent 0bb3089fe73527c64f1afc40b86ecb8dfe7fd7aa
# Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/
# hg pull
https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r f854d5636592
# EXP-Topic color
color: update the help table
We also need to reference the new topic in the great old help table.
diff --git a/mercurial/help.py b/mercurial/help.py
--- a/mercurial/help.py
+++ b/mercurial/help.py
@@ -208,6 +208,7 @@ def internalshelp(ui):
return ''.join(lines)
helptable = sorted([
+(['color'], _("Colorizing Outputs"), loaddoc('color')),
(["config", "hgrc"], _("Configuration Files"), loaddoc('config')),
(["dates"], _("Date Formats"), loaddoc('dates')),
(["patterns"], _("File Name Patterns"), loaddoc('patterns')),
diff --git a/tests/test-globalopts.t b/tests/test-globalopts.t
--- a/tests/test-globalopts.t
+++ b/tests/test-globalopts.t
@@ -340,6 +340,7 @@ Testing -h/--help:
additional help topics:
+ color Colorizing Outputs
configConfiguration Files
dates Date Formats
diffs Diff Formats
@@ -422,6 +423,7 @@ Testing -h/--help:
additional help topics:
+ color Colorizing Outputs
configConfiguration Files
dates Date Formats
diffs Diff Formats
diff --git a/tests/test-help.t b/tests/test-help.t
--- a/tests/test-help.t
+++ b/tests/test-help.t
@@ -102,6 +102,7 @@ Short help:
additional help topics:
+ color Colorizing Outputs
configConfiguration Files
dates Date Formats
diffs Diff Formats
@@ -178,6 +179,7 @@ Short help:
additional help topics:
+ color Colorizing Outputs
configConfiguration Files
dates Date Formats
diffs Diff Formats
@@ -825,6 +827,7 @@ Test that default list of commands omits
additional help topics:
+ color Colorizing Outputs
configConfiguration Files
dates Date Formats
diffs Diff Formats
@@ -1859,6 +1862,13 @@ Dish up an empty repo; serve it cold.
Topics
+
+ color
+
+
+ Colorizing Outputs
+
+
config
diff --git a/tests/test-hgweb-json.t b/tests/test-hgweb-json.t
--- a/tests/test-hgweb-json.t
+++ b/tests/test-hgweb-json.t
@@ -1549,6 +1549,10 @@ help/ shows help topics
],
"topics": [
{
+"summary": "Colorizing Outputs",
+"topic": "color"
+ },
+ {
"summary": "Configuration Files",
"topic": "config"
},
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 3 of 5] clonebundle: use 'repo.vfs' instead of 'repo.opener'
# HG changeset patch
# User Pierre-Yves David
# Date 1488421398 -3600
# Thu Mar 02 03:23:18 2017 +0100
# Node ID dc2ae1022304ad29959c52e1f549bb05472e9f43
# Parent 66df771e26f3e26aa2000b150427852fb8c88bf3
# EXP-Topic vfs.cleanup
clonebundle: use 'repo.vfs' instead of 'repo.opener'
The later is a 5 years old form.
diff --git a/hgext/clonebundles.py b/hgext/clonebundles.py
--- a/hgext/clonebundles.py
+++ b/hgext/clonebundles.py
@@ -177,7 +177,7 @@ def capabilities(orig, repo, proto):
# Only advertise if a manifest exists. This does add some I/O to requests.
# But this should be cheaper than a wasted network round trip due to
# missing file.
-if repo.opener.exists('clonebundles.manifest'):
+if repo.vfs.exists('clonebundles.manifest'):
caps.append('clonebundles')
return caps
diff --git a/mercurial/wireproto.py b/mercurial/wireproto.py
--- a/mercurial/wireproto.py
+++ b/mercurial/wireproto.py
@@ -736,7 +736,7 @@ def clonebundles(repo, proto):
depending on the request. e.g. you could advertise URLs for the closest
data center given the client's IP address.
"""
-return repo.opener.tryread('clonebundles.manifest')
+return repo.vfs.tryread('clonebundles.manifest')
wireprotocaps = ['lookup', 'changegroupsubset', 'branchmap', 'pushkey',
'known', 'getbundle', 'unbundlehash', 'batch']
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 1 of 5] localrepo: add some comment about role of various vfs object
# HG changeset patch
# User Pierre-Yves David
# Date 1470397745 -7200
# Fri Aug 05 13:49:05 2016 +0200
# Node ID b34202587b00d949dd77ff97399e50d6340779a6
# Parent 3f8f53190d6afed0aca6c6527236edad28ce785c
# EXP-Topic vfs.cleanup
localrepo: add some comment about role of various vfs object
This should make things clearer for most people.
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -253,7 +253,12 @@ class localrepository(object):
def __init__(self, baseui, path, create=False):
self.requirements = set()
+# vfs to access the working copy
self.wvfs = scmutil.vfs(path, expandpath=True, realpath=True)
+# vfs to access the content of the repository
+self.vfs = None
+# vfs to access the store part of the repository
+self.svfs = None
self.wopener = self.wvfs
self.root = self.wvfs.base
self.path = self.wvfs.join(".hg")
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 5 of 5] localrepo: deprecated 'repo.opener' (API)
# HG changeset patch
# User Pierre-Yves David
# Date 1470398170 -7200
# Fri Aug 05 13:56:10 2016 +0200
# Node ID aacf8b01b81483b4815a974f17d0ff5d214c4d3d
# Parent 437a39859c33901ea29cd22341d93be752e4acc0
# EXP-Topic vfs.cleanup
localrepo: deprecated 'repo.opener' (API)
The "new" 'repo.vfs' attribute have been around for almost 5 years. I think we
can deprecate the old form now ;-)
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -266,7 +266,6 @@ class localrepository(object):
self.nofsauditor = pathutil.pathauditor(self.root, self._checknested,
realfs=False)
self.vfs = scmutil.vfs(self.path)
-self.opener = self.vfs
self.baseui = baseui
self.ui = baseui.copy()
self.ui.copy = baseui.copy # prevent copying repo configuration
@@ -379,6 +378,11 @@ class localrepository(object):
self.ui.deprecwarn("use 'repo.wvfs' instead of 'repo.wopener'", '4.2')
return self.wvfs
+@property
+def opener(self):
+self.ui.deprecwarn("use 'repo.vfs' instead of 'repo.opener'", '4.2')
+return self.vfs
+
def close(self):
self._writecaches()
diff --git a/mercurial/statichttprepo.py b/mercurial/statichttprepo.py
--- a/mercurial/statichttprepo.py
+++ b/mercurial/statichttprepo.py
@@ -123,7 +123,6 @@ class statichttprepository(localrepo.loc
vfsclass = build_opener(ui, authinfo)
self.vfs = vfsclass(self.path)
-self.opener = self.vfs
self._phasedefaults = []
self.names = namespaces.namespaces()
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 4 of 5] statichttp: use 'repo.vfs' as the main attribute
# HG changeset patch
# User Pierre-Yves David
# Date 1488421498 -3600
# Thu Mar 02 03:24:58 2017 +0100
# Node ID 437a39859c33901ea29cd22341d93be752e4acc0
# Parent dc2ae1022304ad29959c52e1f549bb05472e9f43
# EXP-Topic vfs.cleanup
statichttp: use 'repo.vfs' as the main attribute
We are about to deprecate the repo.opener attribute, we prepare the static http
code to be ready for this change.
diff --git a/mercurial/statichttprepo.py b/mercurial/statichttprepo.py
--- a/mercurial/statichttprepo.py
+++ b/mercurial/statichttprepo.py
@@ -121,9 +121,9 @@ class statichttprepository(localrepo.loc
u = util.url(path.rstrip('/') + "/.hg")
self.path, authinfo = u.authinfo()
-opener = build_opener(ui, authinfo)
-self.opener = opener(self.path)
-self.vfs = self.opener
+vfsclass = build_opener(ui, authinfo)
+self.vfs = vfsclass(self.path)
+self.opener = self.vfs
self._phasedefaults = []
self.names = namespaces.namespaces()
@@ -148,7 +148,7 @@ class statichttprepository(localrepo.loc
raise error.RepoError(msg)
# setup store
-self.store = store.store(requirements, self.path, opener)
+self.store = store.store(requirements, self.path, vfsclass)
self.spath = self.store.path
self.svfs = self.store.opener
self.sjoin = self.store.join
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 2 of 5] localrepo: deprecated 'repo.wopener' (API)
# HG changeset patch
# User Pierre-Yves David
# Date 1470398025 -7200
# Fri Aug 05 13:53:45 2016 +0200
# Node ID 66df771e26f3e26aa2000b150427852fb8c88bf3
# Parent b34202587b00d949dd77ff97399e50d6340779a6
# EXP-Topic vfs.cleanup
localrepo: deprecated 'repo.wopener' (API)
The "new" 'repo.wvfs' attribute have been around for almost 5 years. I think we
can deprecate the old form now ;-)
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -259,7 +259,6 @@ class localrepository(object):
self.vfs = None
# vfs to access the store part of the repository
self.svfs = None
-self.wopener = self.wvfs
self.root = self.wvfs.base
self.path = self.wvfs.join(".hg")
self.origroot = path
@@ -375,6 +374,11 @@ class localrepository(object):
# generic mapping between names and nodes
self.names = namespaces.namespaces()
+@property
+def wopener(self):
+self.ui.deprecwarn("use 'repo.wvfs' instead of 'repo.wopener'", '4.2')
+return self.wvfs
+
def close(self):
self._writecaches()
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
