[PATCH 2 of 2] help: clarify that colons are allowed in fingerprints values
# HG changeset patch # User Gregory Szorc # Date 1494486152 25200 # Thu May 11 00:02:32 2017 -0700 # Node ID 202d58d5b516b5daf8262d9901a7dd456556127d # Parent fc01a88a85d64a3a440971c5e3b6c8f7db030170 help: clarify that colons are allowed in fingerprints values This was suggested by Lars Rohwedder in issue5559. diff --git a/mercurial/help/config.txt b/mercurial/help/config.txt --- a/mercurial/help/config.txt +++ b/mercurial/help/config.txt @@ -1132,6 +1132,7 @@ The following per-host settings can be d A list of hashes of the DER encoded peer/remote certificate. Values have the form ``algorithm``:``fingerprint``. e.g. ``sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2``. +In addition, colons (``:``) can appear in the fingerprint part. The following algorithms/prefixes are supported: ``sha1``, ``sha256``, ``sha512``. @@ -1181,6 +1182,7 @@ For example:: [hostsecurity] hg.example.com:fingerprints = sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2 hg2.example.com:fingerprints = sha1:914f1aff87249c09b6859b88b1906d30756491ca, sha1:fc:e2:8d:d9:51:cd:cb:c1:4d:18:6b:b7:44:8d:49:72:57:e6:cd:33 +hg3.example.com:fingerprints = sha256:9a:b0:dc:e2:75:ad:8a:b7:84:58:e5:1f:07:32:f1:87:e6:bd:24:22:af:b7:ce:8e:9c:b4:10:cf:b9:f4:0e:d2 foo.example.com:verifycertsfile = /etc/ssl/trusted-ca-certs.pem To change the default minimum protocol version to TLS 1.2 but to allow TLS 1.1 ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 1 of 2] sslutil: tweak the legacy [hostfingerprints] warning message
# HG changeset patch # User Gregory Szorc # Date 1494485377 25200 # Wed May 10 23:49:37 2017 -0700 # Node ID fc01a88a85d64a3a440971c5e3b6c8f7db030170 # Parent 1ada3d18e7fbc9069910f2c036992d2f2b28e058 sslutil: tweak the legacy [hostfingerprints] warning message Lars Rohwedder noted in issue5559 that the previous wording was confusing. I agree. diff --git a/mercurial/sslutil.py b/mercurial/sslutil.py --- a/mercurial/sslutil.py +++ b/mercurial/sslutil.py @@ -820,13 +820,11 @@ def validatesocket(sock): if settings['legacyfingerprint']: ui.warn(_('(SHA-1 fingerprint for %s found in legacy ' '[hostfingerprints] section; ' - 'if you trust this fingerprint, set the ' - 'following config value in [hostsecurity] and ' - 'remove the old one from [hostfingerprints] ' - 'to upgrade to a more secure SHA-256 ' - 'fingerprint: ' - '%s.fingerprints=%s)\n') % ( - host, host, nicefingerprint)) + 'if you trust this fingerprint, remove the old ' + 'SHA-1 fingerprint from [hostfingerprints] and ' + 'add the following entry to the new ' + '[hostsecurity] section: %s.fingerprints=%s)\n') % +(host, host, nicefingerprint)) return # Pinned fingerprint didn't match. This is a fatal error. diff --git a/tests/test-https.t b/tests/test-https.t --- a/tests/test-https.t +++ b/tests/test-https.t @@ -372,7 +372,7 @@ Fingerprints - works without cacerts (hostfingerprints) $ hg -R copy-pull id https://localhost:$HGPORT/ --insecure --config hostfingerprints.localhost=ec:d8:7c:d6:b3:86:d0:4f:c1:b8:b4:1c:9d:8f:5e:16:8e:ef:1c:03 warning: connecting to localhost using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?) - (SHA-1 fingerprint for localhost found in legacy [hostfingerprints] section; if you trust this fingerprint, set the following config value in [hostsecurity] and remove the old one from [hostfingerprints] to upgrade to a more secure SHA-256 fingerprint: localhost.fingerprints=sha256:20:de:b3:ad:b4:cd:a5:42:f0:74:41:1c:a2:70:1e:da:6e:c0:5c:16:9e:e7:22:0f:f1:b7:e5:6e:e4:92:af:7e) + (SHA-1 fingerprint for localhost found in legacy [hostfingerprints] section; if you trust this fingerprint, remove the old SHA-1 fingerprint from [hostfingerprints] and add the following entry to the new [hostsecurity] section: localhost.fingerprints=sha256:20:de:b3:ad:b4:cd:a5:42:f0:74:41:1c:a2:70:1e:da:6e:c0:5c:16:9e:e7:22:0f:f1:b7:e5:6e:e4:92:af:7e) 5fed3813f7f5 - works without cacerts (hostsecurity) @@ -387,7 +387,7 @@ Fingerprints - multiple fingerprints specified and first matches $ hg --config 'hostfingerprints.localhost=ecd87cd6b386d04fc1b8b41c9d8f5e168eef1c03, deadbeefdeadbeefdeadbeefdeadbeefdeadbeef' -R copy-pull id https://localhost:$HGPORT/ --insecure warning: connecting to localhost using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?) - (SHA-1 fingerprint for localhost found in legacy [hostfingerprints] section; if you trust this fingerprint, set the following config value in [hostsecurity] and remove the old one from [hostfingerprints] to upgrade to a more secure SHA-256 fingerprint: localhost.fingerprints=sha256:20:de:b3:ad:b4:cd:a5:42:f0:74:41:1c:a2:70:1e:da:6e:c0:5c:16:9e:e7:22:0f:f1:b7:e5:6e:e4:92:af:7e) + (SHA-1 fingerprint for localhost found in legacy [hostfingerprints] section; if you trust this fingerprint, remove the old SHA-1 fingerprint from [hostfingerprints] and add the following entry to the new [hostsecurity] section: localhost.fingerprints=sha256:20:de:b3:ad:b4:cd:a5:42:f0:74:41:1c:a2:70:1e:da:6e:c0:5c:16:9e:e7:22:0f:f1:b7:e5:6e:e4:92:af:7e) 5fed3813f7f5 $ hg --config 'hostsecurity.localhost:fingerprints=sha1:ecd87cd6b386d04fc1b8b41c9d8f5e168eef1c03, sha1:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef' -R copy-pull id https://localhost:$HGPORT/ @@ -397,7 +397,7 @@ Fingerprints - multiple fingerprints specified and last matches $ hg --config 'hostfingerprints.localhost=deadbeefdeadbeefdeadbeefdeadbeefdeadbeef, ecd87cd6b386d04fc1b8b41c9d8f5e168eef1c03' -R copy-pull id https://localhost:$HGPORT/ --insecure warning: connecting to localhost using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?) - (SHA-1 fingerprint for localhost found in legacy [hostfingerprints] section; if you trust this fingerprint, set the following config value in [hostsecurity] and remove the old one from [hostfingerprints] to upgrade to a more secure SHA-256 fingerpri
GSoC 2017 student
Hey! I would like to thank everyone in the organisation for accepting my proposal and making me a part of such an amazing organisation. I hope to complete my proposed project adhering the timeline and make the most of my GSoC time. I also wish to continue to work with the community and help resolve issues even after the project is over. PS: Apologies for getting here this late. I just shifted to a new city and was facing a lot of problems with the internet. I plan to get started from today onwards as now I'm not facing that problem anymore. Thanks Rishabh Madan ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 1 of 8] transaction: introduce "changes" dictionary to precisely track updates
So, what is the status of this series? I've important speedup work stuck behind it so I would like to see the topic move forward. On 05/03/2017 09:09 PM, Pierre-Yves David wrote: On 05/03/2017 03:46 PM, Yuya Nishihara wrote: On Wed, 3 May 2017 15:06:05 +0200, Pierre-Yves David wrote: On 05/03/2017 09:51 AM, Yuya Nishihara wrote: On Wed, 03 May 2017 01:43:38 +0200, Pierre-Yves David wrote: # HG changeset patch # User Pierre-Yves David # Date 1493742678 -7200 # Tue May 02 18:31:18 2017 +0200 # Branch stable # Node ID 6697da7c4eab3fbe3588a2f91fa3f99b16f808ac # Parent fbb5f4bf94928b98fa87871e84bb2ef972ec2d51 # EXP-Topic obscache # Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ # hg pull https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r 6697da7c4eab transaction: introduce "changes" dictionary to precisely track updates The transaction is already tracking some data intended for hooks (in 'hookargs'). However, that information is minimal as we optimise for passing data to other processes through environment variables. There are multiple places were we could use more complete and lower level information locally (eg: cache update, better report of changes to hooks, etc...). For this purpose we introduces a 'changes' dictionary on the transaction. It is intended to track every changes happening to the repository (eg: new revs, bookmarks move, phases move, obs-markers, etc). For now we just adds the 'changes' dictionary. We'll adds more tracking and usages over time. diff --git a/mercurial/transaction.py b/mercurial/transaction.py --- a/mercurial/transaction.py +++ b/mercurial/transaction.py @@ -137,6 +137,10 @@ class transaction(object): releasefn = lambda tr, success: None self.releasefn = releasefn +# A dict dedicated to precisely tracking the changes introduced in the +# transaction. +self.changes = {} I'm not sure if it's good idea to add more free-form dict to the transaction class, since that would make code less manageable in general. Can you elaborate on your worries here? The content of the dictionnary should be strictly defined by the localrepo object, and the data should be filled at low level by code that already requires a transaction to be present. So I think the result will be quite manageable. We -need- something else than "hookargs" to carry transaction related information. For example we needs to tracks bookmark movement, phases changes and tags movement. The "hookargs" dictionnary is not appropriate to track such data. My concern is unclear, sorry. The transaction code seems hard to review because it has arbitrary hooks, weakrefs in them, hookargs dict, etc. And now 'changes' dict. They all seem to make things too abstract. I believe the result looks better overall than before, but I can't say the transaction itself gets better. The net result at the end of my work should be clearer: - code tracking changes should be clearer and safer, - code reacting to changes would have more well defined spot to react to these changes. (as a bonus, we might be able to compute the hookargs, dict for the 'changes' one at some point) Cheers, -- Pierre-Yves David ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH STABLE] hghave: prefill more version of Mercurial
# HG changeset patch # User Pierre-Yves David # Date 1494335327 -7200 # Tue May 09 15:08:47 2017 +0200 # Branch stable # Node ID 7a61f40a4fada0ccadc65d9e6c8fae4483f00b85 # Parent bb96d4a497432722623ae60d9bc734a1e360179e # EXP-Topic hghave # Available At https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ # hg pull https://www.mercurial-scm.org/repo/users/marmoute/mercurial/ -r 7a61f40a4fad hghave: prefill more version of Mercurial The previous code was unable to go above version 4.0. diff --git a/tests/hghave.py b/tests/hghave.py --- a/tests/hghave.py +++ b/tests/hghave.py @@ -247,7 +247,7 @@ def gethgversion(): return (int(m.group(1)), int(m.group(2))) @checkvers("hg", "Mercurial >= %s", -list([(1.0 * x) / 10 for x in range(9, 40)])) +list([(1.0 * x) / 10 for x in range(9, 99)])) def has_hg_range(v): major, minor = v.split('.')[0:2] return gethgversion() >= (int(major), int(minor)) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 1 of 4 shelve-ext v4] scmutil: add simplekeyvaluefile reading test
On Sun, 7 May 2017 06:07:03 -0700, Kostia Balytskyi wrote: > # HG changeset patch > # User Kostia Balytskyi > # Date 1494157223 25200 > # Sun May 07 04:40:23 2017 -0700 > # Node ID e9b77b6f16c04efced06169735a813d5db82dddf > # Parent 31f42e683321f225eb9c306f8d4b3a9e8d82a1da > scmutil: add simplekeyvaluefile reading test > > Before this patch, mockvfs did not emulate readlines correctly > and there was no test for simplekeyvaluefile reading. > > diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py > --- a/mercurial/scmutil.py > +++ b/mercurial/scmutil.py > @@ -925,7 +925,10 @@ class simplekeyvaluefile(object): > def read(self): > lines = self.vfs.readlines(self.path) > try: > -d = dict(line[:-1].split('=', 1) for line in lines if line) > +# the 'if line.strip()' part prevents us from failing on empty > +# lines which only contain '\n' therefore are not skipped > +# by 'if line' > +d = dict(line[:-1].split('=', 1) for line in lines if > line.strip()) > except ValueError as e: > raise error.CorruptedState(str(e)) > return d > diff --git a/tests/test-simplekeyvaluefile.py > b/tests/test-simplekeyvaluefile.py > --- a/tests/test-simplekeyvaluefile.py > +++ b/tests/test-simplekeyvaluefile.py > @@ -33,7 +33,8 @@ class mockvfs(object): > return mockfile(path, self).read() > > def readlines(self, path): > -return mockfile(path, self).read().split('\n') > +# lines need to contain the trailing '\n' to mock the real readlines > +return [l + '\n' for l in mockfile(path, self).read().split('\n')] Just a nit. this could be .splitlines(keepends=True) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 3 of 4 shelve-ext v4] shelve: refactor shelvestate loading
On Sun, 7 May 2017 06:07:05 -0700, Kostia Balytskyi wrote: > # HG changeset patch > # User Kostia Balytskyi > # Date 1494113427 25200 > # Sat May 06 16:30:27 2017 -0700 > # Node ID 497904bddbaa75b9086c168ab2e03381dfaff165 > # Parent d78507e5b31ac9d5dbf3b8ab45c0c94b01491a0b > shelve: refactor shelvestate loading > > This is a preparatory patch which separates file reading from the > minimal validation we have (like turning version into int and > checking that this version is supported). The purpose of this patch > is to be able to read statefile form simplekeyvaluefile, which is > implemented in the following patch. > > diff --git a/hgext/shelve.py b/hgext/shelve.py > --- a/hgext/shelve.py > +++ b/hgext/shelve.py > @@ -176,38 +176,46 @@ class shelvedstate(object): > > @classmethod > def load(cls, repo): > +# Order is important, because old shelvestate file uses it > +# to detemine values of fields (i.g. version is on the first line, > +# name is on the second and so forth). Please do not change. > +keys = ['version', 'name', 'originalwctx', 'pendingctx', 'parents', > +'nodestoremove', 'branchtorestore', 'keep', 'activebook'] > +d = {} > fp = repo.vfs(cls._filename) > try: > -version = int(fp.readline().strip()) > - > -if version != cls._version: > -raise error.Abort(_('this version of shelve is incompatible ' > - 'with the version used in this repo')) > -name = fp.readline().strip() > -wctx = nodemod.bin(fp.readline().strip()) > -pendingctx = nodemod.bin(fp.readline().strip()) > -parents = [nodemod.bin(h) for h in fp.readline().split()] > -nodestoremove = [nodemod.bin(h) for h in fp.readline().split()] > -branchtorestore = fp.readline().strip() > -keep = fp.readline().strip() == cls._keep > -activebook = fp.readline().strip() > -except (ValueError, TypeError) as err: > -raise error.CorruptedState(str(err)) > +for key in keys: > +d[key] = fp.readline().strip() > finally: > fp.close() > > +# some basic syntactic verification and transformation > +try: > +d['version'] = int(d.get('version')) > +if d.get('version') != cls._version: > +raise error.Abort(_('this version of shelve is incompatible ' > +'with the version used in this repo')) > +d['originalwctx'] = nodemod.bin(d.get('originalwctx')) > +d['pendingctx'] = nodemod.bin(d.get('pendingctx')) > +d['parents'] = [nodemod.bin(h) > +for h in d.get('parents').split(' ')] > +d['nodestoremove'] = [nodemod.bin(h) > + for h in d.get('nodestoremove').split(' ')] > +except (ValueError, TypeError, AttributeError) as err: > +raise error.CorruptedState(str(err)) Perhaps it's better to catch KeyError instead of suppressing implicit AttributeError caused by d.get(key). > + > try: > obj = cls() > -obj.name = name > -obj.wctx = repo[wctx] > -obj.pendingctx = repo[pendingctx] > -obj.parents = parents > -obj.nodestoremove = nodestoremove > -obj.branchtorestore = branchtorestore > -obj.keep = keep > +obj.name = d.get('name') > +obj.wctx = repo[d.get('originalwctx')] > +obj.pendingctx = repo[d.get('pendingctx')] > +obj.parents = d.get('parents') > +obj.nodestoremove = d.get('nodestoremove') > +obj.branchtorestore = d.get('branchtorestore') > +obj.keep = d.get('keep') == cls._keep > obj.activebookmark = '' > -if activebook != cls._noactivebook: > -obj.activebookmark = activebook > +if d.get('activebook') != cls._noactivebook: > +obj.activebookmark = d.get('activebook') Are these d.get(key) uses valid? For example, repo[d.get('pendingctx')] would be a workingctx object if 'pendingctx=' is missing. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 4 of 4 shelve-ext v4] shelve: make shelvestate use simplekeyvaluefile
On Sun, 7 May 2017 06:07:06 -0700, Kostia Balytskyi wrote: > # HG changeset patch > # User Kostia Balytskyi > # Date 1494162279 25200 > # Sun May 07 06:04:39 2017 -0700 > # Node ID 9f24868156f15473b08a418765411341c96e892b > # Parent 497904bddbaa75b9086c168ab2e03381dfaff165 > shelve: make shelvestate use simplekeyvaluefile This looks good to me. > +@classmethod > +def _getversion(cls, repo): > +"""Read version information from shelvestate file""" > +fp = repo.vfs(cls._filename) > +try: > +version = int(fp.readline().strip()) > +except ValueError as err: > +raise error.CorruptedState(str(err)) > +finally: > +fp.close() > +return version > + > +@classmethod > +def _readold(cls, repo): > +"""Read the old position-based version of a shelvestate file""" > +# Order is important, because old shelvestate file uses it > +# to detemine values of fields (i.g. name is on the second line, > +# originalwctx is on the third and so forth). Please do not change. > +keys = ['version', 'name', 'originalwctx', 'pendingctx', 'parents', > +'nodestoremove', 'branchtorestore', 'keep', 'activebook'] > +# this is executed only seldomly, so it is not a big deal > +# that we open this file twice > +fp = repo.vfs(cls._filename) > +d = {} > +try: > +for key in keys: > +d[key] = fp.readline().strip() > +finally: > +fp.close() > +return d > + > +@classmethod > +def load(cls, repo): > +version = cls._getversion(repo) > +if version < cls._version: > +d = cls._readold(repo) > +elif version == cls._version: > +d = scmutil.simplekeyvaluefile(repo.vfs, cls._filename)\ > + .read(firstlinenonkeyval=True) This is okay, but I really don't understand why the simplekeyvaluefile has to: - be a class instead of load/dump functions (like json) - take a vfs instead of a file object IMHO, these design decisions just make things complicated. > diff --git a/tests/test-shelve.t b/tests/test-shelve.t > --- a/tests/test-shelve.t > +++ b/tests/test-shelve.t > @@ -1578,7 +1578,7 @@ shelve on new branch, conflict with prev >$ echo "ccc" >> a >$ hg status >M a > - $ hg unshelve > + $ hg unshelve --config shelve.oldstatefile=on Maybe this has to be updated to use a pre-generated v1 state file? >unshelving change 'default' >temporarily committing pending changes (restore with 'hg unshelve --abort') >rebasing shelved changes > @@ -1591,9 +1591,8 @@ shelve on new branch, conflict with prev > Removing restore branch information from shelvedstate file(making it looks > like > in previous versions) and running unshelve --continue > > - $ head -n 6 < .hg/shelvedstate > .hg/shelvedstate_oldformat > - $ rm .hg/shelvedstate > - $ mv .hg/shelvedstate_oldformat .hg/shelvedstate > + $ cp .hg/shelvedstate .hg/shelvedstate_old > + $ cat .hg/shelvedstate_old | grep -v 'branchtorestore' > .hg/shelvedstate ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 2 of 4 shelve-ext v4] scmutil: make simplekeyvaluefile able to have a non-key-value first line
On Sun, 7 May 2017 06:07:04 -0700, Kostia Balytskyi wrote: > # HG changeset patch > # User Kostia Balytskyi > # Date 1494158131 25200 > # Sun May 07 04:55:31 2017 -0700 > # Node ID d78507e5b31ac9d5dbf3b8ab45c0c94b01491a0b > # Parent e9b77b6f16c04efced06169735a813d5db82dddf > scmutil: make simplekeyvaluefile able to have a non-key-value first line > > To ease migration from files with version numbers in their first lines, > we want simplekeyvaluefile to support a non-key-value first line. In this > way, old versions of Mercurial will read such files, discover a newer version > than the one they know how to handle and fail gracefully, rather than with > exception. Shelve's shelvestate file is an example. > > diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py > --- a/mercurial/scmutil.py > +++ b/mercurial/scmutil.py > @@ -917,28 +917,50 @@ class simplekeyvaluefile(object): > > Keys must be alphanumerics and start with a letter, values must not > contain '\n' characters""" > +firstlinekey = '__firstline' > > def __init__(self, vfs, path, keys=None): > self.vfs = vfs > self.path = path > > -def read(self): > +def read(self, firstlinenonkeyval=False): > +"""Read the contents of a simple key-value file > + > +'firstlinenonkeyval' indicates whether the first line of file should > +be treated as a key-value pair or reuturned fully under the > +__firstline key.""" > lines = self.vfs.readlines(self.path) > +d = {} > +if firstlinenonkeyval: > +# we don't want to include '\n' in the __firstline > +d[self.firstlinekey] = lines[0][:-1] > +del lines[0] lines may be an empty list. In which case, CorruptedState should be raised. > try: > # the 'if line.strip()' part prevents us from failing on empty > # lines which only contain '\n' therefore are not skipped > # by 'if line' > -d = dict(line[:-1].split('=', 1) for line in lines if > line.strip()) > +d.update(dict(line[:-1].split('=', 1) for line in lines > + if line.strip())) Perhaps '__firstline=' should be rejected as CorruptedState for consistency with write() ? > except ValueError as e: > raise error.CorruptedState(str(e)) > return d > > -def write(self, data): > +def write(self, data, firstline=None): > """Write key=>value mapping to a file > data is a dict. Keys must be alphanumerical and start with a letter. > -Values must not contain newline characters.""" > +Values must not contain newline characters. > + > +If 'firstline' is not None, it is written to file before > +everything else, as it is, not in a key=value form""" > lines = [] > +if firstline is not None: > +lines.append('%s\n' % firstline) > + > for k, v in data.items(): > +if k == self.firstlinekey: > +e = "key name '%s' is reserved" % self.firstlinekey > +raise error.ProgrammingError(e) > if not k[0].isalpha(): > e = "keys must start with a letter in a key-value file" > raise error.ProgrammingError(e) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH V2 STABLE] serve: move hg-ssh readonly logic into hg serve
On Wed, 10 May 2017 08:50:38 -0700, Durham Goode wrote: > On 5/10/17 7:29 AM, Yuya Nishihara wrote: > > On Tue, 9 May 2017 15:48:50 -0700, Durham Goode wrote: > >> On 4/28/17 6:27 AM, Yuya Nishihara wrote: > >>> On Thu, 27 Apr 2017 10:13:21 -0400, Augie Fackler wrote: > On Wed, Apr 26, 2017 at 06:24:52PM -0700, Durham Goode wrote: > > # HG changeset patch > > # User Durham Goode > > # Date 1493255976 25200 > > # Wed Apr 26 18:19:36 2017 -0700 > > # Branch stable > > # Node ID b1964bbc387fb53b4152f19d6e929309e3f21ac6 > > # Parent 6e0368b6e0bb2aa5210daec091c0200583553a78 > > serve: move hg-ssh readonly logic into hg serve > > This looks good, but I'm very hesitant to add a new feature like this > 4 days before a release. Does anyone feel strongly that this should be > in 4.2? > >>> > >>> Agreed. The feature seems good, but no need to hurry to pick it up. > >>> > +if realcmd == 'serve' and '--read-only' in req.args: > +req.args.remove('--read-only') > >>> > >>> This increases the risk of wrong command parsing. We can't say --read-only > >>> is always a flag. > >>> > +req.ui.setconfig('hooks', 'pretxnopen.readonlyrejectpush', > + rejectpush, 'dispatch') > +req.ui.setconfig('hooks', 'prepushkey.readonlyrejectpush', > + rejectpush, 'dispatch') > >>> > >>> And --read-only won't work as expected in command server since there are > >>> write > >>> operations other than push. > >> > >> Are there write operations that wouldn't trigger the pretxnopen hook? > > > > Perhaps, "hg rollback" ? > > > >> Alternatively, I could rename this flag to be > >> --add-hgssh-push-blocking-hooks, remove it from the hg server command > >> definition entirely (just watch for it at the dispatch level and strip > >> it like I currently do). That way we can maintain the current pattern > >> while letting it be a special path for hg-ssh only. > >> > >> And if someone passes --add-hgssh-push-blocking-hooks to hg serve in a > >> position that isn't a flag, well it'll just break for them. > > > > Another idea is adding --unsafe-stdio option so the frontend script may do > > anything on the 'hg serve --stdio' process. > > I'm not sure I understand this proposal. If we have --unsafe-stdio, > doesn't that imply --stdio is safe and can block write operations? It > sounds like we don't have a way to block things like rollback, so we > couldn't guarantee --stdio is safe. I meant we could add --unsafe-stdio (or --unchecked-stdio) as an alias of --stdio. A frontend script (like hg-ssh) may bypass the constraints of 'serve --stdio' by using 'serve --unsafe-stdio'. This might sound evil, but I think it's okay as long as 'serve ' is not an API of the ssh wire protocol. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 2 of 2 stable] graft: fix graft across merges of duplicates of grafted changes
On 05/10/2017 03:42 PM, Yuya Nishihara wrote: On Tue, 09 May 2017 00:45:02 +0200, Mads Kiilerich wrote: # HG changeset patch # User Mads Kiilerich # Date 1494283376 -7200 # Tue May 09 00:42:56 2017 +0200 # Branch stable # Node ID dd256de590dfd363fa5d497d566d456f471b8d52 # Parent 6710017995b4e8b361d6ad5b897ff7d0cc658285 graft: fix graft across merges of duplicates of grafted changes Looks good to me. A couple of nits inline. Graft used findmissingrevs to find the candidates for graft duplicates in the destination. That function operates with the constraint: 2. N is not an ancestor of any node in 'common' For our purpose, we do however need: 2. There are nodes in 'common' that doesn't have N as ancestor The right candiates for graft duplicates could be computed with a revset: only(destination,transitiveroots(graftset)) I guess it actually can be computed as only(destination,roots(graftset+roots(graftset)::heads(graftset))) BUT I realize it also is wrong. There could be criss-cross-ish cases where multiple graftset roots have been merged to different branches that have grafts of other roots as ancestor. My proposed patch using min(graftset) would also fail that. Instead, the only changesets we can be sure doesn't contain grafts of any changeset in the graftset, are the ones that are common ancestors of *all* changesets in the graftset: only(destination,ancestor(graftset)) It will exclude one ancestor. In criss-cross cases where there will be more ancestors, it might be inefficient but still correct. Resending ... /Mads where transitiveroots would be a revset function for 'transitive root' and return changesets in set with no ancestor changeset in set. There doesn't seem to be any such function readily available, and we thus use the approximation of just using the smallest revision in the graft set. Can you copy this message as a code comment? It will help future readers. This change will graft more correctly, but it will also in some cases make graft slower by making it search through a bigger and unnecessary large sets of changes to find duplicates. Suppose revisions to be grafted are linear in general, I think this is acceptable. @@ -2295,7 +2295,8 @@ def _dograft(ui, repo, *revs, **opts): # check ancestors for earlier grafts ui.debug('scanning for duplicate grafts\n') -for rev in repo.changelog.findmissingrevs(revs, [crev]): +expr = revsetlang.formatspec('only(%d,min(%ld))', crev, revs) +for rev in scmutil.revrange(repo, [expr]): scmutil.revrange() may expand user aliases. Please use repo.revs() instead. Alternatively, maybe we could use findmissingrevs(min(revs), ...) to minimize the change? ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 12 of 12 RFC] policy: add helper to import cext/pure module
On Wed, 10 May 2017 11:04:41 -0700, Sean Farley wrote: > Yuya Nishihara writes: > > On Tue, 9 May 2017 11:08:03 -0700, Jun Wu wrote: > >> > > Patch 12 is where I disagree. _dualmod still requires careful renaming > >> > > and > >> > > could cause subtle problems that does not get warned by any code > >> > > checking > >> > > tools. They also require extra effort in writing and reviewing related > >> > > code. > >> > > I'd like to go through the explicit versioning way so even the renaming > >> > > churn could be avoided. > >> > > > >> > > [1]: > >> > > https://www.mercurial-scm.org/pipermail/mercurial-devel/2017-May/097329.html > >> > > > >> > > >> > I'm okay for the explicit versioning. I can adjust the last patch for > >> > that. > >> > > >> > I don't have strong preference since managing API versions would be as > >> > simple as managing explicit version constants unless we have internal > >> > dependency in C layer. > >> > >> We do have complex cases: module-level exception (mpatch.mpatchError), and > >> parsers.c with complex Python objects. Think about if someone wants to > >> catch > >> mpatchError, or change the return value of lmiter_iterkeysnext in > >> manifest.c > >> for example. > > > > Yeah, I know. My opinion is that I won't care much about that as we don't > > right now. FWIW, mpatchError isn't exported by the C module. > > > > That said, there seems no objection to switching to the per-module > > versioning, > > so maybe we can go for it. > > Maybe would could have a discussion of all the pros and cons? If that's > overkill, then I trust Yuya to make a good call here. Pros of module versioning (cons of dualdmod): a. more consistent for inner dependencies (attributes in pure/cext modules will never be mixed) b. no need to rename a function (and its callers) on API change c. module loader will be slightly simpler (unless we add the auto-compile option) Pros of dualmod (cons of module versioning): a. more permissive (only missing attributes will fallback to pure's) b. no need to maintain module version for API changes ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 08 of 11] filecommit: reuse filenode matching fparents regardless of type(fctx)
On Wed, 10 May 2017 01:34:27 -0700, Jun Wu wrote: > # HG changeset patch > # User Jun Wu > # Date 1494387835 25200 > # Tue May 09 20:43:55 2017 -0700 > # Node ID e4ad86c15b06986e51d4754f3c11bd24cbae30bb > # Parent eb7b96bc42d08670983d5d75635198f62c3459a8 > # Available At https://bitbucket.org/quark-zju/hg-draft > # hg pull https://bitbucket.org/quark-zju/hg-draft -r > e4ad86c15b06 > filecommit: reuse filenode matching fparents regardless of type(fctx) > > context.filectx is not the only class providing reusable filenode. > overlayfilectx may provide reusable filenode too. So drop the isinstance > check. > > Other filectx classes (memfilectx and workingfilectx) are subclasses of > committablefilectx, which has set filenode to None. So filenode won't be > reused as expected. > > The debug message is changed slightly to be distinguish from other kind of > reuses introduced in a future patch. > > diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py > --- a/mercurial/localrepo.py > +++ b/mercurial/localrepo.py > @@ -1423,11 +1423,10 @@ class localrepository(object): > fparent1 = manifest1.get(fname, nullid) > fparent2 = manifest2.get(fname, nullid) > -if isinstance(fctx, context.filectx): > -node = fctx.filenode() > -if node in [fparent1, fparent2]: > -self.ui.debug('reusing %s filelog entry\n' % fname) > -if manifest1.flags(fname) != fctx.flags(): > -changelist.append(fname) > -return node > +node = getattr(fctx, 'filenode', lambda: None)() Nit: it appears fctx.filenode() is a mandatory API of basefilectx. I've double-reviewed up to this patch, and timed out. I'll look the remainder tomorrow. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 2 of 2 stable v2] graft: fix graft across merges of duplicates of grafted changes
# HG changeset patch # User Mads Kiilerich # Date 1494515920 -7200 # Thu May 11 17:18:40 2017 +0200 # Branch stable # Node ID cf6482170698ea9cd590f6e72745348bef5703b4 # Parent 6710017995b4e8b361d6ad5b897ff7d0cc658285 graft: fix graft across merges of duplicates of grafted changes Graft used findmissingrevs to find the candidates for graft duplicates in the destination. That function operates with the constraint: 1. N is an ancestor of some node in 'heads' 2. N is not an ancestor of any node in 'common' For our purpose, we do however have to work correctly in cases where the graft set has multiple roots or where merges between graft ranges are skipped. The only changesets we can be sure doesn't have ancestors that are grafts of any changeset in the graftset, are the ones that are common ancestors of *all* changesets in the graftset. We thus need: 2. N is not an ancestor of all nodes in 'common' This change will graft more correctly, but it will also in some cases make graft slower by making it search through a bigger and unnecessary large sets of changes to find duplicates. In the general case of grafting individual or linear sets, we do the same amount of work as before. diff --git a/mercurial/commands.py b/mercurial/commands.py --- a/mercurial/commands.py +++ b/mercurial/commands.py @@ -2295,7 +2295,9 @@ def _dograft(ui, repo, *revs, **opts): # check ancestors for earlier grafts ui.debug('scanning for duplicate grafts\n') -for rev in repo.changelog.findmissingrevs(revs, [crev]): +# The only changesets we can be sure doesn't contain grafts of any +# revs, are the ones that are common ancestors of *all* revs: +for rev in repo.revs('only(%d,ancestor(%ld))', crev, revs): ctx = repo[rev] n = ctx.extra().get('source') if n in ids: diff --git a/tests/test-graft.t b/tests/test-graft.t --- a/tests/test-graft.t +++ b/tests/test-graft.t @@ -1341,16 +1341,15 @@ Grafting of plain changes correctly dete skipping already grafted revision 5:43e9eb70dab0 (was grafted from 4:6c9a1289e5f1) grafting 2:42127f193bcd "b" -Extending the graft range to include a merge will unfortunately make us miss -that 3 and 5 should be skipped: +Extending the graft range to include a (skipped) merge of 3 will not prevent us from +also detecting that both 3 and 5 should be skipped: $ hg up -qCr 4 $ hg graft --tool :local -r 2::7 skipping ungraftable merge revision 6 + skipping already grafted revision 3:ca093ca2f1d9 (was grafted from 1:13ec5badbf2a) skipping already grafted revision 5:43e9eb70dab0 (was grafted from 4:6c9a1289e5f1) grafting 2:42127f193bcd "b" - grafting 3:ca093ca2f1d9 "x" - note: graft of 3:ca093ca2f1d9 created no changes to commit grafting 7:d3c3f2b38ecc "xx" note: graft of 7:d3c3f2b38ecc created no changes to commit ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 1 of 2 stable v2] graft: test coverage of grafts and how merges can break duplicate detection
# HG changeset patch # User Mads Kiilerich # Date 1494281490 -7200 # Tue May 09 00:11:30 2017 +0200 # Branch stable # Node ID 6710017995b4e8b361d6ad5b897ff7d0cc658285 # Parent 247bb7a2c492d8a946cc7ed61a627d53566b7c16 graft: test coverage of grafts and how merges can break duplicate detection This demonstrates unfortunate behaviour: extending the graft range cause the graft to behave differently. When the graft range includes a merge, we fail to detect duplicates that are ancestors of the merge. diff --git a/tests/test-graft.t b/tests/test-graft.t --- a/tests/test-graft.t +++ b/tests/test-graft.t @@ -1309,4 +1309,49 @@ Graft a change into a new file previousl $ hg status --change . M b/x +Prepare for test of skipped changesets and how merges can influence it: + + $ hg merge -q -r 1 --tool :local + $ hg ci -m m + $ echo xx >> b/x + $ hg ci -m xx + + $ hg log -G -T '{rev} {desc|firstline}' + @ 7 xx + | + o6 m + |\ + | o 5 y + | | + +---o 4 y + | | + | o 3 x + | | + | o 2 b + | | + o | 1 x + |/ + o 0 a + +Grafting of plain changes correctly detects that 3 and 5 should be skipped: + + $ hg up -qCr 4 + $ hg graft --tool :local -r 2::5 + skipping already grafted revision 3:ca093ca2f1d9 (was grafted from 1:13ec5badbf2a) + skipping already grafted revision 5:43e9eb70dab0 (was grafted from 4:6c9a1289e5f1) + grafting 2:42127f193bcd "b" + +Extending the graft range to include a merge will unfortunately make us miss +that 3 and 5 should be skipped: + + $ hg up -qCr 4 + $ hg graft --tool :local -r 2::7 + skipping ungraftable merge revision 6 + skipping already grafted revision 5:43e9eb70dab0 (was grafted from 4:6c9a1289e5f1) + grafting 2:42127f193bcd "b" + grafting 3:ca093ca2f1d9 "x" + note: graft of 3:ca093ca2f1d9 created no changes to commit + grafting 7:d3c3f2b38ecc "xx" + note: graft of 7:d3c3f2b38ecc created no changes to commit + $ cd .. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 08 of 11] filecommit: reuse filenode matching fparents regardless of type(fctx)
Excerpts from Yuya Nishihara's message of 2017-05-12 00:15:41 +0900: > Nit: it appears fctx.filenode() is a mandatory API of basefilectx. Good catch. The only exception is tests/drawdag.py. I'll update it. > I've double-reviewed up to this patch, and timed out. I'll look the > remainder tomorrow. Thanks! ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[Bug 5564] New: The execution bit is lost when merging conflicts with an external merge tool
https://bz.mercurial-scm.org/show_bug.cgi?id=5564 Bug ID: 5564 Summary: The execution bit is lost when merging conflicts with an external merge tool Product: Mercurial Version: stable branch Hardware: PC OS: Linux Status: UNCONFIRMED Severity: feature Priority: wish Component: Mercurial Assignee: bugzi...@mercurial-scm.org Reporter: fgiovana...@aragnet.com CC: mercurial-devel@mercurial-scm.org When a merge or a rebase find a conflict on a executable file the execution bit gets lost when the file gets merged by an external tool ( kdiff3 ). To reproduce make a conflicting change ( even an automatically resolvable one) on an executable file, then: hg rebase -r 181 -d 183 --keep will preserve the executable bit, but hg rebase -r 181 -d 183 --keep -t kdiff3 won't preserve it. Note that the conflict gets automatically resolved by both tools, so no window is created. I'm filing a feature request because the bug is neither a mercurial issue neither a kdiff3 one ( I know that it preserve the bit) but mainly an integration issue. Regards. Federico -- You are receiving this mail because: You are on the CC list for the bug. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
RE: [PATCH 3 of 4 shelve-ext v4] shelve: refactor shelvestate loading
> -Original Message- > From: Yuya Nishihara [mailto:you...@gmail.com] On Behalf Of Yuya > Nishihara > Sent: Thursday, 11 May, 2017 14:38 > To: Kostia Balytskyi > Cc: mercurial-devel@mercurial-scm.org > Subject: Re: [PATCH 3 of 4 shelve-ext v4] shelve: refactor shelvestate loading > > On Sun, 7 May 2017 06:07:05 -0700, Kostia Balytskyi wrote: > > # HG changeset patch > > # User Kostia Balytskyi # Date 1494113427 25200 > > # Sat May 06 16:30:27 2017 -0700 > > # Node ID 497904bddbaa75b9086c168ab2e03381dfaff165 > > # Parent d78507e5b31ac9d5dbf3b8ab45c0c94b01491a0b > > shelve: refactor shelvestate loading > > > > This is a preparatory patch which separates file reading from the > > minimal validation we have (like turning version into int and checking > > that this version is supported). The purpose of this patch is to be > > able to read statefile form simplekeyvaluefile, which is implemented > > in the following patch. > > > > diff --git a/hgext/shelve.py b/hgext/shelve.py > > --- a/hgext/shelve.py > > +++ b/hgext/shelve.py > > @@ -176,38 +176,46 @@ class shelvedstate(object): > > > > @classmethod > > def load(cls, repo): > > +# Order is important, because old shelvestate file uses it > > +# to detemine values of fields (i.g. version is on the first line, > > +# name is on the second and so forth). Please do not change. > > +keys = ['version', 'name', 'originalwctx', 'pendingctx', 'parents', > > +'nodestoremove', 'branchtorestore', 'keep', 'activebook'] > > +d = {} > > fp = repo.vfs(cls._filename) > > try: > > -version = int(fp.readline().strip()) > > - > > -if version != cls._version: > > -raise error.Abort(_('this version of shelve is > > incompatible ' > > - 'with the version used in this repo')) > > -name = fp.readline().strip() > > -wctx = nodemod.bin(fp.readline().strip()) > > -pendingctx = nodemod.bin(fp.readline().strip()) > > -parents = [nodemod.bin(h) for h in fp.readline().split()] > > -nodestoremove = [nodemod.bin(h) for h in fp.readline().split()] > > -branchtorestore = fp.readline().strip() > > -keep = fp.readline().strip() == cls._keep > > -activebook = fp.readline().strip() > > -except (ValueError, TypeError) as err: > > -raise error.CorruptedState(str(err)) > > +for key in keys: > > +d[key] = fp.readline().strip() > > finally: > > fp.close() > > > > +# some basic syntactic verification and transformation > > +try: > > +d['version'] = int(d.get('version')) > > +if d.get('version') != cls._version: > > +raise error.Abort(_('this version of shelve is > > incompatible ' > > +'with the version used in this repo')) > > +d['originalwctx'] = nodemod.bin(d.get('originalwctx')) > > +d['pendingctx'] = nodemod.bin(d.get('pendingctx')) > > +d['parents'] = [nodemod.bin(h) > > +for h in d.get('parents').split(' ')] > > +d['nodestoremove'] = [nodemod.bin(h) > > + for h in d.get('nodestoremove').split(' > > ')] > > +except (ValueError, TypeError, AttributeError) as err: > > +raise error.CorruptedState(str(err)) > > Perhaps it's better to catch KeyError instead of suppressing implicit > AttributeError caused by d.get(key). > > > + > > try: > > obj = cls() > > -obj.name = name > > -obj.wctx = repo[wctx] > > -obj.pendingctx = repo[pendingctx] > > -obj.parents = parents > > -obj.nodestoremove = nodestoremove > > -obj.branchtorestore = branchtorestore > > -obj.keep = keep > > +obj.name = d.get('name') > > +obj.wctx = repo[d.get('originalwctx')] > > +obj.pendingctx = repo[d.get('pendingctx')] > > +obj.parents = d.get('parents') > > +obj.nodestoremove = d.get('nodestoremove') > > +obj.branchtorestore = d.get('branchtorestore') > > +obj.keep = d.get('keep') == cls._keep > > obj.activebookmark = '' > > -if activebook != cls._noactivebook: > > -obj.activebookmark = activebook > > +if d.get('activebook') != cls._noactivebook: > > +obj.activebookmark = d.get('activebook') > > Are these d.get(key) uses valid? For example, repo[d.get('pendingctx')] > would be a workingctx object if 'pendingctx=' is missing. You are right. I will change the stuff above to use [] instead of .get(), and here will only change d.get('activebook') to d.get('activebook', ''). Together these things should handle all the cases. _
[PATCH 1 of 4 V2] bundle2: don't check for whether we can do stream clones
# HG changeset patch # User Siddharth Agarwal # Date 1494289851 25200 # Mon May 08 17:30:51 2017 -0700 # Node ID b433f29233e32d1c1e665b0c04ae0ef356ce0396 # Parent 8f1a2b848b52ea7bf3fe2404e3b62924c7aae93f bundle2: don't check for whether we can do stream clones At the moment this isn't used and all stream clones use the legacy protocol. In an upcoming diff, canperformstreamclone will print out a message if a stream clone was requested but couldn't happen for some reason. Removing this call ensures the message isn't printed twice. diff --git a/mercurial/exchange.py b/mercurial/exchange.py --- a/mercurial/exchange.py +++ b/mercurial/exchange.py @@ -1332,7 +1332,9 @@ def _pullbundle2(pullop): For now, the only supported data are changegroup.""" kwargs = {'bundlecaps': caps20to10(pullop.repo)} -streaming, streamreqs = streamclone.canperformstreamclone(pullop) +# At the moment we don't do stream clones over bundle2. If that is +# implemented then here's where the check for that will go. +streaming = False # pulling changegroup pullop.stepsdone.add('changegroup') ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 2 of 4 V2] clone: test streaming disabled because client is missing requirement
# HG changeset patch # User Siddharth Agarwal # Date 149429 25200 # Mon May 08 18:47:24 2017 -0700 # Node ID 16696b166a70288f05d9306591d66fdd87c51702 # Parent b433f29233e32d1c1e665b0c04ae0ef356ce0396 clone: test streaming disabled because client is missing requirement Turns out we had no coverage for this important case. diff --git a/tests/test-http-bundle1.t b/tests/test-http-bundle1.t --- a/tests/test-http-bundle1.t +++ b/tests/test-http-bundle1.t @@ -66,6 +66,23 @@ try to clone via stream, should use pull updating to branch default 4 files updated, 0 files merged, 0 files removed, 0 files unresolved +try to clone via stream but missing requirements, so should use pull instead + + $ cat > $TESTTMP/removesupportedformat.py << EOF + > from mercurial import localrepo + > def extsetup(ui): + > localrepo.localrepository.supportedformats.remove('generaldelta') + > EOF + + $ hg clone --config extensions.rsf=$TESTTMP/removesupportedformat.py --uncompressed http://localhost:$HGPORT/ copy3 + requesting all changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 4 changes to 4 files + updating to branch default + 4 files updated, 0 files merged, 0 files removed, 0 files unresolved + clone via pull $ hg clone http://localhost:$HGPORT1/ copy-pull diff --git a/tests/test-http.t b/tests/test-http.t --- a/tests/test-http.t +++ b/tests/test-http.t @@ -57,6 +57,23 @@ try to clone via stream, should use pull updating to branch default 4 files updated, 0 files merged, 0 files removed, 0 files unresolved +try to clone via stream but missing requirements, so should use pull instead + + $ cat > $TESTTMP/removesupportedformat.py << EOF + > from mercurial import localrepo + > def extsetup(ui): + > localrepo.localrepository.supportedformats.remove('generaldelta') + > EOF + + $ hg clone --config extensions.rsf=$TESTTMP/removesupportedformat.py --uncompressed http://localhost:$HGPORT/ copy3 + requesting all changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 4 changes to 4 files + updating to branch default + 4 files updated, 0 files merged, 0 files removed, 0 files unresolved + clone via pull $ hg clone http://localhost:$HGPORT1/ copy-pull ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 3 of 4 V2] clone: warn when streaming was requested but couldn't be performed
# HG changeset patch # User Siddharth Agarwal # Date 1494298866 25200 # Mon May 08 20:01:06 2017 -0700 # Node ID 052bd5cfe3769b10c64a4a39d9734a2740d44e16 # Parent 16696b166a70288f05d9306591d66fdd87c51702 clone: warn when streaming was requested but couldn't be performed This helps both users and the people who support them figure out why a stream clone couldn't be performed. In an upcoming patch we're going to add a way for servers to hard abort on a full getbundle. In those cases servers might expect clients to perform a stream clone, so it's important to communicate why one couldn't be done. diff --git a/mercurial/streamclone.py b/mercurial/streamclone.py --- a/mercurial/streamclone.py +++ b/mercurial/streamclone.py @@ -80,11 +80,21 @@ def canperformstreamclone(pullop, bailif streamreqs = remote.capable('streamreqs') # This is weird and shouldn't happen with modern servers. if not streamreqs: +pullop.repo.ui.warn(_( +'warning: stream clone requested but server has them ' +'disabled\n')) return False, None streamreqs = set(streamreqs.split(',')) # Server requires something we don't support. Bail. -if streamreqs - repo.supportedformats: +missingreqs = streamreqs - repo.supportedformats +if missingreqs: +pullop.repo.ui.warn(_( +'warning: stream clone requested but client is missing ' +'requirements: %s\n') % ', '.join(sorted(missingreqs))) +pullop.repo.ui.warn( +_('(see https://www.mercurial-scm.org/wiki/MissingRequirement ' + 'for more information)\n')) return False, None requirements = streamreqs diff --git a/tests/test-http-bundle1.t b/tests/test-http-bundle1.t --- a/tests/test-http-bundle1.t +++ b/tests/test-http-bundle1.t @@ -58,6 +58,7 @@ clone via stream try to clone via stream, should use pull instead $ hg clone --uncompressed http://localhost:$HGPORT1/ copy2 + warning: stream clone requested but server has them disabled requesting all changes adding changesets adding manifests @@ -75,6 +76,8 @@ try to clone via stream but missing requ > EOF $ hg clone --config extensions.rsf=$TESTTMP/removesupportedformat.py --uncompressed http://localhost:$HGPORT/ copy3 + warning: stream clone requested but client is missing requirements: generaldelta + (see https://www.mercurial-scm.org/wiki/MissingRequirement for more information) requesting all changes adding changesets adding manifests diff --git a/tests/test-http.t b/tests/test-http.t --- a/tests/test-http.t +++ b/tests/test-http.t @@ -49,6 +49,7 @@ clone via stream try to clone via stream, should use pull instead $ hg clone --uncompressed http://localhost:$HGPORT1/ copy2 + warning: stream clone requested but server has them disabled requesting all changes adding changesets adding manifests @@ -66,6 +67,8 @@ try to clone via stream but missing requ > EOF $ hg clone --config extensions.rsf=$TESTTMP/removesupportedformat.py --uncompressed http://localhost:$HGPORT/ copy3 + warning: stream clone requested but client is missing requirements: generaldelta + (see https://www.mercurial-scm.org/wiki/MissingRequirement for more information) requesting all changes adding changesets adding manifests ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 4 of 4 V2] clone: add a server-side option to disable full getbundles (pull-based clones)
# HG changeset patch # User Siddharth Agarwal # Date 1494525005 25200 # Thu May 11 10:50:05 2017 -0700 # Node ID fa570690257581e3197f8730b6522e80d58a3f45 # Parent 052bd5cfe3769b10c64a4a39d9734a2740d44e16 clone: add a server-side option to disable full getbundles (pull-based clones) For large enough repositories, pull-based clones take too long, and an attempt to use them indicates some sort of configuration or other issue or maybe an outdated Mercurial. Add a config option to disable them. diff --git a/mercurial/help/config.txt b/mercurial/help/config.txt --- a/mercurial/help/config.txt +++ b/mercurial/help/config.txt @@ -1660,6 +1660,12 @@ Controls generic server settings. When set, clients will try to use the uncompressed streaming protocol. (default: False) +``disablefullbundle`` +When set, servers will refuse attempts to do pull-based clones. +If this option is set, ``preferuncompressed`` and/or clone bundles +are highly recommended. Partial clones will still be allowed. +(default: False) + ``validate`` Whether to validate the completeness of pushed changesets by checking that all new file revisions specified in manifests are diff --git a/mercurial/wireproto.py b/mercurial/wireproto.py --- a/mercurial/wireproto.py +++ b/mercurial/wireproto.py @@ -16,6 +16,7 @@ from .i18n import _ from .node import ( bin, hex, +nullid, ) from . import ( @@ -841,6 +842,17 @@ def getbundle(repo, proto, others): hint=bundle2requiredhint) try: +if repo.ui.configbool('server', 'disablefullbundle', False): +# Check to see if this is a full clone. +clheads = set(repo.changelog.heads()) +heads = set(opts.get('heads', set())) +common = set(opts.get('common', set())) +common.discard(nullid) +if not common and clheads == heads: +raise error.Abort( +_('server has pull-based clones disabled'), +hint=_('remove --pull if specified or upgrade Mercurial')) + chunks = exchange.getbundlechunks(repo, 'serve', **opts) except error.Abort as exc: # cleanly forward Abort error to the client diff --git a/tests/test-http-bundle1.t b/tests/test-http-bundle1.t --- a/tests/test-http-bundle1.t +++ b/tests/test-http-bundle1.t @@ -365,3 +365,41 @@ Check error reporting while pulling/clon this is an exercise [255] $ cat error.log + +disable pull-based clones + + $ hg -R test serve -p $HGPORT1 -d --pid-file=hg4.pid -E error.log --config server.disablefullbundle=True + $ cat hg4.pid >> $DAEMON_PIDS + $ hg clone http://localhost:$HGPORT1/ disable-pull-clone + requesting all changes + abort: remote error: + server has pull-based clones disabled + [255] + +... but keep stream clones working + + $ hg clone --uncompressed --noupdate http://localhost:$HGPORT1/ test-stream-clone + streaming all changes + * files to transfer, * of data (glob) + transferred * in * seconds (* KB/sec) (glob) + searching for changes + no changes found + +... and also keep partial clones and pulls working + $ hg clone http://localhost:$HGPORT1 --rev 0 test-partial-clone + adding changesets + adding manifests + adding file changes + added 1 changesets with 4 changes to 4 files + updating to branch default + 4 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg pull -R test-partial-clone + pulling from http://localhost:$HGPORT1/ + searching for changes + adding changesets + adding manifests + adding file changes + added 2 changesets with 3 changes to 3 files + (run 'hg update' to get a working copy) + + $ cat error.log diff --git a/tests/test-http.t b/tests/test-http.t --- a/tests/test-http.t +++ b/tests/test-http.t @@ -354,6 +354,44 @@ check abort error reporting while pullin [255] $ cat error.log +disable pull-based clones + + $ hg -R test serve -p $HGPORT1 -d --pid-file=hg4.pid -E error.log --config server.disablefullbundle=True + $ cat hg4.pid >> $DAEMON_PIDS + $ hg clone http://localhost:$HGPORT1/ disable-pull-clone + requesting all changes + remote: abort: server has pull-based clones disabled + abort: pull failed on remote + (remove --pull if specified or upgrade Mercurial) + [255] + +... but keep stream clones working + + $ hg clone --uncompressed --noupdate http://localhost:$HGPORT1/ test-stream-clone + streaming all changes + * files to transfer, * of data (glob) + transferred * in * seconds (*/sec) (glob) + searching for changes + no changes found + $ cat error.log + +... and also keep partial clones and pulls working + $ hg clone http://localhost:$HGPORT1 --rev 0 test-partial-clone + adding changesets + adding manifests + adding file changes + added 1 changesets with 4 changes to 4 files + updating to branch default + 4 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg pull -R test-partial-clone +
Re: [PATCH 2 of 3] rebase: allow rebase even if some revisions need no rebase (BC) (issue5422)
On Fri, May 5, 2017 at 11:14 PM, Martin von Zweigbergk wrote: > On Fri, May 5, 2017 at 7:19 PM, Yuya Nishihara wrote: >> On Thu, 04 May 2017 13:20:11 -0700, Martin von Zweigbergk via >> Mercurial-devel wrote: >>> # HG changeset patch >>> # User Martin von Zweigbergk >>> # Date 1489219466 28800 >>> # Sat Mar 11 00:04:26 2017 -0800 >>> # Node ID 2765672140ea50f5587be486b25af10509a3d35b >>> # Parent 5af88edb7a0bbafad999a2cef8968efbfd64becf >>> rebase: allow rebase even if some revisions need no rebase (BC) (issue5422) >>> >>> This allows you to do e.g. "hg rebase -d @ -r 'draft()'" even if some >>> drafts are already based off of @. You'd still need to exclude >>> obsolete and troubled revisions, though. We will deal with those cases >>> later. >>> >>> Implemented by treating state[rev]==rev as "no need to rebase". I >>> considered adding another fake revision number like revdone=-6. That >>> would make the code clearer in a few places, but would add extra code >>> in other places. >> >> Perhaps updatebookmarks() will need a guard to not flag >> tr.hookargs['bookmark_moved'] by null move. > > Ah, good catch! I will also try to add more tests. I couldn't actually find any concrete effect of this. The only difference if we don't consider bookmarks "moves" from X to X is that deletedivergent() does not get called. I can't see how that would make a difference in practice either. I'll still change it in v2 because it makes sense to not include the un-moved nodes in the 'nstates' dict used for bookmarks. > >> >> I'm not sure if the fake revision is better. Using 'state[rev] = rev' makes >> sense, but we'll have to check all places where state[rev] are used anyway. > > I'm not sure which I prefer either, so I'll just pick the 'state[rev] > = rev' version for v2, but let me know if you change you mind. ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH v2] rebase: allow rebase even if some revisions need no rebase (BC) (issue5422)
# HG changeset patch # User Martin von Zweigbergk # Date 1494527838 25200 # Thu May 11 11:37:18 2017 -0700 # Node ID 14379c69a54f03157b09410dbc0116d7410d6623 # Parent 1ada3d18e7fbc9069910f2c036992d2f2b28e058 rebase: allow rebase even if some revisions need no rebase (BC) (issue5422) This allows you to do e.g. "hg rebase -d @ -r 'draft()'" even if some drafts are already based off of @. You'd still need to exclude obsolete and troubled revisions, though. We will deal with those cases later. Implemented by treating state[rev]==rev as "no need to rebase". I considered adding another fake revision number like revdone=-6. That would make the code clearer in a few places, but would add extra code in other places. I moved the existing test out of test-rebase-base.t and into a new file and added more tests there, since not all are using --base. diff --git a/hgext/rebase.py b/hgext/rebase.py --- a/hgext/rebase.py +++ b/hgext/rebase.py @@ -384,7 +384,9 @@ names = repo.nodetags(ctx.node()) + repo.nodebookmarks(ctx.node()) if names: desc += ' (%s)' % ' '.join(names) -if self.state[rev] == revtodo: +if self.state[rev] == rev: +ui.status(_('already rebased %s\n') % desc) +elif self.state[rev] == revtodo: pos += 1 ui.status(_('rebasing %s\n') % desc) ui.progress(_("rebasing"), pos, ("%d:%s" % (rev, ctx)), @@ -508,7 +510,7 @@ # Nodeids are needed to reset bookmarks nstate = {} for k, v in self.state.iteritems(): -if v > nullmerge: +if v > nullmerge and v != k: nstate[repo[k].node()] = repo[v].node() elif v == revprecursor: succ = self.obsoletenotrebased[k] @@ -1248,6 +1250,7 @@ roots.sort() state = dict.fromkeys(rebaseset, revtodo) detachset = set() +emptyrebase = True for root in roots: commonbase = root.ancestor(dest) if commonbase == root: @@ -1260,9 +1263,13 @@ else: samebranch = root.branch() == dest.branch() if not collapse and samebranch and root in dest.children(): +# mark the revision as done by setting its new revision +# equal to its old (current) revisions +state[root.rev()] = root.rev() repo.ui.debug('source is a child of destination\n') -return None +continue +emptyrebase = False repo.ui.debug('rebase onto %s starting from %s\n' % (dest, root)) # Rebase tries to turn into a parent of while # preserving the number of parents of rebased changesets: @@ -1305,6 +1312,13 @@ # ancestors of not ancestors of detachset.update(repo.changelog.findmissingrevs([commonbase.rev()], [root.rev()])) +if emptyrebase: +return None +for rev in sorted(state): +parents = [p for p in repo.changelog.parentrevs(rev) if p != nullrev] +# if all parents of this revision are done, then so is this revision +if parents and all((state.get(p) == p for p in parents)): +state[rev] = rev for r in detachset: if r not in state: state[r] = nullmerge @@ -1332,7 +1346,7 @@ if obsolete.isenabled(repo, obsolete.createmarkersopt): markers = [] for rev, newrev in sorted(state.items()): -if newrev >= 0: +if newrev >= 0 and newrev != rev: if rev in skipped: succs = () elif collapsedas is not None: @@ -1343,7 +1357,8 @@ if markers: obsolete.createmarkers(repo, markers) else: -rebased = [rev for rev in state if state[rev] > nullmerge] +rebased = [rev for rev in state + if state[rev] > nullmerge and state[rev] != rev] if rebased: stripped = [] for root in repo.set('roots(%ld)', rebased): diff --git a/tests/test-rebase-base.t b/tests/test-rebase-base.t --- a/tests/test-rebase-base.t +++ b/tests/test-rebase-base.t @@ -298,18 +298,6 @@ | o 0: M0 -Mixed rebasable and non-rebasable bases (unresolved, issue5422): - - $ rebasewithdag -b C+D -d B <<'EOS' - > D - > / - > B C - > |/ - > A - > EOS - nothing to rebase - [1] - Disconnected graph: $ rebasewithdag -b B -d Z <<'EOS' diff --git a/tests/test-rebase-partial.t b/tests/test-rebase-partial.t new file mode 100644 --- /dev/null +++ b/tests/test-rebase-partial.t @@ -0,0 +1,95 @@ +Tests rebasing with part of the rebase set already in the +destination (issue5422) + + $ cat >> $HGRCPATH < [extensions] + > rebase= + > drawdag=$TESTDIR/drawdag.py + > + > [experimental] + > evolution=createmarkers,allowunstable + > + > [alias]
RE: [PATCH 4 of 4 shelve-ext v4] shelve: make shelvestate use simplekeyvaluefile
> -Original Message- > From: Yuya Nishihara [mailto:you...@gmail.com] On Behalf Of Yuya > Nishihara > Sent: Thursday, 11 May, 2017 14:48 > To: Kostia Balytskyi > Cc: mercurial-devel@mercurial-scm.org > Subject: Re: [PATCH 4 of 4 shelve-ext v4] shelve: make shelvestate use > simplekeyvaluefile > > On Sun, 7 May 2017 06:07:06 -0700, Kostia Balytskyi wrote: > > # HG changeset patch > > # User Kostia Balytskyi # Date 1494162279 25200 > > # Sun May 07 06:04:39 2017 -0700 > > # Node ID 9f24868156f15473b08a418765411341c96e892b > > # Parent 497904bddbaa75b9086c168ab2e03381dfaff165 > > shelve: make shelvestate use simplekeyvaluefile > > This looks good to me. > > > +@classmethod > > +def _getversion(cls, repo): > > +"""Read version information from shelvestate file""" > > +fp = repo.vfs(cls._filename) > > +try: > > +version = int(fp.readline().strip()) > > +except ValueError as err: > > +raise error.CorruptedState(str(err)) > > +finally: > > +fp.close() > > +return version > > + > > +@classmethod > > +def _readold(cls, repo): > > +"""Read the old position-based version of a shelvestate file""" > > +# Order is important, because old shelvestate file uses it > > +# to detemine values of fields (i.g. name is on the second line, > > +# originalwctx is on the third and so forth). Please do not change. > > +keys = ['version', 'name', 'originalwctx', 'pendingctx', 'parents', > > +'nodestoremove', 'branchtorestore', 'keep', 'activebook'] > > +# this is executed only seldomly, so it is not a big deal > > +# that we open this file twice > > +fp = repo.vfs(cls._filename) > > +d = {} > > +try: > > +for key in keys: > > +d[key] = fp.readline().strip() > > +finally: > > +fp.close() > > +return d > > + > > +@classmethod > > +def load(cls, repo): > > +version = cls._getversion(repo) > > +if version < cls._version: > > +d = cls._readold(repo) > > +elif version == cls._version: > > +d = scmutil.simplekeyvaluefile(repo.vfs, cls._filename)\ > > + .read(firstlinenonkeyval=True) > > This is okay, but I really don't understand why the simplekeyvaluefile has to: > > - be a class instead of load/dump functions (like json) > - take a vfs instead of a file object > > IMHO, these design decisions just make things complicated. I like it being a class because it allows inheritance, which allows simplekeyvalue users to add validation. Also, it can serve as a container for firstlinekey. vfs vs a file object - I don't have an answer to this, maybe file object would've been better, but I'd prefer to not rewrite it. > > > diff --git a/tests/test-shelve.t b/tests/test-shelve.t > > --- a/tests/test-shelve.t > > +++ b/tests/test-shelve.t > > @@ -1578,7 +1578,7 @@ shelve on new branch, conflict with prev > >$ echo "ccc" >> a > >$ hg status > >M a > > - $ hg unshelve > > + $ hg unshelve --config shelve.oldstatefile=on > > Maybe this has to be updated to use a pre-generated v1 state file? > > >unshelving change 'default' > >temporarily committing pending changes (restore with 'hg unshelve -- > abort') > >rebasing shelved changes > > @@ -1591,9 +1591,8 @@ shelve on new branch, conflict with prev > > Removing restore branch information from shelvedstate file(making it > > looks like in previous versions) and running unshelve --continue > > > > - $ head -n 6 < .hg/shelvedstate > .hg/shelvedstate_oldformat > > - $ rm .hg/shelvedstate > > - $ mv .hg/shelvedstate_oldformat .hg/shelvedstate > > + $ cp .hg/shelvedstate .hg/shelvedstate_old $ cat > > + .hg/shelvedstate_old | grep -v 'branchtorestore' > .hg/shelvedstate ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 3 of 4 shelve-ext v5] shelve: refactor shelvestate loading
# HG changeset patch # User Kostia Balytskyi # Date 1494520029 25200 # Thu May 11 09:27:09 2017 -0700 # Node ID 9170d8e0460bfe9ae4ea79394786cf2cac686fcb # Parent b79f27451d7af524c07b89a983cf750bd96292c5 shelve: refactor shelvestate loading This is a preparatory patch which separates file reading from the minimal validation we have (like turning version into int and checking that this version is supported). The purpose of this patch is to be able to read statefile form simplekeyvaluefile, which is implemented in the following patch. diff --git a/hgext/shelve.py b/hgext/shelve.py --- a/hgext/shelve.py +++ b/hgext/shelve.py @@ -176,38 +176,46 @@ class shelvedstate(object): @classmethod def load(cls, repo): +# Order is important, because old shelvestate file uses it +# to detemine values of fields (i.g. version is on the first line, +# name is on the second and so forth). Please do not change. +keys = ['version', 'name', 'originalwctx', 'pendingctx', 'parents', +'nodestoremove', 'branchtorestore', 'keep', 'activebook'] +d = {} fp = repo.vfs(cls._filename) try: -version = int(fp.readline().strip()) - -if version != cls._version: -raise error.Abort(_('this version of shelve is incompatible ' - 'with the version used in this repo')) -name = fp.readline().strip() -wctx = nodemod.bin(fp.readline().strip()) -pendingctx = nodemod.bin(fp.readline().strip()) -parents = [nodemod.bin(h) for h in fp.readline().split()] -nodestoremove = [nodemod.bin(h) for h in fp.readline().split()] -branchtorestore = fp.readline().strip() -keep = fp.readline().strip() == cls._keep -activebook = fp.readline().strip() -except (ValueError, TypeError) as err: -raise error.CorruptedState(str(err)) +for key in keys: +d[key] = fp.readline().strip() finally: fp.close() +# some basic syntactic verification and transformation +try: +d['version'] = int(d['version']) +if d['version'] != cls._version: +raise error.Abort(_('this version of shelve is incompatible ' +'with the version used in this repo')) +d['originalwctx'] = nodemod.bin(d['originalwctx']) +d['pendingctx'] = nodemod.bin(d['pendingctx']) +d['parents'] = [nodemod.bin(h) +for h in d['parents'].split(' ')] +d['nodestoremove'] = [nodemod.bin(h) + for h in d['nodestoremove'].split(' ')] +except (ValueError, TypeError, KeyError) as err: +raise error.CorruptedState(str(err)) + try: obj = cls() -obj.name = name -obj.wctx = repo[wctx] -obj.pendingctx = repo[pendingctx] -obj.parents = parents -obj.nodestoremove = nodestoremove -obj.branchtorestore = branchtorestore -obj.keep = keep +obj.name = d.get('name') +obj.wctx = repo[d.get('originalwctx')] +obj.pendingctx = repo[d.get('pendingctx')] +obj.parents = d.get('parents') +obj.nodestoremove = d.get('nodestoremove') +obj.branchtorestore = d.get('branchtorestore') +obj.keep = d.get('keep') == cls._keep obj.activebookmark = '' -if activebook != cls._noactivebook: -obj.activebookmark = activebook +if d.get('activebook', '') != cls._noactivebook: +obj.activebookmark = d.get('activebook') except error.RepoLookupError as err: raise error.CorruptedState(str(err)) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 2 of 4 shelve-ext v5] scmutil: make simplekeyvaluefile able to have a non-key-value first line
# HG changeset patch # User Kostia Balytskyi # Date 1494517773 25200 # Thu May 11 08:49:33 2017 -0700 # Node ID b79f27451d7af524c07b89a983cf750bd96292c5 # Parent ebf9affb2f7013a6a597ddd0765c786ed36f3722 scmutil: make simplekeyvaluefile able to have a non-key-value first line To ease migration from files with version numbers in their first lines, we want simplekeyvaluefile to support a non-key-value first line. In this way, old versions of Mercurial will read such files, discover a newer version than the one they know how to handle and fail gracefully, rather than with exception. Shelve's shelvestate file is an example. diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py --- a/mercurial/scmutil.py +++ b/mercurial/scmutil.py @@ -917,28 +917,57 @@ class simplekeyvaluefile(object): Keys must be alphanumerics and start with a letter, values must not contain '\n' characters""" +firstlinekey = '__firstline' def __init__(self, vfs, path, keys=None): self.vfs = vfs self.path = path -def read(self): +def read(self, firstlinenonkeyval=False): +"""Read the contents of a simple key-value file + +'firstlinenonkeyval' indicates whether the first line of file should +be treated as a key-value pair or reuturned fully under the +__firstline key.""" lines = self.vfs.readlines(self.path) +d = {} +if firstlinenonkeyval: +if not lines: +e = _("empty simplekeyvalue file") +raise error.CorruptedState(e) +# we don't want to include '\n' in the __firstline +d[self.firstlinekey] = lines[0][:-1] +del lines[0] + try: # the 'if line.strip()' part prevents us from failing on empty # lines which only contain '\n' therefore are not skipped # by 'if line' -d = dict(line[:-1].split('=', 1) for line in lines if line.strip()) +update_dict = dict(line[:-1].split('=', 1) for line in lines + if line.strip()) +if self.firstlinekey in update_dict: +e = _("%r can't be used as a key") +raise error.CorruptedState(e % self.firstlinekey) +d.update(update_dict) except ValueError as e: raise error.CorruptedState(str(e)) return d -def write(self, data): +def write(self, data, firstline=None): """Write key=>value mapping to a file data is a dict. Keys must be alphanumerical and start with a letter. -Values must not contain newline characters.""" +Values must not contain newline characters. + +If 'firstline' is not None, it is written to file before +everything else, as it is, not in a key=value form""" lines = [] +if firstline is not None: +lines.append('%s\n' % firstline) + for k, v in data.items(): +if k == self.firstlinekey: +e = "key name '%s' is reserved" % self.firstlinekey +raise error.ProgrammingError(e) if not k[0].isalpha(): e = "keys must start with a letter in a key-value file" raise error.ProgrammingError(e) diff --git a/tests/test-simplekeyvaluefile.py b/tests/test-simplekeyvaluefile.py --- a/tests/test-simplekeyvaluefile.py +++ b/tests/test-simplekeyvaluefile.py @@ -72,5 +72,13 @@ class testsimplekeyvaluefile(unittest.Te self.assertRaises(error.CorruptedState, scmutil.simplekeyvaluefile(self.vfs, 'badfile').read) +def testfirstline(self): +dw = {'key1': 'value1'} +scmutil.simplekeyvaluefile(self.vfs, 'fl').write(dw, firstline='1.0') +self.assertEqual(self.vfs.read('fl'), '1.0\nkey1=value1\n') +dr = scmutil.simplekeyvaluefile(self.vfs, 'fl')\ +.read(firstlinenonkeyval=True) +self.assertEqual(dr, {'__firstline': '1.0', 'key1': 'value1'}) + if __name__ == "__main__": silenttestrunner.main(__name__) ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 1 of 4 shelve-ext v5] scmutil: add simplekeyvaluefile reading test
# HG changeset patch # User Kostia Balytskyi # Date 1494517184 25200 # Thu May 11 08:39:44 2017 -0700 # Node ID ebf9affb2f7013a6a597ddd0765c786ed36f3722 # Parent 1ada3d18e7fbc9069910f2c036992d2f2b28e058 scmutil: add simplekeyvaluefile reading test Before this patch, mockvfs did not emulate readlines correctly and there was no test for simplekeyvaluefile reading. diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py --- a/mercurial/scmutil.py +++ b/mercurial/scmutil.py @@ -925,7 +925,10 @@ class simplekeyvaluefile(object): def read(self): lines = self.vfs.readlines(self.path) try: -d = dict(line[:-1].split('=', 1) for line in lines if line) +# the 'if line.strip()' part prevents us from failing on empty +# lines which only contain '\n' therefore are not skipped +# by 'if line' +d = dict(line[:-1].split('=', 1) for line in lines if line.strip()) except ValueError as e: raise error.CorruptedState(str(e)) return d diff --git a/tests/test-simplekeyvaluefile.py b/tests/test-simplekeyvaluefile.py --- a/tests/test-simplekeyvaluefile.py +++ b/tests/test-simplekeyvaluefile.py @@ -33,7 +33,8 @@ class mockvfs(object): return mockfile(path, self).read() def readlines(self, path): -return mockfile(path, self).read().split('\n') +# lines need to contain the trailing '\n' to mock the real readlines +return [l for l in mockfile(path, self).read().splitlines(True)] def __call__(self, path, mode, atomictemp): return mockfile(path, self) @@ -42,11 +43,13 @@ class testsimplekeyvaluefile(unittest.Te def setUp(self): self.vfs = mockvfs() -def testbasicwriting(self): -d = {'key1': 'value1', 'Key2': 'value2'} -scmutil.simplekeyvaluefile(self.vfs, 'kvfile').write(d) +def testbasicwritingiandreading(self): +dw = {'key1': 'value1', 'Key2': 'value2'} +scmutil.simplekeyvaluefile(self.vfs, 'kvfile').write(dw) self.assertEqual(sorted(self.vfs.read('kvfile').split('\n')), ['', 'Key2=value2', 'key1=value1']) +dr = scmutil.simplekeyvaluefile(self.vfs, 'kvfile').read() +self.assertEqual(dr, dw) def testinvalidkeys(self): d = {'0key1': 'value1', 'Key2': 'value2'} ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 4 of 4 shelve-ext v5] shelve: make shelvestate use simplekeyvaluefile
# HG changeset patch # User Kostia Balytskyi # Date 1494537058 25200 # Thu May 11 14:10:58 2017 -0700 # Node ID 5a494382891182deaddd2b18642d5c0320684bc4 # Parent 9170d8e0460bfe9ae4ea79394786cf2cac686fcb shelve: make shelvestate use simplekeyvaluefile Currently shelvestate uses line ordering to differentiate fields. This makes it hard for extensions to wrap shelve, since if two alternative versions of code add a new line, correct merging is going to be problematic. simplekeyvaluefile was introduced fot this purpose specifically. After this patch: - shelve will always write a simplekeyvaluefile - unshelve will check the first line of the file for a version, and if the version is 1, will read it in a position-based way, if the version is 2, will read it in a key-value way As discussed with Yuya previously, this will be able to handle old-style shelvedstate files, but old Mercurial versions will fail on the attempt to read shelvedstate file of version 2 with a self-explanatory message: 'abort: this version of shelve is incompatible with the version used in this repo' diff --git a/hgext/shelve.py b/hgext/shelve.py --- a/hgext/shelve.py +++ b/hgext/shelve.py @@ -167,7 +167,7 @@ class shelvedstate(object): Handles saving and restoring a shelved state. Ensures that different versions of a shelved state are possible and handles them appropriately. """ -_version = 1 +_version = 2 _filename = 'shelvedstate' _keep = 'keep' _nokeep = 'nokeep' @@ -175,26 +175,9 @@ class shelvedstate(object): _noactivebook = ':no-active-bookmark' @classmethod -def load(cls, repo): -# Order is important, because old shelvestate file uses it -# to detemine values of fields (i.g. version is on the first line, -# name is on the second and so forth). Please do not change. -keys = ['version', 'name', 'originalwctx', 'pendingctx', 'parents', -'nodestoremove', 'branchtorestore', 'keep', 'activebook'] -d = {} -fp = repo.vfs(cls._filename) +def _verifyandtransform(cls, d): +"""Some basic shelvestate syntactic verification and transformation""" try: -for key in keys: -d[key] = fp.readline().strip() -finally: -fp.close() - -# some basic syntactic verification and transformation -try: -d['version'] = int(d['version']) -if d['version'] != cls._version: -raise error.Abort(_('this version of shelve is incompatible ' -'with the version used in this repo')) d['originalwctx'] = nodemod.bin(d['originalwctx']) d['pendingctx'] = nodemod.bin(d['pendingctx']) d['parents'] = [nodemod.bin(h) @@ -204,6 +187,50 @@ class shelvedstate(object): except (ValueError, TypeError, KeyError) as err: raise error.CorruptedState(str(err)) +@classmethod +def _getversion(cls, repo): +"""Read version information from shelvestate file""" +fp = repo.vfs(cls._filename) +try: +version = int(fp.readline().strip()) +except ValueError as err: +raise error.CorruptedState(str(err)) +finally: +fp.close() +return version + +@classmethod +def _readold(cls, repo): +"""Read the old position-based version of a shelvestate file""" +# Order is important, because old shelvestate file uses it +# to detemine values of fields (i.g. name is on the second line, +# originalwctx is on the third and so forth). Please do not change. +keys = ['version', 'name', 'originalwctx', 'pendingctx', 'parents', +'nodestoremove', 'branchtorestore', 'keep', 'activebook'] +# this is executed only seldomly, so it is not a big deal +# that we open this file twice +fp = repo.vfs(cls._filename) +d = {} +try: +for key in keys: +d[key] = fp.readline().strip() +finally: +fp.close() +return d + +@classmethod +def load(cls, repo): +version = cls._getversion(repo) +if version < cls._version: +d = cls._readold(repo) +elif version == cls._version: +d = scmutil.simplekeyvaluefile(repo.vfs, cls._filename)\ + .read(firstlinenonkeyval=True) +else: +raise error.Abort(_('this version of shelve is incompatible ' +'with the version used in this repo')) + +cls._verifyandtransform(d) try: obj = cls() obj.name = d.get('name') @@ -224,19 +251,20 @@ class shelvedstate(object): @classmethod def save(cls, repo, name, originalwctx, pendingctx, nodestoremove, branchtorestore, keep=False, activebook=''): -fp = repo.vfs(cls._filename, 'wb') -
[PATCH 1 of 2] verify: always check rawsize
# HG changeset patch # User Jun Wu # Date 1494539522 25200 # Thu May 11 14:52:02 2017 -0700 # Node ID 1008583138d3dca114bc9d4998a27e845fbc13b0 # Parent 1ada3d18e7fbc9069910f2c036992d2f2b28e058 # Available At https://bitbucket.org/quark-zju/hg-draft # hg pull https://bitbucket.org/quark-zju/hg-draft -r 1008583138d3 verify: always check rawsize Previously, verify only checks "rawsize == len(rawtext)", if "len(fl.read()) != fl.size()". With flag processor, "len(fl.read()) != fl.size()" does not necessarily mean "rawsize == len(rawtext)". So we may miss a useful check. This patch removes the "if len(fl.read()) != fl.size()" condition so the rawsize check is always performed. With the condition removed, "fl.read(n)" looks unnecessary so a comment was added to explain the side effect is wanted. diff --git a/mercurial/verify.py b/mercurial/verify.py --- a/mercurial/verify.py +++ b/mercurial/verify.py @@ -428,11 +428,12 @@ class verifier(object): # use either "text" (external), or "rawtext" (in revlog). try: -l = len(fl.read(n)) +fl.read(n) # side effect: read content and do checkhash rp = fl.renamed(n) -if l != fl.size(i): -# the "L1 == L2" check -if len(fl.revision(n, raw=True)) != fl.rawsize(i): -self.err(lr, _("unpacked size is %s, %s expected") % - (l, fl.size(i)), f) +# the "L1 == L2" check +l1 = fl.rawsize(i) +l2 = len(fl.revision(n, raw=True)) +if l1 != l2: +self.err(lr, _("unpacked size is %s, %s expected") % + (l2, l1), f) except error.CensoredNodeError: # experimental config: censor.policy ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH 2 of 2] verify: add an option to skip checking transformed revisions
# HG changeset patch # User Jun Wu # Date 1494543792 25200 # Thu May 11 16:03:12 2017 -0700 # Node ID f63f6396ebda80cd0462aaedc4dc0d95fa5ca7b0 # Parent 1008583138d3dca114bc9d4998a27e845fbc13b0 # Available At https://bitbucket.org/quark-zju/hg-draft # hg pull https://bitbucket.org/quark-zju/hg-draft -r f63f6396ebda verify: add an option to skip checking transformed revisions Previously, "hg verify" verifies everything, which could be undesirable when there are expensive flag processor contents. This patch adds an "--skip-transform" flag to skip data transformation. That means we don't go through potentially expensive "read" flag processor and the check could be fast. Note: "renamed" is also skipped since it could trigger a transformation by default. In the LFS usecase, that means "hg verify --skip-transform" will not download all LFS blobs, which could be too large to be stored locally. diff --git a/mercurial/commands.py b/mercurial/commands.py --- a/mercurial/commands.py +++ b/mercurial/commands.py @@ -5444,6 +5444,8 @@ def update(ui, repo, node=None, rev=None updatecheck=updatecheck) -@command('verify', []) -def verify(ui, repo): +@command('verify', + [('', 'skip-transform', None, + _('do not check transformed file revisions (ADVANCED)'))]) +def verify(ui, repo, **opts): """verify the integrity of the repository @@ -5461,4 +5463,7 @@ def verify(ui, repo): Returns 0 on success, 1 if errors are encountered. """ +if opts['skip_transform']: +# internal config: verify.skiptransform +repo.ui.setconfig('verify', 'skiptransform', True) return hg.verify(repo) diff --git a/mercurial/verify.py b/mercurial/verify.py --- a/mercurial/verify.py +++ b/mercurial/verify.py @@ -50,4 +50,5 @@ class verifier(object): self.refersmf = False self.fncachewarned = False +self.skiptransform = repo.ui.configbool('verify', 'skiptransform') def warn(self, msg): @@ -428,6 +429,10 @@ class verifier(object): # use either "text" (external), or "rawtext" (in revlog). try: -fl.read(n) # side effect: read content and do checkhash -rp = fl.renamed(n) +skipread = self.skiptransform +if skipread and fl.flags(i) == 0: +skipread = False # no transform when flags is 0 +if not skipread: +fl.read(n) # side effect: read content and do checkhash +rp = fl.renamed(n) # the "L1 == L2" check l1 = fl.rawsize(i) diff --git a/tests/test-completion.t b/tests/test-completion.t --- a/tests/test-completion.t +++ b/tests/test-completion.t @@ -311,5 +311,5 @@ Show all commands + options tip: patch, git, style, template unbundle: update - verify: + verify: skip-transform version: template diff --git a/tests/test-help.t b/tests/test-help.t --- a/tests/test-help.t +++ b/tests/test-help.t @@ -507,4 +507,6 @@ Test command without options Returns 0 on success, 1 if errors are encountered. + options: + (some details hidden, use --verbose to show complete help) diff --git a/tests/test-verify.t b/tests/test-verify.t --- a/tests/test-verify.t +++ b/tests/test-verify.t @@ -318,2 +318,46 @@ test revlog format 0 1 files, 1 changesets, 1 total revisions $ cd .. + +test flag processor and --skip-transform + + $ hg init skip-transform + $ cd skip-transform + $ cat >> .hg/hgrc < [extensions] + > flagprocesor=$RUNTESTDIR/flagprocessorext.py + > EOF + $ echo '[BASE64]content' > base64 + $ hg commit -Aqm 'flag processor content' base64 + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 1 changesets, 1 total revisions + + $ cat >> $TESTTMP/break-base64.py < from __future__ import absolute_import + > import base64 + > base64.b64decode=lambda x: x + > EOF + $ cat >> .hg/hgrc < breakbase64=$TESTTMP/break-base64.py + > EOF + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + base64@0: unpacking 794ceecb: integrity check failed on data/base64.i:0 + 1 files, 1 changesets, 1 total revisions + 1 integrity errors encountered! + (first damaged changeset appears to be 0) + [1] + $ hg verify --skip-transform + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 1 changesets, 1 total revisions + ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH] merge: use repo.wvfs.setflags() instead of util.setflags()
# HG changeset patch # User Phil Cohen # Date 1494553123 25200 # Thu May 11 18:38:43 2017 -0700 # Node ID 90180de768113d9f6b1ec990a634ec4b872636a0 # Parent 8f1a2b848b52ea7bf3fe2404e3b62924c7aae93f merge: use repo.wvfs.setflags() instead of util.setflags() Most merge.py code goes through the vfs instead of maniulating files directly, so let's do the same here. diff --git a/mercurial/merge.py b/mercurial/merge.py --- a/mercurial/merge.py +++ b/mercurial/merge.py @@ -1288,7 +1288,7 @@ progress(_updating, z, item=f, total=numupdates, unit=_files) flags, = args audit(f) -util.setflags(repo.wjoin(f), 'l' in flags, 'x' in flags) +repo.wvfs.setflags(f, 'l' in flags, 'x' in flags) updated += 1 # the ordering is important here -- ms.mergedriver will raise if the merge ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH 1 of 2] sslutil: tweak the legacy [hostfingerprints] warning message
On Thu, 11 May 2017 03:02:42 -0400, Gregory Szorc wrote: # HG changeset patch # User Gregory Szorc # Date 1494485377 25200 # Wed May 10 23:49:37 2017 -0700 # Node ID fc01a88a85d64a3a440971c5e3b6c8f7db030170 # Parent 1ada3d18e7fbc9069910f2c036992d2f2b28e058 sslutil: tweak the legacy [hostfingerprints] warning message Lars Rohwedder noted in issue5559 that the previous wording was confusing. I agree. diff --git a/mercurial/sslutil.py b/mercurial/sslutil.py --- a/mercurial/sslutil.py +++ b/mercurial/sslutil.py @@ -820,13 +820,11 @@ def validatesocket(sock): if settings['legacyfingerprint']: ui.warn(_('(SHA-1 fingerprint for %s found in legacy ' '[hostfingerprints] section; ' - 'if you trust this fingerprint, set the ' - 'following config value in [hostsecurity] and ' - 'remove the old one from [hostfingerprints] ' - 'to upgrade to a more secure SHA-256 ' - 'fingerprint: ' - '%s.fingerprints=%s)\n') % ( - host, host, nicefingerprint)) + 'if you trust this fingerprint, remove the old ' + 'SHA-1 fingerprint from [hostfingerprints] and ' + 'add the following entry to the new ' + '[hostsecurity] section: %s.fingerprints=%s)\n') % +(host, host, nicefingerprint)) return # Pinned fingerprint didn't match. This is a fatal error. I'm guessing it's because stable hasn't been merged since 5559, but s/%s.fingerprints/%s:fingerprints/ ? ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Re: [PATCH] merge: use repo.wvfs.setflags() instead of util.setflags()
Phil Cohen writes: > # HG changeset patch > # User Phil Cohen > # Date 1494553123 25200 > # Thu May 11 18:38:43 2017 -0700 > # Node ID 90180de768113d9f6b1ec990a634ec4b872636a0 > # Parent 8f1a2b848b52ea7bf3fe2404e3b62924c7aae93f > merge: use repo.wvfs.setflags() instead of util.setflags() > > Most merge.py code goes through the vfs instead of maniulating > files directly, so let's do the same here. (already reviewed in person) This looks good to me! signature.asc Description: PGP signature ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[PATCH] extdiff: copy back execbit-only changes to the working directory
# HG changeset patch # User Matt Harbison # Date 1494556425 14400 # Thu May 11 22:33:45 2017 -0400 # Node ID 862a9d7605a23d7108e54b1a533dd3b091a6b12d # Parent b83aecac21cae0ea9e0256a00c744ca120799bc3 extdiff: copy back execbit-only changes to the working directory Some tools like BeyondCompare allow the file mode to be changed. The change was previously applied if the content of the file changed (either according to size or mtime), but was not being copied back for a mode-only change. That would seem to indicate handling this in an 'elif' branch, but I opted not to in order to avoid copying back the mode without the content changes when mtime and size are unchanged. (Yes, that's a rare corner case, but all the more reason not to have a subtle difference in behavior.) The only way I can think to handle this undetected change is to set each file in the non-wdir() snapshot to readonly, and check for that attribute (as well as mtime) when deciding to copy back. That would avoid the overhead of copying the whole file when only the mode changed. But a chmod in a diff tool is likely rare. See also affd753ddaf1. diff --git a/hgext/extdiff.py b/hgext/extdiff.py --- a/hgext/extdiff.py +++ b/hgext/extdiff.py @@ -280,7 +280,11 @@ # all changes. A size check will detect more cases, but not all. # The only certain way to detect every case is to diff all files, # which could be expensive. -if cpstat.st_mtime != st.st_mtime or cpstat.st_size != st.st_size: +# copyfile() carries over the permission, so the mode check could +# be in an 'elif' branch, but for the case where the file has +# changed without affecting mtime or size. +if (cpstat.st_mtime != st.st_mtime or cpstat.st_size != st.st_size +or (cpstat.st_mode & 0o100) != (st.st_mode & 0o100)): ui.debug('file changed while diffing. ' 'Overwriting: %s (src: %s)\n' % (working_fn, copy_fn)) util.copyfile(copy_fn, working_fn) diff --git a/tests/test-extdiff.t b/tests/test-extdiff.t --- a/tests/test-extdiff.t +++ b/tests/test-extdiff.t @@ -329,6 +329,7 @@ > # Mimic a tool that syncs all attrs, including mtime > cp $1/a $2/a > touch -r $1/a $2/a + > chmod +x $2/a > echo "** custom diff **" > EOF #if execbit @@ -366,6 +367,32 @@ $ cat a a +#if execbit + $ [ -x a ] + + $ cat > 'dir/tool.sh' << 'EOF' + > #!/bin/sh + > chmod -x $2/a + > echo "** custom diff **" + > EOF + + $ hg --debug tl --config extdiff.tl= --config merge-tools.tl.executable=$tool + making snapshot of 2 files from rev * (glob) +a +b + making snapshot of 2 files from working directory +a +b + running '$TESTTMP/a/dir/tool.sh a.* a' in */extdiff.* (glob) + ** custom diff ** + file changed while diffing. Overwriting: $TESTTMP/a/a (src: */extdiff.*/a/a) (glob) + cleaning up temp directory + [1] + + $ [ -x a ] + [1] +#endif + $ cd .. #if symlink ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel