D2887: filemerge: move temp file unlinks to _maketempfiles

[spectral (Kyle Lippincott)]

spectral created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2887

AFFECTED FILES
  mercurial/filemerge.py

CHANGE DETAILS

diff --git a/mercurial/filemerge.py b/mercurial/filemerge.py
--- a/mercurial/filemerge.py
+++ b/mercurial/filemerge.py
@@ -7,6 +7,7 @@
 
 from __future__ import absolute_import
 
+import contextlib
 import os
 import re
 import tempfile
@@ -510,8 +511,8 @@
 return False, 1, None
 unused, unused, unused, back = files
 localpath = _workingpath(repo, fcd)
-basepath, otherpath = _maketempfiles(repo, fco, fca)
-try:
+with _maketempfiles(repo, fco, fca) as temppaths:
+basepath, otherpath = temppaths
 outpath = ""
 mylabel, otherlabel = labels[:2]
 if len(labels) >= 3:
@@ -549,9 +550,6 @@
 r = ui.system(cmd, cwd=repo.root, environ=env, blockedtag='mergetool')
 repo.ui.debug('merge tool returned: %d\n' % r)
 return True, r, False
-finally:
-util.unlink(basepath)
-util.unlink(otherpath)
 
 def _formatconflictmarker(ctx, template, label, pad):
 """Applies the given template to the ctx, prefixed by the label.
@@ -665,6 +663,7 @@
 # the backup context regardless of where it lives.
 return context.arbitraryfilectx(back, repo=repo)
 
+@contextlib.contextmanager
 def _maketempfiles(repo, fco, fca):
 """Writes out `fco` and `fca` as temporary files, so an external merge
 tool may use them.
@@ -681,8 +680,11 @@
 
 b = temp("base", fca)
 c = temp("other", fco)
-
-return b, c
+try:
+yield b, c
+finally:
+util.unlink(b)
+util.unlink(c)
 
 def _filemerge(premerge, repo, wctx, mynode, orig, fcd, fco, fca, labels=None):
 """perform a 3-way merge in the working directory



To: spectral, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2886: filemerge: give some variables in _xmerge more descriptive names

[spectral (Kyle Lippincott)]

spectral created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2886

AFFECTED FILES
  mercurial/filemerge.py

CHANGE DETAILS

diff --git a/mercurial/filemerge.py b/mercurial/filemerge.py
--- a/mercurial/filemerge.py
+++ b/mercurial/filemerge.py
@@ -509,10 +509,10 @@
'for %s\n') % (tool, fcd.path()))
 return False, 1, None
 unused, unused, unused, back = files
-a = _workingpath(repo, fcd)
-b, c = _maketempfiles(repo, fco, fca)
+localpath = _workingpath(repo, fcd)
+basepath, otherpath = _maketempfiles(repo, fco, fca)
 try:
-out = ""
+outpath = ""
 mylabel, otherlabel = labels[:2]
 if len(labels) >= 3:
 baselabel = labels[2]
@@ -534,11 +534,11 @@
 args = _toolstr(ui, tool, "args")
 if "$output" in args:
 # read input from backup, write to original
-out = a
-a = repo.wvfs.join(back.path())
-replace = {'local': a, 'base': b, 'other': c, 'output': out,
-   'labellocal': mylabel, 'labelother': otherlabel,
-   'labelbase': baselabel}
+outpath = localpath
+localpath = repo.wvfs.join(back.path())
+replace = {'local': localpath, 'base': basepath, 'other': otherpath,
+   'output': outpath, 'labellocal': mylabel,
+   'labelother': otherlabel, 'labelbase': baselabel}
 args = util.interpolate(br'\$', replace, args,
 lambda s: util.shellquote(util.localpath(s)))
 cmd = toolpath + ' ' + args
@@ -550,8 +550,8 @@
 repo.ui.debug('merge tool returned: %d\n' % r)
 return True, r, False
 finally:
-util.unlink(b)
-util.unlink(c)
+util.unlink(basepath)
+util.unlink(otherpath)
 
 def _formatconflictmarker(ctx, template, label, pad):
 """Applies the given template to the ctx, prefixed by the label.



To: spectral, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2888: filemerge: use a single temp dir instead of temp files

[spectral (Kyle Lippincott)]

spectral created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  This can help to remove the clutter from UIs that display just the filenames;
  instead of seeing foo~local.C9ru9r.txt and foo~base.2DMV22.txt (in the /tmp
  directory on most platforms), we create a single new directory and use that,
  producing filenames like /tmp/hgmerge.C9ru9r/foo~local.txt.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2888

AFFECTED FILES
  mercurial/configitems.py
  mercurial/filemerge.py
  tests/test-merge-tools.t

CHANGE DETAILS

diff --git a/tests/test-merge-tools.t b/tests/test-merge-tools.t
--- a/tests/test-merge-tools.t
+++ b/tests/test-merge-tools.t
@@ -1363,6 +1363,33 @@
   (branch merge, don't forget to commit)
   $ rm -f 'printargs_merge_tool'
 
+Same test with experimental.mergetempdirprefix set:
+
+  $ beforemerge
+  [merge-tools]
+  false.whatever=
+  true.priority=1
+  true.executable=cat
+  # hg update -C 1
+  $ cat < printargs_merge_tool
+  > while test \$# -gt 0; do echo arg: \"\$1\"; shift; done
+  > EOF
+  $ hg --config experimental.mergetempdirprefix=$TESTTMP/hgmerge. \
+  >--config merge-tools.true.executable='sh' \
+  >--config merge-tools.true.args='./printargs_merge_tool ll:$labellocal 
lo: $labelother lb:$labelbase": "$base' \
+  >--config merge-tools.true.mergemarkertemplate='tooltmpl {short(node)}' \
+  >--config ui.mergemarkertemplate='uitmpl {rev}' \
+  >--config ui.mergemarkers=detailed \
+  >merge -r 2
+  merging f
+  arg: "ll:working copy"
+  arg: "lo:"
+  arg: "merge rev"
+  arg: "lb:base: $TESTTMP/hgmerge.??/f~base" (glob)
+  0 files updated, 1 files merged, 0 files removed, 0 files unresolved
+  (branch merge, don't forget to commit)
+  $ rm -f 'printargs_merge_tool'
+
 Merge using a tool that supports labellocal, labelother, and labelbase, 
checking
 that they're quoted properly as well. This is using 'detailed' mergemarkers,
 even though ui.mergemarkers is 'basic', and using the tool's
@@ -1562,6 +1589,20 @@
   0 files updated, 1 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 
+Verify naming of temporary files and that extension is preserved
+(experimental.mergetempdirprefix version):
+
+  $ hg update -q -C 1
+  $ hg mv f f.txt
+  $ hg ci -qm "f.txt"
+  $ hg update -q -C 2
+  $ hg merge -y -r tip --tool echo \
+  >--config merge-tools.echo.args='$base $local $other $output' \
+  >--config experimental.mergetempdirprefix=$TESTTMP/hgmerge.
+  merging f and f.txt to f.txt
+  $TESTTMP/hgmerge.??/f~base $TESTTMP/f.txt.orig 
$TESTTMP/hgmerge.??/f~other.txt $TESTTMP/f.txt (glob)
+  0 files updated, 1 files merged, 0 files removed, 0 files unresolved
+  (branch merge, don't forget to commit)
 Check that debugpicktool examines which merge tool is chosen for
 specified file as expected
 
diff --git a/mercurial/filemerge.py b/mercurial/filemerge.py
--- a/mercurial/filemerge.py
+++ b/mercurial/filemerge.py
@@ -10,6 +10,7 @@
 import contextlib
 import os
 import re
+import shutil
 import tempfile
 
 from .i18n import _
@@ -668,12 +669,23 @@
 """Writes out `fco` and `fca` as temporary files, so an external merge
 tool may use them.
 """
+tmproot = None
+tmprootprefix = repo.ui.config('experimental', 'mergetempdirprefix')
+if tmprootprefix:
+tmproot = tempfile.mkdtemp(prefix=tmprootprefix)
+
 def temp(prefix, ctx):
 fullbase, ext = os.path.splitext(ctx.path())
-pre = "%s~%s." % (os.path.basename(fullbase), prefix)
-(fd, name) = tempfile.mkstemp(prefix=pre, suffix=ext)
+pre = "%s~%s" % (os.path.basename(fullbase), prefix)
+if tmproot:
+name = os.path.join(tmproot, pre)
+if ext:
+name += ext
+f = open(name, r"wb")
+else:
+(fd, name) = tempfile.mkstemp(prefix=pre + '.', suffix=ext)
+f = os.fdopen(fd, r"wb")
 data = repo.wwritedata(ctx.path(), ctx.data())
-f = os.fdopen(fd, r"wb")
 f.write(data)
 f.close()
 return name
@@ -683,8 +695,11 @@
 try:
 yield b, c
 finally:
-util.unlink(b)
-util.unlink(c)
+if tmproot:
+shutil.rmtree(tmproot)
+else:
+util.unlink(b)
+util.unlink(c)
 
 def _filemerge(premerge, repo, wctx, mynode, orig, fcd, fco, fca, labels=None):
 """perform a 3-way merge in the working directory
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -502,6 +502,9 @@
 coreconfigitem('experimental', 'maxdeltachainspan',
 default=-1,
 )
+coreconfigitem('experimental', 'mergetempdirprefix',
+default=None,
+)
 coreconfigitem('experimental', 'mmapindexthreshold',
 default=None,
 )



To: spectral, #hg-reviewers
Cc: mercurial-devel
__

D2889: filemerge: make the 'local' path match the format that 'base' and 'other' use

[spectral (Kyle Lippincott)]

spectral created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  If we pass a separate '$output' arg to the merge tool, we produce four files:
  local, base, other, and output.  In this situation, 'output' will be the
  original filename, 'base' and 'other' are temporary files, and previously
  'local' would be the backup file (so if 'output' was foo.txt, 'local' would be
  foo.txt.orig).
  
  This change makes it so that 'local' follows the same pattern as 'base' and
  'other' - it will be a temporary file either in the
  `experimental.mergetempdirprefix`-controlled directory with a name like
  foo~local.txt, or in the normal system-wide temp dir with a name like
  foo~local.RaNd0m.txt.
  
  For the cases where the merge tool does not use an '$output' arg, 'local' is
  still the destination filename, and 'base' and 'other' are unchanged.
  
  The hope is that this is much easier for people to reason about; rather than
  having a tool like Meld pop up with three panes, one of them with the filename
  "foo.txt.orig", one with the filename "foo.txt", and one with
  "foo~other.StuFf2.txt", we can (when the merge temp dir stuff is enabled) make
  it show up as "foo~local.txt", "foo.txt" and "foo~other.txt", respectively.
  
  This also opens the door to future customization, such as getting the
  operation-provided labels and a hash prefix into the filenames (so we see
  something like "foo~dest.abc123", "foo.txt", and "foo~src.d4e5f6").

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2889

AFFECTED FILES
  mercurial/filemerge.py
  tests/test-merge-tools.t

CHANGE DETAILS

diff --git a/tests/test-merge-tools.t b/tests/test-merge-tools.t
--- a/tests/test-merge-tools.t
+++ b/tests/test-merge-tools.t
@@ -1585,7 +1585,7 @@
   $ hg update -q -C 2
   $ hg merge -y -r tip --tool echo --config merge-tools.echo.args='$base 
$local $other $output'
   merging f and f.txt to f.txt
-  */f~base.?? $TESTTMP/f.txt.orig */f~other.??.txt $TESTTMP/f.txt 
(glob)
+  */f~base.?? */f~local.??.txt */f~other.??.txt $TESTTMP/f.txt 
(glob)
   0 files updated, 1 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 
@@ -1600,7 +1600,7 @@
   >--config merge-tools.echo.args='$base $local $other $output' \
   >--config experimental.mergetempdirprefix=$TESTTMP/hgmerge.
   merging f and f.txt to f.txt
-  $TESTTMP/hgmerge.??/f~base $TESTTMP/f.txt.orig 
$TESTTMP/hgmerge.??/f~other.txt $TESTTMP/f.txt (glob)
+  $TESTTMP/hgmerge.??/f~base $TESTTMP/hgmerge.??/f~local.txt 
$TESTTMP/hgmerge.??/f~other.txt $TESTTMP/f.txt (glob)
   0 files updated, 1 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 Check that debugpicktool examines which merge tool is chosen for
diff --git a/mercurial/filemerge.py b/mercurial/filemerge.py
--- a/mercurial/filemerge.py
+++ b/mercurial/filemerge.py
@@ -512,8 +512,11 @@
 return False, 1, None
 unused, unused, unused, back = files
 localpath = _workingpath(repo, fcd)
-with _maketempfiles(repo, fco, fca) as temppaths:
-basepath, otherpath = temppaths
+args = _toolstr(repo.ui, tool, "args")
+
+with _maketempfiles(repo, fco, fca, repo.wvfs.join(back.path()),
+"$output" in args) as temppaths:
+basepath, otherpath, localoutputpath = temppaths
 outpath = ""
 mylabel, otherlabel = labels[:2]
 if len(labels) >= 3:
@@ -533,11 +536,10 @@
}
 ui = repo.ui
 
-args = _toolstr(ui, tool, "args")
 if "$output" in args:
 # read input from backup, write to original
 outpath = localpath
-localpath = repo.wvfs.join(back.path())
+localpath = localoutputpath
 replace = {'local': localpath, 'base': basepath, 'other': otherpath,
'output': outpath, 'labellocal': mylabel,
'labelother': otherlabel, 'labelbase': baselabel}
@@ -665,41 +667,59 @@
 return context.arbitraryfilectx(back, repo=repo)
 
 @contextlib.contextmanager
-def _maketempfiles(repo, fco, fca):
-"""Writes out `fco` and `fca` as temporary files, so an external merge
-tool may use them.
+def _maketempfiles(repo, fco, fca, localpath, uselocalpath):
+"""Writes out `fco` and `fca` as temporary files, and (if not None) copies
+`localpath` to another temporary file, so an external merge tool may use
+them.
 """
 tmproot = None
 tmprootprefix = repo.ui.config('experimental', 'mergetempdirprefix')
 if tmprootprefix:
 tmproot = tempfile.mkdtemp(prefix=tmprootprefix)
 
-def temp(prefix, ctx):
-fullbase, ext = os.path.splitext(ctx.path())
+def maketempfrompath(prefix, path):
+fullbase, ext = os.path.splitext(path)
 pre = "%s~%s" % (os.path.basename(fullbase), prefix

[Bug 5823] New: hg evolve should return 0 status code when it doesn't find any problems to fix

[mercurial-bugs]

https://bz.mercurial-scm.org/show_bug.cgi?id=5823

Bug ID: 5823
   Summary: hg evolve should return 0 status code when it doesn't
find any problems to fix
   Product: Mercurial
   Version: 4.5
  Hardware: PC
OS: Linux
Status: UNCONFIRMED
  Severity: feature
  Priority: wish
 Component: evolution
  Assignee: bugzi...@mercurial-scm.org
  Reporter: mta...@google.com
CC: mercurial-devel@mercurial-scm.org,
pierre-yves.da...@ens-lyon.org

repro:
1. run `hg evolve` on a repo with no trouble/instability/anything wrong

expected:
it returns with status code 0

actual:
it prints `no troubled changesets` and returns status code 1, seeming to
indicate failure, but really this is not a failure since there was no work to
do in the first place

-- 
You are receiving this mail because:
You are on the CC list for the bug.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2856: wireproto: nominally don't expose "batch" to version 2 wire transports

[indygreg (Gregory Szorc)]

indygreg updated this revision to Diff 7083.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D2856?vs=7049&id=7083

REVISION DETAIL
  https://phab.mercurial-scm.org/D2856

AFFECTED FILES
  mercurial/wireproto.py
  mercurial/wireprotoserver.py
  tests/test-debugcommands.t
  tests/test-hgweb-commands.t
  tests/test-http-bad-server.t
  tests/test-http-protocol.t
  tests/test-ssh-bundle1.t
  tests/test-ssh-proto-unbundle.t
  tests/test-ssh-proto.t
  tests/test-ssh.t

CHANGE DETAILS

diff --git a/tests/test-ssh.t b/tests/test-ssh.t
--- a/tests/test-ssh.t
+++ b/tests/test-ssh.t
@@ -498,7 +498,7 @@
   sending between command
   remote: 403 (sshv1 !)
   protocol upgraded to exp-ssh-v2-0001 (sshv2 !)
-  remote: capabilities: lookup branchmap pushkey known getbundle unbundlehash 
batch changegroupsubset streamreqs=generaldelta,revlogv1 
$USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  remote: capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch
   remote: 1 (sshv1 !)
   query 1; heads
   devel-peer-request: batched-content
diff --git a/tests/test-ssh-proto.t b/tests/test-ssh-proto.t
--- a/tests/test-ssh-proto.t
+++ b/tests/test-ssh-proto.t
@@ -64,7 +64,7 @@
   devel-peer-request:   pairs: 81 bytes
   sending between command
   remote: 403
-  remote: capabilities: lookup branchmap pushkey known getbundle unbundlehash 
batch changegroupsubset streamreqs=generaldelta,revlogv1 
$USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  remote: capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch
   remote: 1
   url: ssh://user@dummy/server
   local: no
@@ -84,16 +84,16 @@
   o> readline() -> 4:
   o> 403\n
   o> readline() -> 403:
-  o> capabilities: lookup branchmap pushkey known getbundle unbundlehash 
batch changegroupsubset streamreqs=generaldelta,revlogv1 
$USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN\n
+  o> capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch\n
 
 `hg debugserve --sshstdio` works
 
   $ cd server
   $ hg debugserve --sshstdio << EOF
   > hello
   > EOF
   403
-  capabilities: lookup branchmap pushkey known getbundle unbundlehash batch 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN
+  capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch
 
 I/O logging works
 
@@ -103,22 +103,22 @@
   o> write(4) -> 4:
   o> 403\n
   o> write(403) -> 403:
-  o> capabilities: lookup branchmap pushkey known getbundle unbundlehash 
batch changegroupsubset streamreqs=generaldelta,revlogv1 
$USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN\n
+  o> capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch\n
   403
-  capabilities: lookup branchmap pushkey known getbundle unbundlehash batch 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN
+  capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch
   o> flush() -> None
 
   $ hg debugserve --sshstdio --logiofile $TESTTMP/io << EOF
   > hello
   > EOF
   403
-  capabilities: lookup branchmap pushkey known getbundle unbundlehash batch 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN
+  capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch
 
   $ cat $TESTTMP/io
   o> write(4) -> 4:
   o> 403\n
   o> write(403) -> 403:
-  o> capabilities: lookup branchmap pushkey known getbundle unbundlehash 
batch changegroupsubset streamreqs=generaldelta,revlogv1 
$USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN\n
+  o> capabilities: lookup branchmap pushkey known getbundle unbundlehash 
changegroupsubset streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ 
unbundle=HG10GZ,HG10BZ,HG10UN batch\n
   o> flush() -> None
 
   $ cd ..
@@ -145,7 +145,7 @@
   o> readline() -> 4:
   o> 403\n
   o> readline() -> 403:
-  o> capabilities: lookup branchmap pushkey known getbundle unbundlehash 
batch changegroupsubset streamreqs=generaldelta,revlogv1 
$USUAL_BUNDLE2_CAPS_SERVER$ unbundle=H

D2885: RFC: use Redis to cache file data

[indygreg (Gregory Szorc)]

indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  This [very hacky] commit implements a new files store that uses
  Redis to cache files data. It services requests from Redis (if
  available) and falls back to a base store / interface (revlogs in
  our case) on misses.
  
  The purpose of this commit is to first demonstrate the value in having
  interfaces for storage. If we code to the interface, then another
  interface can come along and do useful things - like caching.
  
  The other purpose was to investigate performance. Would a memory-backed
  key-value store have a significant impact on performance of our
  experimental wire protocol command to serve file data fulltexts for
  a specific revisions? The answer is a very resounding yet!
  
  Using the same mozilla-unified revision from the previous commit:
  
  - no compression: 1478MB;  ~94s   wall; ~56s   CPU w/ hot redis: 1478MB;   
~9.6s wall;  ~8.6s CPU
  - zstd level 3:343MB;  ~97s   wall; ~57s   CPU w/ hot redis:  343MB;   
~8.5s wall;  ~8.3s CPU
  - zstd level 1 w/ hot redis:  377MB;   ~6.8s wall;  ~6.6s CPU
  - zlib level 6:367MB; ~116s   wall; ~74s   CPU w/ hot redis:  367MB;  
~36.7s wall; ~36s   CPU
  
  For the curious, the ls profiler says that our hotspot without
  compression is in socket I/O. With zstd compression, the hotspot is
  compression.
  
  I reckon the reason for the socket I/O overhead is because we end up
  writing tons more chunks on the wire when uncompressed (compression
  will effectively ensure each output chunk is a similar, large'ish
  size). All those extra Python function calls and system calls do add
  up!
  
  Anyway, I'm definitely happy with the performance improvements. I'd
  say this was a useful experiment!

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2885

AFFECTED FILES
  mercurial/localrepo.py
  mercurial/revlogstore.py

CHANGE DETAILS

diff --git a/mercurial/revlogstore.py b/mercurial/revlogstore.py
--- a/mercurial/revlogstore.py
+++ b/mercurial/revlogstore.py
@@ -35,3 +35,51 @@
 
 data = fl.read(node)
 yield 'ok', path, node, data
+
+def redisfiledatakey(path, node):
+return b'filedata:%s:%s' % (path, node)
+
+class redisacceleratedrevlogfilesstore(repository.basefilesstore):
+A filesstore that can use a redis server to speed up operations."""
+def __init__(self, redis, basestore):
+self._redis = redis
+self._basestore = basestore
+
+def resolvefilesdata(self, entries):
+# Our strategy is to batch requests to redis because this is faster
+# than a command for every entry.
+
+batch = []
+for i, entry in enumerate(entries):
+batch.append(entry)
+
+if i and not i % 1000:
+for res in self._processfiledatabatch(batch):
+yield res
+
+batch = []
+
+if batch:
+for res in self._processfiledatabatch(batch):
+yield res
+
+def _processfiledatabatch(self, batch):
+keys = [redisfiledatakey(path, node) for path, node in batch]
+
+missing = []
+
+for i, redisdata in enumerate(self._redis.mget(keys)):
+path, node = batch[i]
+
+if redisdata is None:
+missing.append((path, node))
+else:
+yield 'ok', path, node, redisdata
+
+# Now resolve all the missing data from the base store.
+for res, path, node, data in self._basestore.resolvefilesdata(missing):
+yield res, path, node, data
+
+# Don't forget to cache it!
+if res == 'ok':
+self._redis.set(redisfiledatakey(path, node), data)
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -480,7 +480,11 @@
 else: # standard vfs
 self.svfs.audit = self._getsvfsward(self.svfs.audit)
 self._applyopenerreqs()
-self.filesstore = revlogstore.revlogfilesstore(self.svfs)
+import redis
+basefilesstore = revlogstore.revlogfilesstore(self.svfs)
+redisconn = redis.StrictRedis(host='localhost', port=6379, db=0)
+self.filesstore = revlogstore.redisacceleratedrevlogfilesstore(
+redisconn, basefilesstore)
 
 if create:
 self._writerequirements()



To: indygreg, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2883: revlogstore: create and implement an interface for repo files storage

[indygreg (Gregory Szorc)]

indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  In order to better support partial clones, we will need to overhaul
  local repository storage. This will be a major effort, as many parts
  of the code assume things like the existence of revlogs for storing
  data.
  
  To help support alternate storage implementations, we will create
  interfaces for accessing storage. The idea is that consumers will
  all code to an interface and any new interface-conforming
  implementation can come along and be swapped in to provide new and
  novel storage mechanisms.
  
  This commit starts the process of defining those interfaces.
  
  We define an interface for accessing files data. It has a single
  method for resolving the fulltext of an iterable of inputs.
  
  The interface is specifically defined to allow out-of-order responses.
  It also provides a mechanism for declaring that files data is censored.
  We *may* also want a mechanism to declare LFS or largefiles data.
  But I'm not sure how that mechanism works or what the best way to
  handle that would be, if any.
  
  We introduce a new "revlogstore" module to hold the definitions of
  these interfaces that use our existing revlog-based storage
  mechanism.
  
  An attribute pointing to the "files store" has been added to
  localrepository.
  
  No consumers of the new interface have been added. The interface
  should still be considered highly experimental and details are
  expected to change.
  
  It was tempting to define the interface as one level higher than
  file storage - in such a way to facilitate accessing changeset
  and manifest data as well. However, I believe these 3 primitives -
  changesets, manifests, and files - each have unique requirements
  that will dictate special, one-off methods on their storage
  interfaces. I'd rather we define our interfaces so they are
  tailored to each type initially. If an implementation wants to
  shoehorn all data into generic key-value blog store, they can
  still do that. And we also reserve the right to combine interfaces
  in the future. I just think that attempting to have the initial
  versions of the interfaces deviate too far from current reality will
  make it very challenging to define and implement them.
  
  The reason I'm defining and implementing this interface now is to
  support new (experimental) wire protocol commands to be used to
  support partial clone. Some of these commands will benefit from
  aggressive caching. I want to prove out the efficacy of the interfaces
  approach by implementing cache-based speedups in the interface layer.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2883

AFFECTED FILES
  mercurial/localrepo.py
  mercurial/repository.py
  mercurial/revlogstore.py

CHANGE DETAILS

diff --git a/mercurial/revlogstore.py b/mercurial/revlogstore.py
new file mode 100644
--- /dev/null
+++ b/mercurial/revlogstore.py
@@ -0,0 +1,37 @@
+# revlogstore.py - storage interface for repositories using revlog storage
+#
+# Copyright 2018 Gregory Szorc 
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+from . import (
+error,
+filelog,
+repository,
+)
+
+class revlogfilesstore(repository.basefilesstore):
+"""Files storage layer using revlogs for files storage."""
+
+def __init__(self, svfs):
+self._svfs = svfs
+
+def resolvefilesdata(self, entries):
+for path, node in entries:
+fl = filelog.filelog(self._svfs, path)
+
+try:
+rev = fl.rev(node)
+except error.LookupError:
+yield 'missing', path, node, None
+continue
+
+if fl.iscensored(rev):
+yield 'censored', path, node, None
+continue
+
+data = fl.read(node)
+yield 'ok', path, node, data
diff --git a/mercurial/repository.py b/mercurial/repository.py
--- a/mercurial/repository.py
+++ b/mercurial/repository.py
@@ -266,3 +266,33 @@
 
 class legacypeer(peer, _baselegacywirecommands):
 """peer but with support for legacy wire protocol commands."""
+
+class basefilesstore(object):
+"""Storage interface for repository files data.
+
+This interface defines mechanisms to access repository files data in a
+storage agnostic manner. The goal of this interface is to abstract storage
+implementations so implementation details of storage don't leak into
+higher-level repository consumers.
+"""
+
+__metaclass__ = abc.ABCMeta
+
+def resolvefilesdata(self, entries):
+"""Resolve the fulltext data for an iterable of files.
+
+Each entry is defined by a 2-tuple of (path, node).
+
+The method is a generator that emits results as they become available.
+Each 

D2884: wireproto: experimental command to emit file data

[indygreg (Gregory Szorc)]

indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  Partial clones will require new wire protocol functionality to
  retrieve repository data. The remotefilelog extensions - which
  implements various aspects of partial clone - adds a handful of
  wire protocol commands:
  
  getflogheads
  
Obtain heads of a filelog
  
  getfile
  
Obtain data for an individual file revision
  
  getfiles
  
Batch version of getfile
  
  getpackv1
  
Obtain a "pack file" containing index and data on multiple
files
  
  (among others)
  
  Recently, the wire protocol has gained support for "obtain repository
  data" in the form of overloading the "getbundle" wire protocol
  command. This is arguaby OK in the context of "all data is attached
  to bundles" and "bundles are a self-contained representation of
  complete repository data." But partial clone invalidates these
  assumptions because in a partial clone world, we no longer can assume
  things like "the client has all the base revisions."
  
  In a partial clone world, we'll need wire protocol commands that allow
  clients to obtain specific pieces of data with vastly different
  access patterns. For example, a client may want to obtain "index"
  data but keep the fulltext data on the server. Or vice-versa. Or a
  client may wish to fetch all revisions of a specific file but only
  the latest revision of another. These access patterns will be
  difficult to shoehorn into single, powerful commands (like
  "getbundle"). Even if we could, doing that isn't wise from a server
  implementation perspective because it makes implementing scalable
  servers hard. We want server-side commands to be small and simple
  so alternate server implementations can come into existence more
  easily.
  
  This is one reason why the frame-based wire protocol I'm implementing
  supports command pipelining and out-of-order responses. This
  property will enable clients performing complex operations to send
  command streams containing dozens or even hundreds of small command
  requests to servers.
  
  Anyway, this commit implements an experimental wire protocol command
  for "get files data." Essentially, you give it a changeset revision
  you are interested in and it spits back all the files and their data
  in that revision, as fulltexts.
  
  This command is just one way a server could emit data for files.
  A variation of this command that accepts specific file paths and nodes
  whose data is to be retrieved would also be useful. And I imagine we'll
  eventually implement that. It would also be useful to emit index
  data. Or have each file blob be individually compressed. (Right now
  compression is performed on the whole stream because that's how the
  wire protocol currently works - but I have plans to evolve the frame
  based protocol to do new and novel things here.)
  
  I'm not even sure this variation of the wire protocol command is a
  good one to have! One reason I want to start with this command is
  that it seems like a useful primitive. For example, with this
  command, one could build a client that is able to realize a working
  directory from a single wire protocol request: you can literally
  stream the response to this command and turn the data into files on
  the filesystem with minimal stream processing!
  
  As implemented, this command is effectively a benchmark of revlog
  reading and/or compression. On the mozilla-unified repository when
  operating on revision c488b8d0e074efb490ebca32db68eb77871bfd2f (a
  recent revision of mozilla-central, the head of Firefox development),
  my i7-6700K yields the following:
  
  - no compression: 1478MB;  ~94s wall; ~56s CPU
  - zstd level 3:343MB;  ~97s wall; ~57s CPU
  - zlib level 6:367MB; ~116s wall; ~74s CPU
  
  For comparison, `hg bundle --base null -r c488b8d0e0 -t zstd-v2`
  (which approximates what `hg clone -r` would be doing on the server)
  yields:
  
1397MB; ~624s wall; ~225s CPU
  
  Of course, these are vastly different operations. But this does
  demonstrate that if your use case of version control is "check out
  revision X" and you were previously relying on `hg clone` [without
  stream clone bundles] to do that, this wire protocol command
  is overall much more efficient on servers. It's worth noting that
  the use case of version control for many automated systems *is*
  "check out revision X." So I think providing a clone mode that can
  realize a working copy as fast as possible is a worthwhile feature
  to have!

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2884

AFFECTED FILES
  mercurial/configitems.py
  mercurial/help/internals/wireprotocol.txt
  mercurial/wireproto.py
  tests/test-wireproto-revsfiledata.t

CHANGE DETAILS

diff --git a/tests/test-wireproto-revsfiledata.t 
b/tests/test-wireproto-revsfiledata.t
new file mode 100644
--- /dev/null
+++ b/tests/

D2882: hgweb: convert an assert to a ProgrammingError

[indygreg (Gregory Szorc)]

indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  Because assert may get optimized away.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2882

AFFECTED FILES
  mercurial/hgweb/webcommands.py

CHANGE DETAILS

diff --git a/mercurial/hgweb/webcommands.py b/mercurial/hgweb/webcommands.py
--- a/mercurial/hgweb/webcommands.py
+++ b/mercurial/hgweb/webcommands.py
@@ -1195,7 +1195,9 @@
 web.res.headers['Content-Encoding'] = encoding
 
 web.res.setbodywillwrite()
-assert list(web.res.sendresponse()) == []
+if list(web.res.sendresponse()):
+raise error.ProgrammingError('sendresponse() should not emit data '
+ 'if writing later')
 
 bodyfh = web.res.getbodyfile()
 



To: indygreg, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

mercurial@36885: 10 new changesets

[Mercurial Commits]

10 new changesets in mercurial:

https://www.mercurial-scm.org/repo/hg/rev/97f44b0720e2
changeset:   36876:97f44b0720e2
user:Gregory Szorc 
date:Sat Mar 10 20:16:20 2018 -0800
summary: hgweb: port archive command to modern response API

https://www.mercurial-scm.org/repo/hg/rev/02bea04b4c54
changeset:   36877:02bea04b4c54
user:Gregory Szorc 
date:Sat Mar 10 18:19:27 2018 -0800
summary: hgweb: transition permissions hooks to modern request type (API)

https://www.mercurial-scm.org/repo/hg/rev/ccb70a77f746
changeset:   36878:ccb70a77f746
user:Gregory Szorc 
date:Sat Mar 10 18:42:00 2018 -0800
summary: hgweb: refactor 304 handling code

https://www.mercurial-scm.org/repo/hg/rev/9675147aec06
changeset:   36879:9675147aec06
user:Gregory Szorc 
date:Sat Mar 10 18:51:32 2018 -0800
summary: hgweb: send errors using new response API

https://www.mercurial-scm.org/repo/hg/rev/67fb0dca29bc
changeset:   36880:67fb0dca29bc
user:Gregory Szorc 
date:Sat Mar 10 20:35:35 2018 -0800
summary: hgweb: always return iterable from @webcommand functions (API)

https://www.mercurial-scm.org/repo/hg/rev/96a93625a824
changeset:   36881:96a93625a824
user:Gregory Szorc 
date:Sat Mar 10 19:08:58 2018 -0800
summary: hgweb: stop setting headers on wsgirequest

https://www.mercurial-scm.org/repo/hg/rev/66f62d120ba2
changeset:   36882:66f62d120ba2
user:Gregory Szorc 
date:Sat Mar 10 19:11:41 2018 -0800
summary: hgweb: use web.req instead of req.req

https://www.mercurial-scm.org/repo/hg/rev/061635d4221c
changeset:   36883:061635d4221c
user:Gregory Szorc 
date:Sat Mar 10 19:41:18 2018 -0800
summary: hgweb: add a sendtemplate() helper function

https://www.mercurial-scm.org/repo/hg/rev/ece242db5000
changeset:   36884:ece242db5000
user:Gregory Szorc 
date:Sat Mar 10 20:38:28 2018 -0800
summary: hgweb: use templater on requestcontext instance

https://www.mercurial-scm.org/repo/hg/rev/c68e79dcf21c
changeset:   36885:c68e79dcf21c
bookmark:@
tag: tip
user:Gregory Szorc 
date:Sat Mar 10 19:46:54 2018 -0800
summary: hgweb: don't redundantly pass templater with requestcontext (API)

-- 
Repository URL: https://www.mercurial-scm.org/repo/hg
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2880: bundle: add the possibility to bundle bookmarks (issue5792)

[martinvonz (Martin von Zweigbergk)]

martinvonz added inline comments.

INLINE COMMENTS

> test-bundle-bookmarks.t:37-63
> +Bookmarks are restored when unbundling
> +  $ hg bundle --all bundle
> +  5 changesets found
> +  $ hg debugbundle bundle
> +  Stream params: {Compression: BZ}
> +  changegroup -- {nbchanges: 5, version: 02}
> +  426bada5c67598ca65036d57d9e4b64b0c1ce7a0

Can we have a similar test case where we create divergence? Create a fork in 
the graph in the debugdrawdag call above. Let's say you have commit F that 
branches off of B, then do something like this:

  $ hg bundle --all bundle
  $ hg strip --no-backup C
  $ hg bookmarks -f -r F D1
  $ hg unbundle -q bundle
  $ hg log -G -T '{desc} {bookmarks}\n'
o  E
|
o  D D1 D2
|
o  C
|
|  o F D1@1
|/
o  B
|
o  A A1

I don't know if "@1" is the appropriate divergence marker, but I can't think of 
a better one. I haven't even looked at your code to try to guess what it would 
be in practice (if it would work at all).

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2880

To: lothiraldan, #hg-reviewers
Cc: martinvonz, indygreg, pulkit, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2881: hgweb: refactor multirequest to be a dict of lists

[indygreg (Gregory Szorc)]

indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  ... instead of a list of 2-tuples.
  
  This makes key lookups faster. The only downside is we lose total
  ordering of all entries. But we weren't relying on that before, so
  it's no loss.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2881

AFFECTED FILES
  mercurial/hgweb/request.py

CHANGE DETAILS

diff --git a/mercurial/hgweb/request.py b/mercurial/hgweb/request.py
--- a/mercurial/hgweb/request.py
+++ b/mercurial/hgweb/request.py
@@ -31,36 +31,22 @@
 # Stores (key, value) 2-tuples. This isn't the most efficient. But we
 # don't rely on parameters that much, so it shouldn't be a perf issue.
 # we can always add dict for fast lookups.
-self._items = []
+self._items = {}
 
 def __getitem__(self, key):
 """Returns the last set value for a key."""
-for k, v in reversed(self._items):
-if k == key:
-return v
-
-raise KeyError(key)
+return self._items[key][-1]
 
 def __setitem__(self, key, value):
 """Replace a values for a key with a new value."""
-try:
-del self[key]
-except KeyError:
-pass
-
-self._items.append((key, value))
+self._items[key] = [value]
 
 def __delitem__(self, key):
 """Delete all values for a key."""
-oldlen = len(self._items)
-
-self._items[:] = [(k, v) for k, v in self._items if k != key]
-
-if oldlen == len(self._items):
-raise KeyError(key)
+del self._items[key]
 
 def __contains__(self, key):
-return any(k == key for k, v in self._items)
+return key in self._items
 
 def __len__(self):
 return len(self._items)
@@ -73,36 +59,26 @@
 
 def add(self, key, value):
 """Add a new value for a key. Does not replace existing values."""
-self._items.append((key, value))
+self._items.setdefault(key, []).append(value)
 
 def getall(self, key):
 """Obtains all values for a key."""
-return [v for k, v in self._items if k == key]
+return self._items.get(key, [])
 
 def getone(self, key):
 """Obtain a single value for a key.
 
 Raises KeyError if key not defined or it has multiple values set.
 """
-vals = self.getall(key)
-
-if not vals:
-raise KeyError(key)
+vals = self._items[key]
 
 if len(vals) > 1:
 raise KeyError('multiple values for %r' % key)
 
 return vals[0]
 
 def asdictoflists(self):
-d = {}
-for k, v in self._items:
-if k in d:
-d[k].append(v)
-else:
-d[k] = [v]
-
-return d
+return {k: list(v) for k, v in self._items.iteritems()}
 
 @attr.s(frozen=True)
 class parsedrequest(object):



To: indygreg, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2792: hgweb: port archive command to modern response API

[yuja (Yuya Nishihara)]

yuja added inline comments.

INLINE COMMENTS

> webcommands.py:1220
> +web.res.setbodywillwrite()
> +assert list(web.res.sendresponse()) == []
> +

`assert` may be deleted by `-O`, but this one is important.

Can you send a followup?

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2792

To: indygreg, #hg-reviewers, durin42
Cc: yuja, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 1 of 7 V4] revbranchcache: add a public function to update the data

[Gregory Szorc]

On Fri, Mar 16, 2018 at 8:20 AM, Yuya Nishihara  wrote:

> On Fri, 16 Mar 2018 12:29:33 +0100, Boris Feld wrote:
> > # HG changeset patch
> > # User Boris Feld 
> > # Date 1516281665 -3600
> > #  Thu Jan 18 14:21:05 2018 +0100
> > # Node ID 695dfd0f5d9849a7f0b81e623dbdb1befdca63c9
> > # Parent  c978ffef167e8c5d343c3cca1ebf23f1082c3c09
> > # EXP-Topic wire-rbc
> > # Available At https://bitbucket.org/octobus/mercurial-devel/
> > #  hg pull https://bitbucket.org/octobus/mercurial-devel/
> -r 695dfd0f5d98
> > revbranchcache: add a public function to update the data
>
> Queued per the last review of V2, thanks.
>

Boris: please follow up on this series with additions to
mercurial/help/internals/bundle2.txt that document the new bundle2 part and
capability string.


> ___
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 3 of 4 V2] formatter: unblock storing fctx as a template resource

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1520771175 -32400
#  Sun Mar 11 21:26:15 2018 +0900
# Node ID f810d481d55c0573912058c68c615bfa30e20805
# Parent  5d94efb75658163ab7db01acbefeb6469bc97739
formatter: unblock storing fctx as a template resource

To keep templateformatter._renderitem() simple, a repo instance is looked
through ctx if available. This is probably good for future subrepo support
where ctx.repo() may be different from the global repo.

diff --git a/mercurial/formatter.py b/mercurial/formatter.py
--- a/mercurial/formatter.py
+++ b/mercurial/formatter.py
@@ -188,7 +188,7 @@ class baseformatter(object):
 def context(self, **ctxs):
 '''insert context objects to be used to render template keywords'''
 ctxs = pycompat.byteskwargs(ctxs)
-assert all(k == 'ctx' for k in ctxs)
+assert all(k in {'ctx', 'fctx'} for k in ctxs)
 if self._converter.storecontext:
 self._item.update(ctxs)
 def data(self, **data):
@@ -395,13 +395,11 @@ class templateformatter(baseformatter):
 return
 ref = self._parts[part]
 
-# TODO: add support for filectx
 props = {}
 # explicitly-defined fields precede templatekw
 props.update(item)
-if 'ctx' in item:
+if 'ctx' in item or 'fctx' in item:
 # but template resources must be always available
-props['repo'] = props['ctx'].repo()
 props['revcache'] = {}
 props = pycompat.strkwargs(props)
 g = self._t(ref, **props)
@@ -513,10 +511,25 @@ def templateresources(ui, repo=None):
 return v
 return resmap.get(key)
 
+def getctx(context, mapping, key):
+ctx = mapping.get('ctx')
+if ctx is not None:
+return ctx
+fctx = mapping.get('fctx')
+if fctx is not None:
+return fctx.changectx()
+
+def getrepo(context, mapping, key):
+ctx = getctx(context, mapping, 'ctx')
+if ctx is not None:
+return ctx.repo()
+return getsome(context, mapping, key)
+
 return {
 'cache': getsome,
-'ctx': getsome,
-'repo': getsome,
+'ctx': getctx,
+'fctx': getsome,
+'repo': getrepo,
 'revcache': getsome,  # per-ctx cache; set later
 'ui': getsome,
 }
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 4 of 4 V2] annotate: add support for template keywords and functions depending on ctx

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1520771788 -32400
#  Sun Mar 11 21:36:28 2018 +0900
# Node ID 654506d13eb9df1cd358266a3fc180dabcd1db52
# Parent  f810d481d55c0573912058c68c615bfa30e20805
annotate: add support for template keywords and functions depending on ctx

diff --git a/mercurial/commands.py b/mercurial/commands.py
--- a/mercurial/commands.py
+++ b/mercurial/commands.py
@@ -405,14 +405,15 @@ def annotate(ui, repo, *pats, **opts):
 formats.append(['%s' for x in l])
 pieces.append(l)
 
-for f, p, l in zip(zip(*formats), zip(*pieces), lines):
+for f, p, (n, l) in zip(zip(*formats), zip(*pieces), lines):
 fm.startitem()
+fm.context(fctx=n.fctx)
 fm.write(fields, "".join(f), *p)
-if l[0].skip:
+if n.skip:
 fmt = "* %s"
 else:
 fmt = ": %s"
-fm.write('line', fmt, l[1])
+fm.write('line', fmt, l)
 
 if not lines[-1][1].endswith('\n'):
 fm.plain('\n')
diff --git a/tests/test-annotate.t b/tests/test-annotate.t
--- a/tests/test-annotate.t
+++ b/tests/test-annotate.t
@@ -71,6 +71,11 @@ annotate (JSON)
}
   ]
 
+log-like templating
+
+  $ hg annotate -T'{lines % "{rev} {node|shortest}: {line}"}' a
+  0 8435: a
+
   $ cat <>a
   > a
   > a
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 1 of 4 V2] templater: convert resources to a table of callables for future extension

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1520769929 -32400
#  Sun Mar 11 21:05:29 2018 +0900
# Node ID 5def08d8870b1a9b4f0d88a80ac8f4696a199ea6
# Parent  c1f88589ddc9885b39025f3d4b19a5cf3cfe3251
templater: convert resources to a table of callables for future extension

I'm going to add a full templating support to the annotate command. As the
annotate is a filectx-oriented command, we'll  need a way to look up a ctx
from a bounded fctx only when necessary.

This is the minimal change to support that. I'm thinking of defining a proper
interface to look up template resources to fix other issues, but that isn't
ready yet.

(Changes from V1: just updated tests and patch descriptions.)

diff --git a/mercurial/formatter.py b/mercurial/formatter.py
--- a/mercurial/formatter.py
+++ b/mercurial/formatter.py
@@ -501,14 +501,23 @@ def maketemplater(ui, tmpl, defaults=Non
 def templateresources(ui, repo=None):
 """Create a dict of template resources designed for the default templatekw
 and function"""
-return {
+resmap = {
 'cache': {},  # for templatekw/funcs to store reusable data
-'ctx': None,
 'repo': repo,
-'revcache': None,  # per-ctx cache; set later
 'ui': ui,
 }
 
+def getsome(context, mapping, key):
+return resmap.get(key)
+
+return {
+'cache': getsome,
+'ctx': getsome,
+'repo': getsome,
+'revcache': getsome,  # per-ctx cache; set later
+'ui': getsome,
+}
+
 def formatter(ui, out, topic, opts):
 template = opts.get("template", "")
 if template == "json":
diff --git a/mercurial/logcmdutil.py b/mercurial/logcmdutil.py
--- a/mercurial/logcmdutil.py
+++ b/mercurial/logcmdutil.py
@@ -423,7 +423,7 @@ class changesettemplater(changesetprinte
  resources=tres,
  cache=templatekw.defaulttempl)
 self._counter = itertools.count()
-self.cache = tres['cache']  # shared with _graphnodeformatter()
+self._getcache = tres['cache']  # shared with _graphnodeformatter()
 
 self._tref = tmplspec.ref
 self._parts = {'header': '', 'footer': '',
@@ -852,7 +852,8 @@ def _graphnodeformatter(ui, displayer):
 spec = templater.unquotestring(spec)
 tres = formatter.templateresources(ui)
 if isinstance(displayer, changesettemplater):
-tres['cache'] = displayer.cache  # reuse cache of slow templates
+# reuse cache of slow templates
+tres['cache'] = displayer._getcache
 templ = formatter.maketemplater(ui, spec, defaults=templatekw.keywords,
 resources=tres)
 def formatnode(repo, ctx):
diff --git a/mercurial/templater.py b/mercurial/templater.py
--- a/mercurial/templater.py
+++ b/mercurial/templater.py
@@ -566,8 +566,8 @@ class engine(object):
 v = None
 if key in self._resources:
 v = mapping.get(key)
-if v is None:
-v = self._resources.get(key)
+if v is None and key in self._resources:
+v = self._resources[key](self, mapping, key)
 if v is None:
 raise templateutil.ResourceUnavailable(
 _('template resource not available: %s') % key)
@@ -670,8 +670,9 @@ class templater(object):
 - ``filters``: a dict of functions to transform a value into another.
 - ``defaults``: a dict of symbol values/functions; may be overridden
   by a ``mapping`` dict.
-- ``resources``: a dict of internal data (e.g. cache), inaccessible
-  from user template; may be overridden by a ``mapping`` dict.
+- ``resources``: a dict of functions returning internal data
+  (e.g. cache), inaccessible from user template; may be overridden by
+  a ``mapping`` dict.
 - ``cache``: a dict of preloaded template fragments.
 - ``aliases``: a list of alias (name, replacement) pairs.
 
@@ -691,7 +692,7 @@ class templater(object):
 self.filters = templatefilters.filters.copy()
 self.filters.update(filters)
 self.defaults = defaults
-self._resources = {'templ': self}
+self._resources = {'templ': lambda context, mapping, key: self}
 self._resources.update(resources)
 self._aliases = aliases
 self.minchunk, self.maxchunk = minchunk, maxchunk
diff --git a/mercurial/templateutil.py b/mercurial/templateutil.py
--- a/mercurial/templateutil.py
+++ b/mercurial/templateutil.py
@@ -350,7 +350,8 @@ def runsymbol(context, mapping, key, def
 v = default
 if callable(v) and getattr(v, '_requires', None) is None:
 # old templatekw: expand all keywords and resources
-props = context._resources.copy()
+props = {k: f(context, mapping, k)
+ for k, f in context._resources.items()}
 props.update(mapping)
 return v(**pycompat.strkwargs(props))
 if callable(v

[PATCH 2 of 4 V2] templater: process mapping dict by resource callables

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1520770322 -32400
#  Sun Mar 11 21:12:02 2018 +0900
# Node ID 5d94efb75658163ab7db01acbefeb6469bc97739
# Parent  5def08d8870b1a9b4f0d88a80ac8f4696a199ea6
templater: process mapping dict by resource callables

A resource item is a callable, so let's make it look up a resource object
by itself.

diff --git a/mercurial/formatter.py b/mercurial/formatter.py
--- a/mercurial/formatter.py
+++ b/mercurial/formatter.py
@@ -508,6 +508,9 @@ def templateresources(ui, repo=None):
 }
 
 def getsome(context, mapping, key):
+v = mapping.get(key)
+if v is not None:
+return v
 return resmap.get(key)
 
 return {
diff --git a/mercurial/templater.py b/mercurial/templater.py
--- a/mercurial/templater.py
+++ b/mercurial/templater.py
@@ -565,8 +565,6 @@ class engine(object):
 evaluation"""
 v = None
 if key in self._resources:
-v = mapping.get(key)
-if v is None and key in self._resources:
 v = self._resources[key](self, mapping, key)
 if v is None:
 raise templateutil.ResourceUnavailable(
@@ -671,8 +669,7 @@ class templater(object):
 - ``defaults``: a dict of symbol values/functions; may be overridden
   by a ``mapping`` dict.
 - ``resources``: a dict of functions returning internal data
-  (e.g. cache), inaccessible from user template; may be overridden by
-  a ``mapping`` dict.
+  (e.g. cache), inaccessible from user template.
 - ``cache``: a dict of preloaded template fragments.
 - ``aliases``: a list of alias (name, replacement) pairs.
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 1 of 3] test-template-engine: deduplicate methods of custom template engine

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1521112254 -32400
#  Thu Mar 15 20:10:54 2018 +0900
# Node ID a305e7f025b030a74ca178f722e7502d6c012d6a
# Parent  451f4ada3589ebf0074f0a882c5b6b03b2c70bdf
test-template-engine: deduplicate methods of custom template engine

diff --git a/tests/test-template-engine.t b/tests/test-template-engine.t
--- a/tests/test-template-engine.t
+++ b/tests/test-template-engine.t
@@ -6,23 +6,12 @@
   > templateutil,
   > )
   > 
-  > class mytemplater(object):
-  > def __init__(self, loader, filters, defaults, resources, aliases):
-  > self.loader = loader
-  > self._defaults = defaults
-  > self._resources = resources
-  > 
-  > def symbol(self, mapping, key):
-  > return mapping[key]
-  > 
-  > def resource(self, mapping, key):
-  > v = self._resources[key]
-  > if v is None:
-  > v = mapping[key]
-  > return v
+  > class mytemplater(templater.engine):
+  > def _load(self, t):
+  > return self._loader(t)
   > 
   > def process(self, t, map):
-  > tmpl = self.loader(t)
+  > tmpl = self._load(t)
   > props = self._defaults.copy()
   > props.update(map)
   > for k, v in props.items():
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 2 of 3] py3: make test-template-engine.t bytes-safe

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1521212353 -32400
#  Fri Mar 16 23:59:13 2018 +0900
# Node ID 7977db549515436936f56befb1b065d0b956996e
# Parent  a305e7f025b030a74ca178f722e7502d6c012d6a
py3: make test-template-engine.t bytes-safe

diff --git a/contrib/python3-whitelist b/contrib/python3-whitelist
--- a/contrib/python3-whitelist
+++ b/contrib/python3-whitelist
@@ -388,6 +388,7 @@ test-subrepo.t
 test-symlinks.t
 test-tag.t
 test-tags.t
+test-template-engine.t
 test-treemanifest.t
 test-unamend.t
 test-uncommit.t
diff --git a/tests/test-template-engine.t b/tests/test-template-engine.t
--- a/tests/test-template-engine.t
+++ b/tests/test-template-engine.t
@@ -2,6 +2,7 @@
   $ cat > engine.py << EOF
   > 
   > from mercurial import (
+  > pycompat,
   > templater,
   > templateutil,
   > )
@@ -15,19 +16,20 @@
   > props = self._defaults.copy()
   > props.update(map)
   > for k, v in props.items():
-  > if k in ('templ', 'ctx', 'repo', 'revcache', 'cache', 
'troubles'):
+  > if k in (b'templ', b'ctx', b'repo', b'revcache', b'cache',
+  >  b'troubles'):
   > continue
   > if callable(v) and getattr(v, '_requires', None) is None:
   > props = self._resources.copy()
   > props.update(map)
-  > v = v(**props)
+  > v = v(**pycompat.strkwargs(props))
   > elif callable(v):
   > v = v(self, props)
   > v = templateutil.stringify(v)
-  > tmpl = tmpl.replace('{{%s}}' % k, v)
+  > tmpl = tmpl.replace(b'{{%s}}' % k, v)
   > yield tmpl
   > 
-  > templater.engines['my'] = mytemplater
+  > templater.engines[b'my'] = mytemplater
   > EOF
   $ hg init test
   $ echo '[extensions]' > test/.hg/hgrc
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 3 of 3] test-template-engine: do not evaluate unused keywords by custom engine

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara 
# Date 1521212774 -32400
#  Sat Mar 17 00:06:14 2018 +0900
# Node ID c1f88589ddc9885b39025f3d4b19a5cf3cfe3251
# Parent  7977db549515436936f56befb1b065d0b956996e
test-template-engine: do not evaluate unused keywords by custom engine

If the custom engine, "mytemplater", were installed as the default, it would
enter to an infinite recursion at stringify(v) because template keywords may
generate a nested mapping containing the same keywords.

Spotted by a future patch which will replace context.resource('templ')(...)
with context.process(...).

diff --git a/tests/test-template-engine.t b/tests/test-template-engine.t
--- a/tests/test-template-engine.t
+++ b/tests/test-template-engine.t
@@ -16,8 +16,7 @@
   > props = self._defaults.copy()
   > props.update(map)
   > for k, v in props.items():
-  > if k in (b'templ', b'ctx', b'repo', b'revcache', b'cache',
-  >  b'troubles'):
+  > if b'{{%s}}' % k not in tmpl:
   > continue
   > if callable(v) and getattr(v, '_requires', None) is None:
   > props = self._resources.copy()
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Re: [PATCH 1 of 7 V4] revbranchcache: add a public function to update the data

[Yuya Nishihara]

On Fri, 16 Mar 2018 12:29:33 +0100, Boris Feld wrote:
> # HG changeset patch
> # User Boris Feld 
> # Date 1516281665 -3600
> #  Thu Jan 18 14:21:05 2018 +0100
> # Node ID 695dfd0f5d9849a7f0b81e623dbdb1befdca63c9
> # Parent  c978ffef167e8c5d343c3cca1ebf23f1082c3c09
> # EXP-Topic wire-rbc
> # Available At https://bitbucket.org/octobus/mercurial-devel/
> #  hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 
> 695dfd0f5d98
> revbranchcache: add a public function to update the data

Queued per the last review of V2, thanks.
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2873: remotenames: add functionality to override -B flag of push

[pulkit (Pulkit Goyal)]

pulkit added a comment.


  I am not sure the functionality and config names should go to core or not, so 
I have introduced them in this extension. Also suggestions for better config 
names are welcome.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2873

To: pulkit, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2880: bundle: add the possibility to bundle bookmarks (issue5792)

[lothiraldan (Boris Feld)]

lothiraldan added a subscriber: indygreg.
lothiraldan added inline comments.

INLINE COMMENTS

> pulkit wrote in configitems.py:422
> Any reason why we have all these config option as experimental? We talked at 
> sprint about moving things out of experimental and we agreed on 
> `experimental.bundle-phases`. This one sounds similar.

I was afraid we might have to delete this option once we have the discussion 
about bundle spec format with @indygreg. If that would be the case, I would 
feel more comfortable deleting an experimental option.

If we agreed on moving bundle-phases out of experimental, I guess we can do the 
same for bundle-bookmarks.

@indygreg Do you think we would still have this option around once we have 
new-style bundle spec?

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2880

To: lothiraldan, #hg-reviewers
Cc: indygreg, pulkit, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2880: bundle: add the possibility to bundle bookmarks (issue5792)

[pulkit (Pulkit Goyal)]

pulkit added inline comments.

INLINE COMMENTS

> configitems.py:422
>  )
> +coreconfigitem('experimental', 'bundle-bookmarks',
> +default=False,

Any reason why we have all these config option as experimental? We talked at 
sprint about moving things out of experimental and we agreed on 
`experimental.bundle-phases`. This one sounds similar.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2880

To: lothiraldan, #hg-reviewers
Cc: pulkit, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D2880: bundle: add the possibility to bundle bookmarks (issue5792)

[lothiraldan (Boris Feld)]

lothiraldan created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  Also take the wlock when unbundling. It shouldn't have a big impact on
  performance.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D2880

AFFECTED FILES
  mercurial/bundle2.py
  mercurial/commands.py
  mercurial/configitems.py
  mercurial/debugcommands.py
  tests/test-bundle-bookmarks.t

CHANGE DETAILS

diff --git a/tests/test-bundle-bookmarks.t b/tests/test-bundle-bookmarks.t
new file mode 100644
--- /dev/null
+++ b/tests/test-bundle-bookmarks.t
@@ -0,0 +1,80 @@
+  $ cat >> $HGRCPATH < [experimental]
+  > bundle-bookmarks=yes
+  > [extensions]
+  > strip=
+  > drawdag=$TESTDIR/drawdag.py
+  > EOF
+
+Set up repo with linear history
+  $ hg init linear
+  $ cd linear
+  $ hg debugdrawdag <<'EOF'
+  > E
+  > |
+  > D
+  > |
+  > C
+  > |
+  > B
+  > |
+  > A
+  > EOF
+  $ hg bookmarks -r A "A1"
+  $ hg bookmarks -r D "D1"
+  $ hg bookmarks -r D "D2"
+  $ hg log -G -T '{desc} {bookmarks}\n'
+  o  E
+  |
+  o  D D1 D2
+  |
+  o  C
+  |
+  o  B
+  |
+  o  A A1
+  
+Bookmarks are restored when unbundling
+  $ hg bundle --all bundle
+  5 changesets found
+  $ hg debugbundle bundle
+  Stream params: {Compression: BZ}
+  changegroup -- {nbchanges: 5, version: 02}
+  426bada5c67598ca65036d57d9e4b64b0c1ce7a0
+  112478962961147124edd43549aedd1a335e44bf
+  26805aba1e600a82e93661149f2313866a221a7b
+  f585351a92f85104bff7c284233c338b10eb1df7
+  9bc730a19041f9ec7cb33c626e811aa233efb18c
+  bookmarks -- {}
+  A1: Bk\xad\xa5\xc6u\x98\xcae\x03mW\xd9\xe4\xb6K\x0c\x1c\xe7\xa0 (esc)
+  D1: \xf5\x855\x1a\x92\xf8Q\x04\xbf\xf7\xc2\x84#<3\x8b\x10\xeb\x1d\xf7 
(esc)
+  D2: \xf5\x855\x1a\x92\xf8Q\x04\xbf\xf7\xc2\x84#<3\x8b\x10\xeb\x1d\xf7 
(esc)
+  $ hg strip --no-backup C
+  $ hg unbundle -q bundle
+  $ hg log -G -T '{desc} {bookmarks}\n'
+  o  E
+  |
+  o  D D1 D2
+  |
+  o  C
+  |
+  o  B
+  |
+  o  A A1
+  
+Bookmarks doesn't conflict with local bookmarks
+
+  $ hg bookmarks -d A1
+  $ hg bookmarks -r A "A2"
+  $ hg unbundle -q bundle
+  $ hg log -G -T '{desc} {bookmarks}\n'
+  o  E
+  |
+  o  D D1 D2
+  |
+  o  C
+  |
+  o  B
+  |
+  o  A A1 A2
+  
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -33,6 +33,7 @@
 short,
 )
 from . import (
+bookmarks,
 bundle2,
 changegroup,
 cmdutil,
@@ -326,6 +327,14 @@
 ui.write(indent_string)
 ui.write('%s %s\n' % (hex(head), phases.phasenames[phase]))
 
+def _debugbookmarks(ui, data, indent=0):
+"""display version and markers contained in 'data'"""
+indent_string = ' ' * indent
+bm = bookmarks.binarydecode(data)
+for bookmark in sorted(bm):
+ui.write(indent_string)
+ui.write('%s: %s\n' % (bookmark[0], bookmark[1]))
+
 def _quasirepr(thing):
 if isinstance(thing, (dict, util.sortdict, collections.OrderedDict)):
 return '{%s}' % (
@@ -353,6 +362,9 @@
 if part.type == 'phase-heads':
 if not ui.quiet:
 _debugphaseheads(ui, part, indent=4)
+if part.type == 'bookmarks':
+if not ui.quiet:
+_debugbookmarks(ui, part, indent=4)
 
 @command('debugbundle',
 [('a', 'all', None, _('show all details')),
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -419,6 +419,9 @@
 coreconfigitem('experimental', 'archivemetatemplate',
 default=dynamicdefault,
 )
+coreconfigitem('experimental', 'bundle-bookmarks',
+default=False,
+)
 coreconfigitem('experimental', 'bundle-phases',
 default=False,
 )
diff --git a/mercurial/commands.py b/mercurial/commands.py
--- a/mercurial/commands.py
+++ b/mercurial/commands.py
@@ -1267,6 +1267,8 @@
 contentopts['obsolescence'] = True
 if repo.ui.configbool('experimental', 'bundle-phases'):
 contentopts['phases'] = True
+if repo.ui.configbool('experimental', 'bundle-bookmarks'):
+contentopts['bookmarks'] = True
 bundle2.writenewbundle(ui, repo, 'bundle', fname, bversion, outgoing,
contentopts, compression=bcompression,
compopts=compopts)
@@ -5406,7 +5408,7 @@
 """
 fnames = (fname1,) + fnames
 
-with repo.lock():
+with repo.wlock(), repo.lock():
 for fname in fnames:
 f = hg.openpath(ui, fname)
 gen = exchange.readbundle(ui, f, fname)
diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -1599,6 +1599,14 @@
 phasedata = phases.binaryencode(headsbyphase)
 bundler.newpart('phase-heads', data=phasedata)
 
+if opts.get('bookmarks', False):
+# From exchange._pushb2bookmarkspart
+data = []
+for book, new in rep

[PATCH 1 of 7 V4] revbranchcache: add a public function to update the data

[Boris Feld]

# HG changeset patch
# User Boris Feld 
# Date 1516281665 -3600
#  Thu Jan 18 14:21:05 2018 +0100
# Node ID 695dfd0f5d9849a7f0b81e623dbdb1befdca63c9
# Parent  c978ffef167e8c5d343c3cca1ebf23f1082c3c09
# EXP-Topic wire-rbc
# Available At https://bitbucket.org/octobus/mercurial-devel/
#  hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 
695dfd0f5d98
revbranchcache: add a public function to update the data

We want to exchange more cached data over the wire. To do so, we need a clean
way to update the cache on the receiving ends.

diff --git a/mercurial/branchmap.py b/mercurial/branchmap.py
--- a/mercurial/branchmap.py
+++ b/mercurial/branchmap.py
@@ -454,6 +454,26 @@ class revbranchcache(object):
 self._setcachedata(rev, reponode, branchidx)
 return b, close
 
+def setdata(self, branch, rev, node, close):
+"""add new data information to the cache"""
+if branch in self._namesreverse:
+branchidx = self._namesreverse[branch]
+else:
+branchidx = len(self._names)
+self._names.append(branch)
+self._namesreverse[branch] = branchidx
+if close:
+branchidx |= _rbccloseflag
+self._setcachedata(rev, node, branchidx)
+# If no cache data were readable (non exists, bad permission, etc)
+# the cache was bypassing itself by setting:
+#
+#   self.branchinfo = self._branchinfo
+#
+# Since we now have data in the cache, we need to drop this bypassing.
+if 'branchinfo' in vars(self):
+del self.branchinfo
+
 def _setcachedata(self, rev, node, branchidx):
 """Writes the node's branch data to the in-memory cache data."""
 if rev == nullrev:
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 6 of 7 V4] revbranchcache: disable the new part for narrow hg bundle

[Boris Feld]

# HG changeset patch
# User Boris Feld 
# Date 1519237601 -3600
#  Wed Feb 21 19:26:41 2018 +0100
# Node ID bb1dcdcff788fbeabd761d5de67db1b8e17561c0
# Parent  6355ab0513a089c60a59f76039bea39d07f9e80c
# EXP-Topic wire-rbc
# Available At https://bitbucket.org/octobus/mercurial-devel/
#  hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 
bb1dcdcff788
revbranchcache: disable the new part for narrow hg bundle

The lack of some revisions confuses the new cache part. To simplify things, we
disable it for now.

diff --git a/hgext/narrow/narrowbundle2.py b/hgext/narrow/narrowbundle2.py
--- a/hgext/narrow/narrowbundle2.py
+++ b/hgext/narrow/narrowbundle2.py
@@ -479,6 +479,19 @@ def setup():
 origcgfn(*args, **kwargs)
 exchange.getbundle2partsmapping['changegroup'] = wrappedcgfn
 
+# disable rev branch cache exchange when serving a narrow bundle
+# (currently incompatible with that part)
+origrbcfn = exchange.getbundle2partsmapping['cache:rev-branch-cache']
+def wrappedcgfn(*args, **kwargs):
+repo = args[1]
+if repo.ui.has_section(_NARROWACL_SECTION):
+return
+elif kwargs.get(r'narrow', False):
+return
+else:
+origrbcfn(*args, **kwargs)
+exchange.getbundle2partsmapping['cache:rev-branch-cache'] = wrappedcgfn
+
 # Extend changegroup receiver so client can fixup after widen requests.
 origcghandler = bundle2.parthandlermapping['changegroup']
 def wrappedcghandler(op, inpart):
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 4 of 7 V4] bundle: include advisory rev branch cache part in bundle2 bundle

[Boris Feld]

# HG changeset patch
# User Boris Feld 
# Date 1519230780 -3600
#  Wed Feb 21 17:33:00 2018 +0100
# Node ID d0daa9803d599b7b6fd4f80a773b08a32cc2ccb3
# Parent  fa45b38924bec627efbd1ff270e68943211a9c8d
# EXP-Topic wire-rbc
# Available At https://bitbucket.org/octobus/mercurial-devel/
#  hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 
d0daa9803d59
bundle: include advisory rev branch cache part in bundle2 bundle

`hg bundle` command producing bundle2 will now include an optional part
containing the revision-branch cache data.

The data sent are mostly nodes so it is quite compact.  The goal of the
rev-branch-cache is to speed up branch map computation, especially when the
branchmap gets invalidated so we send data for all exchanged changesets. In
addition, computing the relevant heads to send in case of partial pulling would
be challenging.

As a reminder, the rev branch cache data significantly speed up branch
computation. Having it around provides a small speedup to pull/clone and much
higher tolerance to branch map cache invalidation that might happens from later
commands.

On the Mercurial repository, computing the visible branchmap from scratch move
from 2.00 seconds to 0.34s (a -83% speedup).

Using this new part, Unbundling the full Mercurial repository moves from 25.736
seconds to 24.030 seconds (around -7% speedup). The bundle size increase is
around 3% (from 22.43 MB to 23.13MB)


On an half a million revision repository with twenty thousand
branches, computing the branchmap moves from 75 seconds to 45 second (-40%) if
the caches is used.

A bundle containing 50 000 changesets in such repository get a 0.5% size
increase from such part for a -3% unbundling time speedup.

diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -1591,6 +1591,7 @@ def _addpartsfromopts(ui, repo, bundler,
 part.addparam('targetphase', '%d' % phases.secret, mandatory=False)
 
 addparttagsfnodescache(repo, bundler, outgoing)
+addpartrevbranchcache(repo, bundler, outgoing)
 
 if opts.get('obsolescence', False):
 obsmarkers = repo.obsstore.relevantmarkers(outgoing.missing)
diff --git a/tests/test-bundle-phases.t b/tests/test-bundle-phases.t
--- a/tests/test-bundle-phases.t
+++ b/tests/test-bundle-phases.t
@@ -42,6 +42,7 @@ Phases are restored when unbundling
   26805aba1e600a82e93661149f2313866a221a7b
   f585351a92f85104bff7c284233c338b10eb1df7
   9bc730a19041f9ec7cb33c626e811aa233efb18c
+  cache:rev-branch-cache -- {}
   phase-heads -- {}
   26805aba1e600a82e93661149f2313866a221a7b draft
   $ hg strip --no-backup C
@@ -233,6 +234,7 @@ Restore bundle of entire repo
   dc0947a82db884575bb76ea10ac97b08536bfa03
   4e4f9194f9f181c57f62e823e8bdfa46ab9e4ff4
   03ca77807e919db8807c3749086dc36fb478cac0
+  cache:rev-branch-cache -- {}
   phase-heads -- {}
   dc0947a82db884575bb76ea10ac97b08536bfa03 public
   03ca77807e919db8807c3749086dc36fb478cac0 draft
@@ -258,6 +260,7 @@ Restore bundle of entire repo
   changegroup -- {nbchanges: 2, targetphase: 2, version: 02}
   112478962961147124edd43549aedd1a335e44bf
   4e4f9194f9f181c57f62e823e8bdfa46ab9e4ff4
+  cache:rev-branch-cache -- {}
   phase-heads -- {}
   $ rm bundle
 
@@ -269,6 +272,7 @@ Restore bundle of entire repo
   112478962961147124edd43549aedd1a335e44bf
   dc0947a82db884575bb76ea10ac97b08536bfa03
   4e4f9194f9f181c57f62e823e8bdfa46ab9e4ff4
+  cache:rev-branch-cache -- {}
   phase-heads -- {}
   dc0947a82db884575bb76ea10ac97b08536bfa03 public
   $ rm bundle
@@ -280,6 +284,7 @@ Restore bundle of entire repo
   changegroup -- {nbchanges: 2, targetphase: 2, version: 02}
   4e4f9194f9f181c57f62e823e8bdfa46ab9e4ff4
   03ca77807e919db8807c3749086dc36fb478cac0
+  cache:rev-branch-cache -- {}
   phase-heads -- {}
   03ca77807e919db8807c3749086dc36fb478cac0 draft
   $ rm bundle
diff --git a/tests/test-bundle-type.t b/tests/test-bundle-type.t
--- a/tests/test-bundle-type.t
+++ b/tests/test-bundle-type.t
@@ -76,6 +76,7 @@ test bundle types
   Stream params: {}
   changegroup -- {nbchanges: 1, version: 02}
   c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
+  cache:rev-branch-cache -- {}
   none-v2
   
   % test bundle type bzip2
@@ -85,6 +86,7 @@ test bundle types
   Stream params: {Compression: BZ}
   changegroup -- {nbchanges: 1, version: 02}
   c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
+  cache:rev-branch-cache -- {}
   bzip2-v2
   
   % test bundle type gzip
@@ -94,6 +96,7 @@ test bundle types
   Stream params: {Compression: GZ}
   changegroup -- {nbchanges: 1, version: 02}
   c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
+  cache:rev-branch-cache -- {}
   gzip-v2
   
   % test bundle type none-v2
@@ -103,6 +106,7 @@ test bundle types
   Stream params: {}
   changegroup -- {nbchanges: 1, version: 02}
   c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
+  cache:rev-branch-cache -- {}
   none-v2
   
   % test 

[PATCH 3 of 7 V4] rev-branch-cache: add a function to generate a part

[Boris Feld]

# HG changeset patch
# User Boris Feld 
# Date 1519230382 -3600
#  Wed Feb 21 17:26:22 2018 +0100
# Node ID fa45b38924bec627efbd1ff270e68943211a9c8d
# Parent  1eb0f8075f0d33fe0aebf7243eaf50db2b33496d
# EXP-Topic wire-rbc
# Available At https://bitbucket.org/octobus/mercurial-devel/
#  hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 
fa45b38924be
rev-branch-cache: add a function to generate a part

The function is able to produce a rbc part consumed by the function introduced
into previous changesets. More details on usage and impact in the next
changesets.

diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -147,6 +147,7 @@ preserve.
 
 from __future__ import absolute_import, division
 
+import collections
 import errno
 import os
 import re
@@ -1624,6 +1625,28 @@ def addparttagsfnodescache(repo, bundler
 if chunks:
 bundler.newpart('hgtagsfnodes', data=''.join(chunks))
 
+def addpartrevbranchcache(repo, bundler, outgoing):
+# we include the rev branch cache for the bundle changeset
+# (as an optional parts)
+cache = repo.revbranchcache()
+cl = repo.unfiltered().changelog
+branchesdata = collections.defaultdict(lambda: (set(), set()))
+for node in outgoing.missing:
+branch, close = cache.branchinfo(cl.rev(node))
+branchesdata[branch][close].add(node)
+
+def generate():
+for branch, (nodes, closed) in sorted(branchesdata.items()):
+utf8branch = encoding.fromlocal(branch)
+yield rbcstruct.pack(len(utf8branch), len(nodes), len(closed))
+yield utf8branch
+for n in sorted(nodes):
+yield n
+for n in sorted(closed):
+yield n
+
+bundler.newpart('cache:rev-branch-cache', data=generate())
+
 def buildobsmarkerspart(bundler, markers):
 """add an obsmarker part to the bundler with 
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 5 of 7 V4] revbranchcache: add the necessary bit to send 'rbc' data over bundle2

[Boris Feld]

# HG changeset patch
# User Boris Feld 
# Date 1516283882 -3600
#  Thu Jan 18 14:58:02 2018 +0100
# Node ID 6355ab0513a089c60a59f76039bea39d07f9e80c
# Parent  d0daa9803d599b7b6fd4f80a773b08a32cc2ccb3
# EXP-Topic wire-rbc
# Available At https://bitbucket.org/octobus/mercurial-devel/
#  hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 
6355ab0513a0
revbranchcache: add the necessary bit to send 'rbc' data over bundle2

Getbundle is now capable of sending rev-branch-cache information for the
changesets it bundle. The data sent are mostly nodes so it is quite compact.
The goal of the rev-branch-cache is to speed up branch map computation,
especially when the branchmap gets invalidated so we send data for all
exchanged changesets. In addition, computing the relevant heads to send in
case of partial pulling would be challenging.

The feature is still inactive since the capability is not advertised yet.

diff --git a/mercurial/exchange.py b/mercurial/exchange.py
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -1939,6 +1939,28 @@ def _getbundletagsfnodes(bundler, repo, 
 outgoing = _computeoutgoing(repo, heads, common)
 bundle2.addparttagsfnodescache(repo, bundler, outgoing)
 
+@getbundle2partsgenerator('cache:rev-branch-cache')
+def _getbundlerevbranchcache(bundler, repo, source, bundlecaps=None,
+ b2caps=None, heads=None, common=None,
+ **kwargs):
+"""Transfer the rev-branch-cache mapping
+
+The payload is a series of data related to each branch
+
+1) branch name length
+2) number of open heads
+3) number of closed heads
+4) open heads nodes
+5) closed heads nodes
+"""
+# Don't send unless:
+# - changeset are being exchanged,
+# - the client supports it.
+if not (kwargs.get(r'cg', True)) or 'rev-branch-cache' not in b2caps:
+return
+outgoing = _computeoutgoing(repo, heads, common)
+bundle2.addpartrevbranchcache(repo, bundler, outgoing)
+
 def check_heads(repo, their_heads, context):
 """check if the heads of a repo have been modified
 
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

[PATCH 2 of 7 V4] revbranchcache: add a bundle2 handler for a rbc part

[Boris Feld]

# HG changeset patch
# User Boris Feld 
# Date 1519230904 -3600
#  Wed Feb 21 17:35:04 2018 +0100
# Node ID 1eb0f8075f0d33fe0aebf7243eaf50db2b33496d
# Parent  695dfd0f5d9849a7f0b81e623dbdb1befdca63c9
# EXP-Topic wire-rbc
# Available At https://bitbucket.org/octobus/mercurial-devel/
#  hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 
1eb0f8075f0d
revbranchcache: add a bundle2 handler for a rbc part

We add the necessary bit to process a part containing rev-branch-cache
information. The local rev branch cache is then updated with the received
information.

Computing branch cache can become a significant part of the time spent pulling.

diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -158,6 +158,7 @@ from .i18n import _
 from . import (
 bookmarks,
 changegroup,
+encoding,
 error,
 node as nodemod,
 obsolete,
@@ -2129,6 +2130,40 @@ def handlehgtagsfnodes(op, inpart):
 cache.write()
 op.ui.debug('applied %i hgtags fnodes cache entries\n' % count)
 
+rbcstruct = struct.Struct('>III')
+
+@parthandler('cache:rev-branch-cache')
+def handlerbc(op, inpart):
+"""receive a rev-branch-cache payload and update the local cache
+
+The payload is a series of data related to each branch
+
+1) branch name length
+2) number of open heads
+3) number of closed heads
+4) open heads nodes
+5) closed heads nodes
+"""
+total = 0
+rawheader = inpart.read(rbcstruct.size)
+cache = op.repo.revbranchcache()
+cl = op.repo.unfiltered().changelog
+while rawheader:
+header = rbcstruct.unpack(rawheader)
+total += header[1] + header[2]
+utf8branch = inpart.read(header[0])
+branch = encoding.tolocal(utf8branch)
+for x in xrange(header[1]):
+node = inpart.read(20)
+rev = cl.rev(node)
+cache.setdata(branch, rev, node, False)
+for x in xrange(header[2]):
+node = inpart.read(20)
+rev = cl.rev(node)
+cache.setdata(branch, rev, node, True)
+rawheader = inpart.read(rbcstruct.size)
+cache.write()
+
 @parthandler('pushvars')
 def bundle2getvars(op, part):
 '''unbundle a bundle2 containing shellvars on the server'''
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel