D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

Closed by commit rHG5962fd0d1045: nodemap: write nodemap data on disk (authored 
by marmoute).
This revision was automatically updated to reflect the committed changes.
This revision was not accepted when it landed; it landed in state "Needs 
Review".

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=20016&id=20103

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=122880, 
sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=122880, 
sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, b'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -932,6 +932,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesd

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 20016.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19883&id=20016

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=122880, 
sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=122880, 
sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, b'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -932,6 +932,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
   

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute added a comment.
marmoute updated this revision to Diff 19883.


  rebase to latest default

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19825&id=19883

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=122880, 
sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=122880, 
sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, b'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute added a comment.
marmoute updated this revision to Diff 19825.


  small doc update on .#s[1]

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19780&id=19825

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, b'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelo

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 19780.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19751&id=19780

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, b'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
   

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 19751.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19425&id=19751

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, 'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute added inline comments.

INLINE COMMENTS

> martinvonz wrote in nodemap.py:49
> and here? this is a series that introduced a lot of new code, so try running 
> all (previously passing) py3 test at least at the end of the series

If you can taje the series starting at D8011 
, if would help me to make sure all 
angles are covered.

> martinvonz wrote in nodemap.py:50-51
> Actually, does `revlog.opener.write(revlog.nodemap_file, data)` work?

probably, but we will need the context manager approach later in the series so 
I don't think it is worth updating.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D7835: nodemap: write nodemap data on disk

[martinvonz (Martin von Zweigbergk)]

martinvonz added inline comments.

INLINE COMMENTS

> martinvonz wrote in nodemap.py:42
> need b'' here and below for py3?

not needed here, it seems, because `ProgrammingError` converts it for us

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D7835: nodemap: write nodemap data on disk

[martinvonz (Martin von Zweigbergk)]

martinvonz added inline comments.

INLINE COMMENTS

> nodemap.py:50-51
> +# store vfs
> +with revlog.opener(revlog.nodemap_file, 'w') as f:
> +f.write(data)
> +# EXP-TODO: if the transaction abort, we should remove the new data and

Actually, does `revlog.opener.write(revlog.nodemap_file, data)` work?

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D7835: nodemap: write nodemap data on disk

[martinvonz (Martin von Zweigbergk)]

martinvonz added inline comments.

INLINE COMMENTS

> nodemap.py:42
> +raise error.ProgrammingError(
> +"cannot persist nodemap of a filtered changelog"
> +)

need b'' here and below for py3?

> nodemap.py:49
> +# EXP-TODO: if this is a cache, this should use a cache vfs, not a
> +# store vfs
> +with revlog.opener(revlog.nodemap_file, 'w') as f:

and here? this is a series that introduced a lot of new code, so try running 
all (previously passing) py3 test at least at the end of the series

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 19425.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19185&id=19425

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, 'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool('experimental', 'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 19185.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19172&id=19185

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, 'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool('experimental', 'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 19172.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19155&id=19172

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, 
sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+"""Install whatever is needed transaction side to persist a nodemap on disk
+
+(only actually persist the nodemap if this is relevant for this revlog)
+"""
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, 'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool('experimental', 'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
 b'experimental', b'server.filesdata.recommended-batch-size', default=5,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@

D7835: nodemap: write nodemap data on disk

[marmoute (Pierre-Yves David)]

marmoute created this revision.
Herald added a reviewer: indygreg.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  Let us start writing data on disk (so that we can read it from there later).
  This series of changeset is going to focus first on having data on disk and
  updating it.
  
  Right now the data is written right next to the revlog data, in the store. We
  might move it to cache (with proper cache validation mechanism) later, but for
  now revlog have a storevfs instance and it is simpler to us it. The right
  location for this data is not the focus of this series.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, 
sha256=5dbe62ab98a26668b544063d4d674ac4452ba903ee8895c52fd21d9bbd771e09
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, 
sha256=5dbe62ab98a26668b544063d4d674ac4452ba903ee8895c52fd21d9bbd771e09
   : ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ||
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,35 @@
 raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+if revlog.nodemap_file is None:
+return  # we do not use persistent_nodemap on this revlog
+callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+if tr.hasfinalize(callback_id):
+return  # no need to register again
+tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+"""Write nodemap data on disk for a given revlog
+"""
+if getattr(revlog, 'filteredrevs', ()):
+raise error.ProgrammingError(
+"cannot persist nodemap of a filtered changelog"
+)
+if revlog.nodemap_file is None:
+msg = "calling persist nodemap on a revlog without the feature enableb"
+raise error.ProgrammingError(msg)
+data = persistent_data(revlog.index)
+# EXP-TODO: if this is a cache, this should use a cache vfs, not a
+# store vfs
+with revlog.opener(revlog.nodemap_file, 'w') as f:
+f.write(data)
+# EXP-TODO: if the transaction abort, we should remove the new data and
+# reinstall the old one. (This will be simpler when the file format get a
+# bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
 mmaplargeindex=False,
 censorable=False,
 upperboundcomp=None,
+persistentnodemap=False,
 ):
 """
 create a revlog object
@@ -418,6 +419,10 @@
 self.upperboundcomp = upperboundcomp
 self.indexfile = indexfile
 self.datafile = datafile or (indexfile[:-2] + b".d")
+self.nodemap_file = None
+if persistentnodemap:
+self.nodemap_file = indexfile[:-2] + b".n"
+
 self.opener = opener
 #  When True, indexfile is opened with checkambig=True at writing, to
 #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
 ifh.write(data[0])
 ifh.write(data[1])
 self._enforceinlinesize(transaction, ifh)
+nodemaputil.setup_persistent_nodemap(transaction, self)
 
 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
 """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
 if ui.configbool(b'experimental', b'rust.index'):
 options[b'rust.index'] = True
+if ui.configbool('experimental', 'exp-persistent-nodemap'):
+options[b'exp-persistent-nodemap'] = True
 
 return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
 b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+b'experimental', b'exp-persistent-nodemap', default=Fal