D8193: nodemap: automatically "vacuum" the persistent nodemap when too sparse

[marmoute (Pierre-Yves David)]

Closed by commit rHGc70bcaf7927b: nodemap: automatically vacuum the 
persistent nodemap when too sparse (authored by marmoute).
This revision was automatically updated to reflect the committed changes.
This revision was not accepted when it landed; it landed in state "Needs 
Review".

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D8193?vs=20897=20935

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D8193/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D8193

AFFECTED FILES
  mercurial/configitems.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -278,12 +278,12 @@
   tip-rev: 5002
   tip-node: 42bf3068c7ddfdfded53c4eb11d02266faeebfee
   data-length: 123456 (pure !)
-  data-length: 246464 (rust !)
+  data-length: 123008 (rust !)
   data-length: 123008 (no-pure no-rust !)
   data-unused: 448 (pure !)
-  data-unused: 123904 (rust !)
+  data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
-  data-unused: 50.273% (rust !)
+  data-unused: 0.000% (rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -143,13 +143,16 @@
 data_changed_count,
 data,
 ) = revlog.index.nodemap_data_incremental()
+new_length = target_docket.data_length + len(data)
+new_unused = target_docket.data_unused + data_changed_count
 if src_docket != target_docket:
 data = None
+elif new_length <= (new_unused * 10):  # under 10% of unused data
+data = None
 else:
 datafile = _rawdata_filepath(revlog, target_docket)
 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
 # store vfs
-new_length = target_docket.data_length + len(data)
 tr.add(datafile, target_docket.data_length)
 with revlog.opener(datafile, b'r+') as fd:
 fd.seek(target_docket.data_length)
@@ -162,7 +165,7 @@
 fd.flush()
 new_data = util.buffer(util.mmapread(fd, new_length))
 target_docket.data_length = new_length
-target_docket.data_unused += data_changed_count
+target_docket.data_unused = new_unused
 
 if data is None:
 # otherwise fallback to a full new export
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -407,7 +407,6 @@
 )
 # TODO before getting `persistent-nodemap` out of experimental
 #
-# * regenerate a new nodemap when the unused/total ration is to high
 # * decide for a "status" of the persistent nodemap and associated location
 #   - part of the store next the revlog itself (new requirements)
 #   - part of the cache directory



To: marmoute, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D8193: nodemap: automatically "vacuum" the persistent nodemap when too sparse

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 20897.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D8193?vs=20744=20897

BRANCH
  default

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D8193/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D8193

AFFECTED FILES
  mercurial/configitems.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -278,12 +278,12 @@
   tip-rev: 5002
   tip-node: 42bf3068c7ddfdfded53c4eb11d02266faeebfee
   data-length: 123456 (pure !)
-  data-length: 246464 (rust !)
+  data-length: 123008 (rust !)
   data-length: 123008 (no-pure no-rust !)
   data-unused: 448 (pure !)
-  data-unused: 123904 (rust !)
+  data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
-  data-unused: 50.273% (rust !)
+  data-unused: 0.000% (rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -143,13 +143,16 @@
 data_changed_count,
 data,
 ) = revlog.index.nodemap_data_incremental()
+new_length = target_docket.data_length + len(data)
+new_unused = target_docket.data_unused + data_changed_count
 if src_docket != target_docket:
 data = None
+elif new_length <= (new_unused * 10):  # under 10% of unused data
+data = None
 else:
 datafile = _rawdata_filepath(revlog, target_docket)
 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
 # store vfs
-new_length = target_docket.data_length + len(data)
 tr.add(datafile, target_docket.data_length)
 with revlog.opener(datafile, b'r+') as fd:
 fd.seek(target_docket.data_length)
@@ -162,7 +165,7 @@
 fd.flush()
 new_data = util.buffer(util.mmapread(fd, new_length))
 target_docket.data_length = new_length
-target_docket.data_unused += data_changed_count
+target_docket.data_unused = new_unused
 
 if data is None:
 # otherwise fallback to a full new export
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -407,7 +407,6 @@
 )
 # TODO before getting `persistent-nodemap` out of experimental
 #
-# * regenerate a new nodemap when the unused/total ration is to high
 # * decide for a "status" of the persistent nodemap and associated location
 #   - part of the store next the revlog itself (new requirements)
 #   - part of the cache directory



To: marmoute, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D8193: nodemap: automatically "vacuum" the persistent nodemap when too sparse

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 20744.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D8193?vs=20688=20744

BRANCH
  default

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D8193/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D8193

AFFECTED FILES
  mercurial/configitems.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -278,12 +278,12 @@
   tip-rev: 5002
   tip-node: 42bf3068c7ddfdfded53c4eb11d02266faeebfee
   data-length: 123456 (pure !)
-  data-length: 246464 (rust !)
+  data-length: 123008 (rust !)
   data-length: 123008 (no-pure no-rust !)
   data-unused: 448 (pure !)
-  data-unused: 123904 (rust !)
+  data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
-  data-unused: 50.273% (rust !)
+  data-unused: 0.000% (rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -143,13 +143,16 @@
 data_changed_count,
 data,
 ) = revlog.index.nodemap_data_incremental()
+new_length = target_docket.data_length + len(data)
+new_unused = target_docket.data_unused + data_changed_count
 if src_docket != target_docket:
 data = None
+elif (new_length // new_unused) < 10:  # under 10% of unused data
+data = None
 else:
 datafile = _rawdata_filepath(revlog, target_docket)
 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
 # store vfs
-new_length = target_docket.data_length + len(data)
 tr.add(datafile, target_docket.data_length)
 with revlog.opener(datafile, b'r+') as fd:
 fd.seek(target_docket.data_length)
@@ -162,7 +165,7 @@
 fd.flush()
 new_data = util.buffer(util.mmapread(fd, new_length))
 target_docket.data_length = new_length
-target_docket.data_unused += data_changed_count
+target_docket.data_unused = new_unused
 
 if data is None:
 # otherwise fallback to a full new export
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -407,7 +407,6 @@
 )
 # TODO before getting `persistent-nodemap` out of experimental
 #
-# * regenerate a new nodemap when the unused/total ration is to high
 # * decide for a "status" of the persistent nodemap and associated location
 #   - part of the store next the revlog itself (new requirements)
 #   - part of the cache directory



To: marmoute, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D8193: nodemap: automatically "vacuum" the persistent nodemap when too sparse

[marmoute (Pierre-Yves David)]

marmoute added a comment.
marmoute updated this revision to Diff 20688.


  rebase above latest default up to landed-D8182

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D8193?vs=20585=20688

BRANCH
  default

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D8193/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D8193

AFFECTED FILES
  mercurial/configitems.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -278,12 +278,12 @@
   tip-rev: 5002
   tip-node: 42bf3068c7ddfdfded53c4eb11d02266faeebfee
   data-length: 123456 (pure !)
-  data-length: 246464 (rust !)
+  data-length: 123008 (rust !)
   data-length: 123008 (no-pure no-rust !)
   data-unused: 448 (pure !)
-  data-unused: 123904 (rust !)
+  data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
-  data-unused: 50.273% (rust !)
+  data-unused: 0.000% (rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -143,13 +143,16 @@
 data_changed_count,
 data,
 ) = revlog.index.nodemap_data_incremental()
+new_length = target_docket.data_length + len(data)
+new_unused = target_docket.data_unused + data_changed_count
 if src_docket != target_docket:
 data = None
+elif (new_length // new_unused) < 10:  # under 10% of unused data
+data = None
 else:
 datafile = _rawdata_filepath(revlog, target_docket)
 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
 # store vfs
-new_length = target_docket.data_length + len(data)
 tr.add(datafile, target_docket.data_length)
 with revlog.opener(datafile, b'r+') as fd:
 fd.seek(target_docket.data_length)
@@ -162,7 +165,7 @@
 fd.flush()
 new_data = util.buffer(util.mmapread(fd, new_length))
 target_docket.data_length = new_length
-target_docket.data_unused += data_changed_count
+target_docket.data_unused = new_unused
 
 if data is None:
 # otherwise fallback to a full new export
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -407,7 +407,6 @@
 )
 # TODO before getting `persistent-nodemap` out of experimental
 #
-# * regenerate a new nodemap when the unused/total ration is to high
 # * decide for a "status" of the persistent nodemap and associated location
 #   - part of the store next the revlog itself (new requirements)
 #   - part of the cache directory



To: marmoute, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D8193: nodemap: automatically "vacuum" the persistent nodemap when too sparse

[marmoute (Pierre-Yves David)]

marmoute added a comment.
marmoute updated this revision to Diff 20585.


  eventless rebase past D8255 

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D8193?vs=20396=20585

BRANCH
  default

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D8193/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D8193

AFFECTED FILES
  mercurial/configitems.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -278,12 +278,12 @@
   tip-rev: 5002
   tip-node: 42bf3068c7ddfdfded53c4eb11d02266faeebfee
   data-length: 123456 (pure !)
-  data-length: 246464 (rust !)
+  data-length: 123008 (rust !)
   data-length: 123008 (no-pure no-rust !)
   data-unused: 448 (pure !)
-  data-unused: 123904 (rust !)
+  data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
-  data-unused: 50.273% (rust !)
+  data-unused: 0.000% (rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -143,13 +143,16 @@
 data_changed_count,
 data,
 ) = revlog.index.nodemap_data_incremental()
+new_length = target_docket.data_length + len(data)
+new_unused = target_docket.data_unused + data_changed_count
 if src_docket != target_docket:
 data = None
+elif (new_length // new_unused) < 10:  # under 10% of unused data
+data = None
 else:
 datafile = _rawdata_filepath(revlog, target_docket)
 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
 # store vfs
-new_length = target_docket.data_length + len(data)
 tr.add(datafile, target_docket.data_length)
 with revlog.opener(datafile, b'r+') as fd:
 fd.seek(target_docket.data_length)
@@ -162,7 +165,7 @@
 fd.flush()
 new_data = util.buffer(util.mmapread(fd, new_length))
 target_docket.data_length = new_length
-target_docket.data_unused += data_changed_count
+target_docket.data_unused = new_unused
 
 if data is None:
 # otherwise fallback to a full new export
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -407,7 +407,6 @@
 )
 # TODO before getting `persistent-nodemap` out of experimental
 #
-# * regenerate a new nodemap when the unused/total ration is to high
 # * decide for a "status" of the persistent nodemap and associated location
 #   - part of the store next the revlog itself (new requirements)
 #   - part of the cache directory



To: marmoute, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

D8193: nodemap: automatically "vacuum" the persistent nodemap when too sparse

[marmoute (Pierre-Yves David)]

marmoute created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  We arbitrarily pick "10%" as the threshold.

REPOSITORY
  rHG Mercurial

BRANCH
  default

REVISION DETAIL
  https://phab.mercurial-scm.org/D8193

AFFECTED FILES
  mercurial/configitems.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -278,12 +278,12 @@
   tip-rev: 5002
   tip-node: 42bf3068c7ddfdfded53c4eb11d02266faeebfee
   data-length: 123456 (pure !)
-  data-length: 246464 (rust !)
+  data-length: 123008 (rust !)
   data-length: 123008 (no-pure no-rust !)
   data-unused: 448 (pure !)
-  data-unused: 123904 (rust !)
+  data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
-  data-unused: 50.273% (rust !)
+  data-unused: 0.000% (rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -143,13 +143,16 @@
 data_changed_count,
 data,
 ) = revlog.index.nodemap_data_incremental()
+new_length = target_docket.data_length + len(data)
+new_unused = target_docket.data_unused + data_changed_count
 if src_docket != target_docket:
 data = None
+elif (new_length // new_unused) < 10:  # under 10% of unused data
+data = None
 else:
 datafile = _rawdata_filepath(revlog, target_docket)
 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
 # store vfs
-new_length = target_docket.data_length + len(data)
 tr.add(datafile, target_docket.data_length)
 with revlog.opener(datafile, b'r+') as fd:
 fd.seek(target_docket.data_length)
@@ -162,7 +165,7 @@
 fd.flush()
 new_data = util.buffer(util.mmapread(fd, new_length))
 target_docket.data_length = new_length
-target_docket.data_unused += data_changed_count
+target_docket.data_unused = new_unused
 
 if data is None:
 # otherwise fallback to a full new export
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -407,7 +407,6 @@
 )
 # TODO before getting `persistent-nodemap` out of experimental
 #
-# * regenerate a new nodemap when the unused/total ration is to high
 # * decide for a "status" of the persistent nodemap and associated location
 #   - part of the store next the revlog itself (new requirements)
 #   - part of the cache directory



To: marmoute, #hg-reviewers
Cc: mercurial-devel
___
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel