Alphare created this revision. Herald added a reviewer: indygreg. Herald added a reviewer: hg-reviewers. Herald added a subscriber: mercurial-patches.
REPOSITORY rHG Mercurial BRANCH default REVISION DETAIL https://phab.mercurial-scm.org/D10026 AFFECTED FILES hgext/remotefilelog/remotefilelog.py hgext/remotefilelog/shallowbundle.py hgext/sqlitestore.py mercurial/bundlerepo.py mercurial/changegroup.py mercurial/debugcommands.py mercurial/exchange.py mercurial/exchangev2.py mercurial/localrepo.py mercurial/revlog.py tests/test-check-interfaces.py tests/test-revlog-raw.py CHANGE DETAILS diff --git a/tests/test-revlog-raw.py b/tests/test-revlog-raw.py --- a/tests/test-revlog-raw.py +++ b/tests/test-revlog-raw.py @@ -147,6 +147,7 @@ b'flags': rlog.flags(r), b'deltabase': rlog.node(deltaparent), b'delta': rlog.revdiff(deltaparent, r), + b'sidedata': rlog.sidedata(r), } def deltaiter(self): @@ -159,10 +160,11 @@ deltabase = chunkdata[b'deltabase'] delta = chunkdata[b'delta'] flags = chunkdata[b'flags'] + sidedata = chunkdata[b'sidedata'] chain = node - yield (node, p1, p2, cs, deltabase, delta, flags) + yield (node, p1, p2, cs, deltabase, delta, flags, sidedata) def linkmap(lnode): return rlog.rev(lnode) diff --git a/tests/test-check-interfaces.py b/tests/test-check-interfaces.py --- a/tests/test-check-interfaces.py +++ b/tests/test-check-interfaces.py @@ -272,6 +272,7 @@ flags=b'', baserevisionsize=None, revision=b'', + sidedata=b'', delta=None, ) checkzobject(rd) diff --git a/mercurial/revlog.py b/mercurial/revlog.py --- a/mercurial/revlog.py +++ b/mercurial/revlog.py @@ -2512,7 +2512,7 @@ deltacomputer = deltautil.deltacomputer(self) # loop through our set of deltas for data in deltas: - node, p1, p2, linknode, deltabase, delta, flags = data + node, p1, p2, linknode, deltabase, delta, flags, sidedata = data link = linkmapper(linknode) flags = flags or REVIDX_DEFAULT_FLAGS diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py --- a/mercurial/localrepo.py +++ b/mercurial/localrepo.py @@ -313,7 +313,13 @@ ) def getbundle( - self, source, heads=None, common=None, bundlecaps=None, **kwargs + self, + source, + heads=None, + common=None, + bundlecaps=None, + remote_sidedata=None, + **kwargs ): chunks = exchange.getbundlechunks( self._repo, @@ -321,6 +327,7 @@ heads=heads, common=common, bundlecaps=bundlecaps, + remote_sidedata=remote_sidedata, **kwargs )[1] cb = util.chunkbuffer(chunks) diff --git a/mercurial/exchangev2.py b/mercurial/exchangev2.py --- a/mercurial/exchangev2.py +++ b/mercurial/exchangev2.py @@ -416,6 +416,8 @@ mdiff.trivialdiffheader(len(data)) + data, # Flags not yet supported. 0, + # Sidedata not yet supported + {}, ) cl.addgroup( @@ -495,6 +497,8 @@ delta, # Flags not yet supported. 0, + # Sidedata not yet supported. + {}, ) progress.increment() @@ -620,6 +624,8 @@ delta, # Flags not yet supported. 0, + # Sidedata not yet supported. + {}, ) progress.increment() @@ -718,6 +724,8 @@ delta, # Flags not yet supported. 0, + # Sidedata not yet supported. + {}, ) progress.increment() diff --git a/mercurial/exchange.py b/mercurial/exchange.py --- a/mercurial/exchange.py +++ b/mercurial/exchange.py @@ -2249,7 +2249,13 @@ def getbundlechunks( - repo, source, heads=None, common=None, bundlecaps=None, **kwargs + repo, + source, + heads=None, + common=None, + bundlecaps=None, + remote_sidedata=None, + **kwargs ): """Return chunks constituting a bundle's raw data. @@ -2279,7 +2285,12 @@ return ( info, changegroup.makestream( - repo, outgoing, b'01', source, bundlecaps=bundlecaps + repo, + outgoing, + b'01', + source, + bundlecaps=bundlecaps, + remote_sidedata=remote_sidedata, ), ) @@ -2303,6 +2314,7 @@ source, bundlecaps=bundlecaps, b2caps=b2caps, + remote_sidedata=remote_sidedata, **pycompat.strkwargs(kwargs) ) @@ -2325,6 +2337,7 @@ b2caps=None, heads=None, common=None, + remote_sidedata=None, **kwargs ): """add a changegroup part to the requested bundle""" @@ -2355,7 +2368,13 @@ matcher = None cgstream = changegroup.makestream( - repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher + repo, + outgoing, + version, + source, + bundlecaps=bundlecaps, + matcher=matcher, + remote_sidedata=remote_sidedata, ) part = bundler.newpart(b'changegroup', data=cgstream) diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py --- a/mercurial/debugcommands.py +++ b/mercurial/debugcommands.py @@ -346,7 +346,7 @@ def showchunks(named): ui.write(b"\n%s%s\n" % (indent_string, named)) for deltadata in gen.deltaiter(): - node, p1, p2, cs, deltabase, delta, flags = deltadata + node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata ui.write( b"%s%s %s %s %s %s %d\n" % ( @@ -372,7 +372,7 @@ raise error.Abort(_(b'use debugbundle2 for this file')) gen.changelogheader() for deltadata in gen.deltaiter(): - node, p1, p2, cs, deltabase, delta, flags = deltadata + node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata ui.write(b"%s%s\n" % (indent_string, hex(node))) diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py --- a/mercurial/changegroup.py +++ b/mercurial/changegroup.py @@ -32,10 +32,12 @@ ) from .interfaces import repository +from .revlogutils import sidedata as sidedatamod _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s") _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s") _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH") +_CHANGEGROUPV4_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sHi") LFS_REQUIREMENT = b'lfs' @@ -202,7 +204,9 @@ header = self.deltaheader.unpack(headerdata) delta = readexactly(self._stream, l - self.deltaheadersize) node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) - return (node, p1, p2, cs, deltabase, delta, flags) + # cg4 forward-compat + sidedata = {} + return (node, p1, p2, cs, deltabase, delta, flags, sidedata) def getchunks(self): """returns all the chunks contains in the bundle @@ -552,6 +556,35 @@ raise error.Abort(_(b"received dir revlog group is empty")) +class cg4unpacker(cg3unpacker): + """Unpacker for cg4 streams. + + cg4 streams add support for exchanging sidedata. + """ + + version = b'04' + + def deltachunk(self, prevnode): + res = super(cg4unpacker, self).deltachunk(prevnode) + if not res: + return res + + (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res + + sidedata_length_raw = readexactly(self._stream, 4) + (sidedata_length,) = struct.unpack(b">l", sidedata_length_raw) + + sidedata = {} + + assert sidedata_length == 0 or sidedata_length >= 4, sidedata_length + if sidedata_length > 0: + sidedata_length = sidedata_length - 4 + sidedata_raw = readexactly(self._stream, sidedata_length) + sidedata = sidedatamod.deserialize_sidedata(sidedata_raw) + + return node, p1, p2, cs, deltabase, delta, flags, sidedata + + class headerlessfixup(object): def __init__(self, fh, h): self._h = h @@ -861,6 +894,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): """Given a source repo, construct a bundler. @@ -893,6 +927,8 @@ nodes. We store this rather than the set of nodes that should be ellipsis because for very large histories we expect this to be significantly smaller. + + remote_sidedata is the set of sidedata categories wanted by the remote. """ assert oldmatcher assert matcher @@ -988,7 +1024,7 @@ for tree, deltas in it: if tree: - assert self.version == b'03' + assert self.version in (b'03', b'04') chunk = _fileheader(tree) size += len(chunk) yield chunk @@ -1394,6 +1430,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.linknode @@ -1424,6 +1461,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.basenode, d.linknode @@ -1453,6 +1491,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags @@ -1473,12 +1512,45 @@ ) +def _makecg4packer( + repo, + oldmatcher, + matcher, + bundlecaps, + ellipses=False, + shallow=False, + ellipsisroots=None, + fullnodes=None, + remote_sidedata=None, +): + builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( + d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags + ) + + return cgpacker( + repo, + oldmatcher, + matcher, + b'04', + builddeltaheader=builddeltaheader, + manifestsend=closechunk(), + bundlecaps=bundlecaps, + ellipses=ellipses, + shallow=shallow, + ellipsisroots=ellipsisroots, + fullnodes=fullnodes, + remote_sidedata=remote_sidedata, + ) + + _packermap = { b'01': (_makecg1packer, cg1unpacker), # cg2 adds support for exchanging generaldelta b'02': (_makecg2packer, cg2unpacker), # cg3 adds support for exchanging revlog flags and treemanifests b'03': (_makecg3packer, cg3unpacker), + # ch4 adds support for exchanging sidedata + b'04': (_makecg4packer, cg4unpacker), } @@ -1498,11 +1570,9 @@ # # (or even to push subset of history) needv03 = True - if b'exp-sidedata-flag' in repo.requirements: - needv03 = True - # don't attempt to use 01/02 until we do sidedata cleaning - versions.discard(b'01') - versions.discard(b'02') + has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements + if not has_revlogv2: + versions.discard(b'04') if not needv03: versions.discard(b'03') return versions @@ -1565,6 +1635,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): assert version in supportedoutgoingversions(repo) @@ -1601,6 +1672,7 @@ shallow=shallow, ellipsisroots=ellipsisroots, fullnodes=fullnodes, + remote_sidedata=remote_sidedata, ) @@ -1644,8 +1716,15 @@ fastpath=False, bundlecaps=None, matcher=None, + remote_sidedata=None, ): - bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher) + bundler = getbundler( + version, + repo, + bundlecaps=bundlecaps, + matcher=matcher, + remote_sidedata=remote_sidedata, + ) repo = repo.unfiltered() commonrevs = outgoing.common diff --git a/mercurial/bundlerepo.py b/mercurial/bundlerepo.py --- a/mercurial/bundlerepo.py +++ b/mercurial/bundlerepo.py @@ -61,7 +61,7 @@ self.repotiprev = n - 1 self.bundlerevs = set() # used by 'bundle()' revset expression for deltadata in cgunpacker.deltaiter(): - node, p1, p2, cs, deltabase, delta, flags = deltadata + node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata size = len(delta) start = cgunpacker.tell() - size diff --git a/hgext/sqlitestore.py b/hgext/sqlitestore.py --- a/hgext/sqlitestore.py +++ b/hgext/sqlitestore.py @@ -681,7 +681,16 @@ ): empty = True - for node, p1, p2, linknode, deltabase, delta, wireflags in deltas: + for ( + node, + p1, + p2, + linknode, + deltabase, + delta, + wireflags, + sidedata, + ) in deltas: storeflags = 0 if wireflags & repository.REVISION_FLAG_CENSORED: diff --git a/hgext/remotefilelog/shallowbundle.py b/hgext/remotefilelog/shallowbundle.py --- a/hgext/remotefilelog/shallowbundle.py +++ b/hgext/remotefilelog/shallowbundle.py @@ -67,7 +67,7 @@ shallowcg1packer, self, nodelist, rlog, lookup, units=units ) - def generatefiles(self, changedfiles, *args): + def generatefiles(self, changedfiles, *args, **kwargs): try: linknodes, commonrevs, source = args except ValueError: @@ -92,7 +92,9 @@ [f for f in changedfiles if not repo.shallowmatch(f)] ) - return super(shallowcg1packer, self).generatefiles(changedfiles, *args) + return super(shallowcg1packer, self).generatefiles( + changedfiles, *args, **kwargs + ) def shouldaddfilegroups(self, source): repo = self._repo @@ -176,9 +178,11 @@ repo.shallowmatch = original -def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args): +def addchangegroupfiles( + orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs +): if not shallowutil.isenabled(repo): - return orig(repo, source, revmap, trp, expectedfiles, *args) + return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs) newfiles = 0 visited = set() @@ -272,7 +276,7 @@ revisiondata = revisiondatas[(f, node)] # revisiondata: (node, p1, p2, cs, deltabase, delta, flags) - node, p1, p2, linknode, deltabase, delta, flags = revisiondata + node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata if not available(f, node, f, deltabase): continue diff --git a/hgext/remotefilelog/remotefilelog.py b/hgext/remotefilelog/remotefilelog.py --- a/hgext/remotefilelog/remotefilelog.py +++ b/hgext/remotefilelog/remotefilelog.py @@ -306,6 +306,7 @@ assumehaveparentrevisions=False, deltaprevious=False, deltamode=None, + sidedata_helpers=None, ): # we don't use any of these parameters here del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious @@ -333,6 +334,8 @@ baserevisionsize=None, revision=revision, delta=delta, + # Sidedata is not supported yet + sidedata=None, ) def revdiff(self, node1, node2): To: Alphare, indygreg, #hg-reviewers Cc: mercurial-patches, mercurial-devel _______________________________________________ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
