Patchwork [2,of,3,cg3,v2] changegroup: add flags field to cg3 delta header

login
register
mail settings
Submitter Augie Fackler
Date Dec. 18, 2015, 3:13 p.m.
Message ID <a01c9a068990ef23d5ed.1450451618@arthedain.pit.corp.google.com>
Download mbox | patch
Permalink /patch/12153/
State Accepted
Headers show

Comments

Augie Fackler - Dec. 18, 2015, 3:13 p.m.
# HG changeset patch
# User Mike Edgar <adgar@google.com>
# Date 1450126512 18000
#      Mon Dec 14 15:55:12 2015 -0500
# Node ID a01c9a068990ef23d5edb910620753179ea28379
# Parent  ac0ef58a23e9e7d44053b8995d0d4203dab3b3b4
# EXP-Topic cg3
changegroup: add flags field to cg3 delta header

This lets revlog flags be transmitted over the wire. Right now this is
useful for censored nodes and for narrowhg's ellipsis nodes.

Patch

diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -32,6 +32,7 @@  from . import (
 
 _CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s"
 _CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s"
+_CHANGEGROUPV3_DELTA_HEADER = ">20s20s20s20s20sH"
 
 def readexactly(stream, n):
     '''read n bytes from stream.read and abort if less was available'''
@@ -246,7 +247,8 @@  class cg1unpacker(object):
             deltabase = p1
         else:
             deltabase = prevnode
-        return node, p1, p2, deltabase, cs
+        flags = 0
+        return node, p1, p2, deltabase, cs, flags
 
     def deltachunk(self, prevnode):
         l = self._chunklength()
@@ -255,9 +257,9 @@  class cg1unpacker(object):
         headerdata = readexactly(self._stream, self.deltaheadersize)
         header = struct.unpack(self.deltaheader, headerdata)
         delta = readexactly(self._stream, l - self.deltaheadersize)
-        node, p1, p2, deltabase, cs = self._deltaheader(header, prevnode)
+        node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
         return {'node': node, 'p1': p1, 'p2': p2, 'cs': cs,
-                'deltabase': deltabase, 'delta': delta}
+                'deltabase': deltabase, 'delta': delta, 'flags': flags}
 
     def getchunks(self):
         """returns all the chunks contains in the bundle
@@ -296,6 +298,7 @@  class cg1unpacker(object):
         # no new manifest will be created and the manifest group will
         # be empty during the pull
         self.manifestheader()
+        repo.manifest.narrowdebug = repo.ui.warn
         repo.manifest.addgroup(self, revmap, trp)
         repo.ui.progress(_('manifests'), None)
 
@@ -495,16 +498,24 @@  class cg2unpacker(cg1unpacker):
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, deltabase, cs = headertuple
-        return node, p1, p2, deltabase, cs
+        flags = 0
+        return node, p1, p2, deltabase, cs, flags
 
 class cg3unpacker(cg2unpacker):
     """Unpacker for cg3 streams.
 
-    cg3 streams add support for exchanging treemanifests, so the only
-    thing that changes is the version number.
+    cg3 streams add support for exchanging treemanifests and revlog
+    flags, so the only changes from cg2 are the delta header and
+    version number.
     """
+    deltaheader = _CHANGEGROUPV3_DELTA_HEADER
+    deltaheadersize = struct.calcsize(deltaheader)
     version = '03'
 
+    def _deltaheader(self, headertuple, prevnode):
+        node, p1, p2, deltabase, cs, flags = headertuple
+        return node, p1, p2, deltabase, cs, flags
+
 class headerlessfixup(object):
     def __init__(self, fh, h):
         self._h = h
@@ -841,14 +852,16 @@  class cg1packer(object):
             delta = revlog.revdiff(base, rev)
         p1n, p2n = revlog.parents(node)
         basenode = revlog.node(base)
-        meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode)
+        flags = revlog.flags(rev)
+        meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
         meta += prefix
         l = len(meta) + len(delta)
         yield chunkheader(l)
         yield meta
         yield delta
-    def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
+    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
         # do nothing with basenode, it is implicitly the previous one in HG10
+        # do nothing with flags, it is implicitly 0 for cg1 and cg2
         return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
 
 class cg2packer(cg1packer):
@@ -871,11 +884,13 @@  class cg2packer(cg1packer):
             return prev
         return dp
 
-    def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
+    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
+        # Do nothing with flags, it is implicitly 0 in cg1 and cg2
         return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode)
 
 class cg3packer(cg2packer):
     version = '03'
+    deltaheader = _CHANGEGROUPV3_DELTA_HEADER
 
     def _packmanifests(self, mfnodes, tmfnodes, lookuplinknode):
         # Note that debug prints are super confusing in this code, as
@@ -894,6 +909,9 @@  class cg3packer(cg2packer):
             for chunk in self.group(nodes, dirlog(name), nodes.get):
                 yield chunk
 
+    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
+        return struct.pack(
+            self.deltaheader, node, p1n, p2n, basenode, linknode, flags)
 
 packermap = {'01': (cg1packer, cg1unpacker),
              # cg2 adds support for exchanging generaldelta
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -1557,6 +1557,7 @@  class revlog(object):
                 cs = chunkdata['cs']
                 deltabase = chunkdata['deltabase']
                 delta = chunkdata['delta']
+                flags = chunkdata['flags'] or REVIDX_DEFAULT_FLAGS
 
                 content.append(node)
 
@@ -1587,8 +1588,7 @@  class revlog(object):
                         raise error.CensoredBaseError(self.indexfile,
                                                       self.node(baserev))
 
-                flags = REVIDX_DEFAULT_FLAGS
-                if self._peek_iscensored(baserev, delta, flush):
+                if not flags and self._peek_iscensored(baserev, delta, flush):
                     flags |= REVIDX_ISCENSORED
 
                 # We assume consumers of addrevisioncb will want to retrieve
diff --git a/mercurial/unionrepo.py b/mercurial/unionrepo.py
--- a/mercurial/unionrepo.py
+++ b/mercurial/unionrepo.py
@@ -51,6 +51,7 @@  class unionrevlog(revlog.revlog):
             rev = self.revlog2.index[rev2]
             # rev numbers - in revlog2, very different from self.rev
             _start, _csize, _rsize, base, linkrev, p1rev, p2rev, node = rev
+            flags = _start & 0xFFFF
 
             if linkmapper is None: # link is to same revlog
                 assert linkrev == rev2 # we never link back
@@ -69,7 +70,7 @@  class unionrevlog(revlog.revlog):
             p1node = self.revlog2.node(p1rev)
             p2node = self.revlog2.node(p2rev)
 
-            e = (None, None, None, base,
+            e = (flags, None, None, base,
                  link, self.rev(p1node), self.rev(p2node), node)
             self.index.insert(-1, e)
             self.nodemap[node] = n
diff --git a/tests/test-acl.t b/tests/test-acl.t
--- a/tests/test-acl.t
+++ b/tests/test-acl.t
@@ -119,7 +119,7 @@  Extension disabled for lack of a hook
   adding foo/file.txt revisions
   adding quux/file.py revisions
   added 3 changesets with 3 changes to 3 files
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -184,7 +184,7 @@  Extension disabled for lack of acl.sourc
   added 3 changesets with 3 changes to 3 files
   calling hook pretxnchangegroup.acl: hgext.acl.hook
   acl: changes have source "push" - skipping
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -260,7 +260,7 @@  No [acl.allow]/[acl.deny]
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -332,7 +332,7 @@  Empty [acl.allow]
   acl: acl.deny not enabled
   acl: branch access granted: "ef1ea85a6374" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" not allowed on "foo/file.txt" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -401,7 +401,7 @@  fred is allowed inside foo/
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -467,7 +467,7 @@  Empty [acl.deny]
   acl: acl.deny enabled, 0 entries for user barney
   acl: branch access granted: "ef1ea85a6374" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "barney" not allowed on "foo/file.txt" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -538,7 +538,7 @@  fred is allowed inside foo/, but not foo
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -608,7 +608,7 @@  fred is allowed inside foo/, but not foo
   acl: path access granted: "ef1ea85a6374"
   acl: branch access granted: "f9cafe1212c8" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" denied on "foo/Bar/file.txt" (changeset "f9cafe1212c8")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -675,7 +675,7 @@  fred is allowed inside foo/, but not foo
   acl: acl.deny enabled, 0 entries for user barney
   acl: branch access granted: "ef1ea85a6374" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "barney" not allowed on "foo/file.txt" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -750,7 +750,7 @@  barney is allowed everywhere
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -833,7 +833,7 @@  wilma can change files with a .txt exten
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "wilma" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -902,7 +902,7 @@  file specified by acl.config does not ex
   calling hook pretxnchangegroup.acl: hgext.acl.hook
   acl: checking access for user "barney"
   error: pretxnchangegroup.acl hook raised an exception: [Errno 2] No such file or directory: '../acl.config'
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -984,7 +984,7 @@  betty is allowed inside foo/ by a acl.co
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "betty" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -1068,7 +1068,7 @@  acl.config can set only [acl.allow]/[acl
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -1154,7 +1154,7 @@  fred is always allowed
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -1234,7 +1234,7 @@  no one is allowed inside foo/Bar/
   acl: path access granted: "ef1ea85a6374"
   acl: branch access granted: "f9cafe1212c8" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" denied on "foo/Bar/file.txt" (changeset "f9cafe1212c8")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -1313,7 +1313,7 @@  OS-level groups
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -1395,7 +1395,7 @@  OS-level groups
   acl: path access granted: "ef1ea85a6374"
   acl: branch access granted: "f9cafe1212c8" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" denied on "foo/Bar/file.txt" (changeset "f9cafe1212c8")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -1515,7 +1515,7 @@  No branch acls specified
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -1602,7 +1602,7 @@  Branch acl deny test
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
   error: pretxnchangegroup.acl hook failed: acl: user "astro" denied on branch "foobar" (changeset "e8fc755d4d82")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -1670,7 +1670,7 @@  Branch acl empty allow test
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "astro" not allowed on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -1740,7 +1740,7 @@  Branch acl allow other
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "astro" not allowed on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -1811,7 +1811,7 @@  Branch acl allow other
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -1904,7 +1904,7 @@  push foobar into the remote
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -1989,7 +1989,7 @@  Branch acl conflicting deny
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "george" denied on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -2065,7 +2065,7 @@  User 'astro' must not be denied
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -2144,7 +2144,7 @@  Non-astro users must be denied
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "george" denied on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
diff --git a/tests/test-largefiles.t b/tests/test-largefiles.t
--- a/tests/test-largefiles.t
+++ b/tests/test-largefiles.t
@@ -1103,16 +1103,16 @@  redo pull with --lfrev and check it pull
   all local heads known remotely
   6 changesets found
   uncompressed size of bundle content:
-      1333 (changelog)
-      1599 (manifests)
-       254  .hglf/large1
-       564  .hglf/large3
-       572  .hglf/sub/large4
-       182  .hglf/sub2/large6
-       182  .hglf/sub2/large7
-       212  normal1
-       457  normal3
-       465  sub/normal4
+      1345 (changelog)
+      1611 (manifests)
+       256  .hglf/large1
+       570  .hglf/large3
+       578  .hglf/sub/large4
+       184  .hglf/sub2/large6
+       184  .hglf/sub2/large7
+       214  normal1
+       463  normal3
+       471  sub/normal4
   adding changesets
   adding manifests
   adding file changes
diff --git a/tests/test-obsolete-changeset-exchange.t b/tests/test-obsolete-changeset-exchange.t
--- a/tests/test-obsolete-changeset-exchange.t
+++ b/tests/test-obsolete-changeset-exchange.t
@@ -144,7 +144,7 @@  client only pulls down 1 changeset
   adding file changes
   adding foo revisions
   added 1 changesets with 1 changes to 1 files (+1 heads)
-  bundle2-input-part: total payload size 474
+  bundle2-input-part: total payload size 480
   bundle2-input-part: "listkeys" (params: 1 mandatory) supported
   bundle2-input-part: "listkeys" (params: 1 mandatory) supported
   bundle2-input-bundle: 2 parts total
diff --git a/tests/test-phases-exchange.t b/tests/test-phases-exchange.t
--- a/tests/test-phases-exchange.t
+++ b/tests/test-phases-exchange.t
@@ -772,9 +772,9 @@  Bare push with next changeset and common
   searching for changes
   1 changesets found
   uncompressed size of bundle content:
-       192 (changelog)
-       165 (manifests)
-       131  a-H
+       194 (changelog)
+       167 (manifests)
+       133  a-H
   adding changesets
   adding manifests
   adding file changes
diff --git a/tests/test-push-warn.t b/tests/test-push-warn.t
--- a/tests/test-push-warn.t
+++ b/tests/test-push-warn.t
@@ -156,9 +156,9 @@  Specifying a revset that evaluates to nu
   searching for changes
   2 changesets found
   uncompressed size of bundle content:
-       348 (changelog)
-       326 (manifests)
-       253  foo
+       352 (changelog)
+       330 (manifests)
+       257  foo
   adding changesets
   adding manifests
   adding file changes