Patchwork [8,of,8] bundle2: support chunk iterator as part data

login
register
mail settings
Submitter Pierre-Yves David
Date April 11, 2014, 10:06 p.m.
Message ID <2d15cb0b1057e12872c0.1397253965@marginatus.alto.octopoid.net>
Download mbox | patch
Permalink /patch/4291/
State Accepted
Headers show

Comments

Pierre-Yves David - April 11, 2014, 10:06 p.m.
# HG changeset patch
# User Pierre-Yves David <pierre-yves.david@fb.com>
# Date 1397228656 25200
#      Fri Apr 11 08:04:16 2014 -0700
# Node ID 2d15cb0b1057e12872c01f92a9d20d4eded7fd12
# Parent  e6261a31c9a819d972f404f5c3301d6e5f5ff3f3
bundle2: support chunk iterator as part data

When the `part.data` attribute is an iterator, we assume it is an iterator of
chunks and use it.

We use a chunkbuffer to yield chunks of 4096 bytes.

The tests are updated to use this feature.

Patch

diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -159,10 +159,12 @@  from i18n import _
 _fparttypesize = '>B'
 _fpartid = '>I'
 _fpayloadsize = '>I'
 _fpartparamcount = '>BB'
 
+preferedchunksize = 4096
+
 def _makefpartparamsizes(nbparams):
     """return a struct format to read part parameter sizes
 
     The number parameters is variable so we need to build that format
     dynamically.
@@ -559,11 +561,17 @@  class part(object):
         """yield chunks of a the part payload
 
         Exists to handle the different methods to provide data to a part."""
         # we only support fixed size data now.
         # This will be improved in the future.
-        if len(self.data):
+        if util.safehasattr(self.data, 'next'):
+            buff = util.chunkbuffer(self.data)
+            chunk = buff.read(preferedchunksize)
+            while chunk:
+                yield chunk
+                chunk = buff.read(preferedchunksize)
+        elif len(self.data):
             yield self.data
 
 @parthandler('changegroup')
 def handlechangegroup(op, inpart):
     """apply a changegroup part on the repo
diff --git a/tests/test-bundle2.t b/tests/test-bundle2.t
--- a/tests/test-bundle2.t
+++ b/tests/test-bundle2.t
@@ -64,15 +64,11 @@  Create an extension to test bundle2 API
   >             bundled = repo.revs('%ld::%ld', revs, revs)
   >             headmissing = [c.node() for c in repo.set('heads(%ld)', revs)]
   >             headcommon  = [c.node() for c in repo.set('parents(%ld) - %ld', revs, revs)]
   >             outgoing = discovery.outgoing(repo.changelog, headcommon, headmissing)
   >             cg = changegroup.getlocalbundle(repo, 'test:bundle2', outgoing, None)
-  >             assert cg is not None
-  >             # make me lazy later
-  >             tempname = changegroup.writebundle(cg, None, 'HG10UN')
-  >             data = open(tempname).read()
-  >             part = bundle2.part('changegroup', data=data)
+  >             part = bundle2.part('changegroup', data=cg.getchunks())
   >             bundler.addpart(part)
   > 
   >     if opts['parts']:
   >        part = bundle2.part('test:empty')
   >        bundler.addpart(part)
@@ -583,10 +579,14 @@  Support for changegroup
   list of changesets:
   32af7686d403cf45b5d95f2d70cebea587ac806a
   9520eea781bcca16c1e15acc0ba14335a0e8e5ba
   eea13746799a9e0bfd88f29d3c2e9dc9389f524f
   02de42196ebee42ef284b6780a87cdc96e8eaab6
+  start emission of HG20 stream
+  bundle parameter: 
+  start of parts
+  bundle part: "changegroup"
   bundling: 1/4 changesets (25.00%)
   bundling: 2/4 changesets (50.00%)
   bundling: 3/4 changesets (75.00%)
   bundling: 4/4 changesets (100.00%)
   bundling: 1/4 manifests (25.00%)
@@ -594,14 +594,10 @@  Support for changegroup
   bundling: 3/4 manifests (75.00%)
   bundling: 4/4 manifests (100.00%)
   bundling: D 1/3 files (33.33%)
   bundling: E 2/3 files (66.67%)
   bundling: H 3/3 files (100.00%)
-  start emission of HG20 stream
-  bundle parameter: 
-  start of parts
-  bundle part: "changegroup"
   end of bundle
 
   $ cat ../rev.hg2
   HG20\x00\x00\x00\x12\x0bchangegroup\x00\x00\x00\x00\x00\x00\x00\x00\x06\x19HG10UN\x00\x00\x00\xa42\xafv\x86\xd4\x03\xcfE\xb5\xd9_-p\xce\xbe\xa5\x87\xac\x80j_\xdd\xd9\x89W\xc8\xa5JMCm\xfe\x1d\xa9\xd8\x7f!\xa1\xb9{\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\xafv\x86\xd4\x03\xcfE\xb5\xd9_-p\xce\xbe\xa5\x87\xac\x80j\x00\x00\x00\x00\x00\x00\x00)\x00\x00\x00)6e1f4c47ecb533ffd0c8e52cdc88afb6cd39e20c (esc)
   \x00\x00\x00f\x00\x00\x00h\x00\x00\x00\x02D (esc)