Patchwork D1856: wireproto: server-side support for pullbundles

login
register
mail settings
Submitter phabricator
Date Jan. 15, 2018, 12:02 a.m.
Message ID <ecbadbedc8e4896750c982ddb7cd98a3@localhost.localdomain>
Download mbox | patch
Permalink /patch/26749/
State Not Applicable
Headers show

Comments

phabricator - Jan. 15, 2018, 12:02 a.m.
joerg.sonnenberger updated this revision to Diff 4826.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D1856?vs=4816&id=4826

REVISION DETAIL
  https://phab.mercurial-scm.org/D1856

AFFECTED FILES
  mercurial/configitems.py
  mercurial/help/config.txt
  mercurial/wireproto.py
  tests/test-pull-r.t

CHANGE DETAILS




To: joerg.sonnenberger, #hg-reviewers, indygreg
Cc: indygreg, durin42, mercurial-devel

Patch

diff --git a/tests/test-pull-r.t b/tests/test-pull-r.t
--- a/tests/test-pull-r.t
+++ b/tests/test-pull-r.t
@@ -145,3 +145,59 @@ 
 
   $ cd ..
   $ killdaemons.py
+
+Test pullbundle functionality
+
+  $ cd repo
+  $ cat <<EOF > .hg/hgrc
+  > [server]
+  > pullbundle = True
+  > EOF
+  $ hg bundle --base null -r 0 .hg/0.hg
+  1 changesets found
+  $ hg bundle --base 0 -r 1 .hg/1.hg
+  1 changesets found
+  $ hg bundle --base 1 -r 2 .hg/2.hg
+  1 changesets found
+  $ cat <<EOF > .hg/pullbundles.manifest
+  > 2.hg heads=effea6de0384e684f44435651cb7bd70b8735bd4 bases=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+  > 1.hg heads=ed1b79f46b9a29f5a6efa59cf12fcfca43bead5a bases=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+  > 0.hg heads=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+  > EOF
+  $ hg serve --debug -p $HGPORT2 --pid-file=../repo.pid > ../repo-server.txt 2>&1 &
+  $ while ! grep listening ../repo-server.txt > /dev/null; do sleep 1; done
+  $ cat ../repo.pid >> $DAEMON_PIDS
+  $ cd ..
+  $ hg clone -r 0 http://localhost:$HGPORT2/ repo.pullbundle
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  new changesets bbd179dfa0a7
+  updating to branch default
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cd repo.pullbundle
+  $ hg pull -r 1
+  pulling from http://localhost:$HGPORT2/
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  new changesets ed1b79f46b9a
+  (run 'hg update' to get a working copy)
+  $ hg pull -r 2
+  pulling from http://localhost:$HGPORT2/
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files (+1 heads)
+  new changesets effea6de0384
+  (run 'hg heads' to see heads, 'hg merge' to merge)
+  $ cd ..
+  $ killdaemons.py
+  $ grep 'sending pullbundle ' repo-server.txt
+  sending pullbundle "0.hg"
+  sending pullbundle "1.hg"
+  sending pullbundle "2.hg"
diff --git a/mercurial/wireproto.py b/mercurial/wireproto.py
--- a/mercurial/wireproto.py
+++ b/mercurial/wireproto.py
@@ -831,6 +831,64 @@ 
     opts = options('debugwireargs', ['three', 'four'], others)
     return repo.debugwireargs(one, two, **pycompat.strkwargs(opts))
 
+def find_pullbundle(repo, opts, clheads, heads, common):
+    """Return a file object for the first matching pullbundle.
+
+    Pullbundles are specified in .hg/pullbundles.manifest similar to
+    clonebundles.
+    For each entry, the bundle specification is checked for compatibility:
+    - Client features vs the BUNDLESPEC.
+    - Revisions shared with the clients vs base revisions of the bundle.
+      A bundle can be applied only if all its base revisions are known by
+      the client.
+    - At least one leaf of the bundle's DAG is missing on the client.
+    - Every leaf of the bundle's DAG is part of node set the client wants.
+      E.g. do not send a bundle of all changes if the client wants only
+      one specific branch of many.
+    """
+    def decodehexstring(s):
+        return set([h.decode('hex') for h in s.split(';')])
+
+    manifest = repo.vfs.tryread('pullbundles.manifest')
+    if not manifest:
+        return None
+    res = exchange.parseclonebundlesmanifest(repo, manifest)
+    res = exchange.filterclonebundleentries(repo, res)
+    if not res:
+        return None
+    cl = repo.changelog
+    heads_anc = cl.ancestors([cl.rev(rev) for rev in heads], inclusive=True)
+    common_anc = cl.ancestors([cl.rev(rev) for rev in common], inclusive=True)
+    for entry in res:
+        if 'heads' in entry:
+            try:
+                bundle_heads = decodehexstring(entry['heads'])
+            except TypeError:
+                # Bad heads entry
+                continue
+            if bundle_heads.issubset(common):
+                continue # Nothing new
+            if all(cl.rev(rev) in common_anc for rev in bundle_heads):
+                continue # Still nothing new
+            if any(cl.rev(rev) not in heads_anc for rev in bundle_heads):
+                continue
+        if 'bases' in entry:
+            try:
+                bundle_bases = decodehexstring(entry['bases'])
+            except TypeError:
+                # Bad bases entry
+                continue
+            if not all(cl.rev(rev) in common_anc for rev in bundle_bases):
+                continue
+        path = entry['URL']
+        repo.ui.debug('sending pullbundle "%s"\n' % path)
+        try:
+            return repo.vfs.open(path)
+        except IOError:
+            repo.ui.debug('pullbundle "%s" not accessible\n' % path)
+            continue
+    return None
+
 @wireprotocommand('getbundle', '*')
 def getbundle(repo, proto, others):
     opts = options('getbundle', gboptsmap.keys(), others)
@@ -861,12 +919,20 @@ 
                               hint=bundle2requiredhint)
 
     try:
+        clheads = set(repo.changelog.heads())
+        heads = set(opts.get('heads', set()))
+        common = set(opts.get('common', set()))
+        common.discard(nullid)
+
+        if repo.ui.configbool('server', 'pullbundle'):
+            # Check if a pre-built bundle covers this request.
+            bundle = find_pullbundle(repo, opts, clheads, heads, common)
+            if bundle:
+                return streamres(gen=util.filechunkiter(bundle),
+                                 prefer_uncompressed=True)
+
         if repo.ui.configbool('server', 'disablefullbundle'):
             # Check to see if this is a full clone.
-            clheads = set(repo.changelog.heads())
-            heads = set(opts.get('heads', set()))
-            common = set(opts.get('common', set()))
-            common.discard(nullid)
             if not common and clheads == heads:
                 raise error.Abort(
                     _('server has pull-based clones disabled'),
diff --git a/mercurial/help/config.txt b/mercurial/help/config.txt
--- a/mercurial/help/config.txt
+++ b/mercurial/help/config.txt
@@ -1772,6 +1772,14 @@ 
     are highly recommended. Partial clones will still be allowed.
     (default: False)
 
+``pullbundle``
+    When set, the server will check pullbundle.manifest for bundles
+    covering the requested heads and common nodes. The first matching
+    entry will be streamed to the client.
+
+    For HTTP transport, the stream will still use zlib compression
+    for older clients.
+
 ``concurrent-push-mode``
     Level of allowed race condition between two pushing clients.
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -877,6 +877,9 @@ 
 coreconfigitem('server', 'disablefullbundle',
     default=False,
 )
+coreconfigitem('server', 'pullbundle',
+    default=False,
+)
 coreconfigitem('server', 'maxhttpheaderlen',
     default=1024,
 )