Patchwork D3243: httppeer: support protocol upgrade

login
register
mail settings
Submitter phabricator
Date April 11, 2018, 5:02 p.m.
Message ID <95640cd61e8b88c84947e7becb73986c@localhost.localdomain>
Download mbox | patch
Permalink /patch/30729/
State Not Applicable
Headers show

Comments

phabricator - April 11, 2018, 5:02 p.m.
This revision was automatically updated to reflect the committed changes.
Closed by commit rHG8a73132214a3: httppeer: support protocol upgrade (authored by indygreg, committed by ).

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D3243?vs=7961&id=8004

REVISION DETAIL
  https://phab.mercurial-scm.org/D3243

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/httppeer.py
  tests/test-http-protocol.t

CHANGE DETAILS




To: indygreg, #hg-reviewers, durin42
Cc: mercurial-devel

Patch

diff --git a/tests/test-http-protocol.t b/tests/test-http-protocol.t
--- a/tests/test-http-protocol.t
+++ b/tests/test-http-protocol.t
@@ -1,3 +1,5 @@ 
+  $ . $TESTDIR/wireprotohelpers.sh
+
   $ cat >> $HGRCPATH << EOF
   > [web]
   > push_ssl = false
@@ -236,4 +238,98 @@ 
   s>     namespaces\t\n
   s>     phases\t
 
+Client with HTTPv2 enabled advertises that and gets old capabilities response from old server
+
+  $ hg --config experimental.httppeer.advertise-v2=true --verbose debugwireproto http://$LOCALIP:$HGPORT << EOF
+  > command heads
+  > EOF
+  s>     GET /?cmd=capabilities HTTP/1.1\r\n
+  s>     Accept-Encoding: identity\r\n
+  s>     vary: X-HgProto-1,X-HgUpgrade-1\r\n
+  s>     x-hgproto-1: cbor\r\n
+  s>     x-hgupgrade-1: exp-http-v2-0001\r\n
+  s>     accept: application/mercurial-0.1\r\n
+  s>     host: $LOCALIP:$HGPORT\r\n (glob)
+  s>     user-agent: Mercurial debugwireproto\r\n
+  s>     \r\n
+  s> makefile('rb', None)
+  s>     HTTP/1.1 200 Script output follows\r\n
+  s>     Server: testing stub value\r\n
+  s>     Date: $HTTP_DATE$\r\n
+  s>     Content-Type: application/mercurial-0.1\r\n
+  s>     Content-Length: 458\r\n
+  s>     \r\n
+  s>     batch branchmap $USUAL_BUNDLE2_CAPS_SERVER$ changegroupsubset compression=$BUNDLE2_COMPRESSIONS$ getbundle httpheader=1024 httpmediatype=0.1rx,0.1tx,0.2tx known lookup pushkey streamreqs=generaldelta,revlogv1 unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
+  sending heads command
+  s>     GET /?cmd=heads HTTP/1.1\r\n
+  s>     Accept-Encoding: identity\r\n
+  s>     vary: X-HgProto-1\r\n
+  s>     x-hgproto-1: 0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull\r\n
+  s>     accept: application/mercurial-0.1\r\n
+  s>     host: $LOCALIP:$HGPORT\r\n (glob)
+  s>     user-agent: Mercurial debugwireproto\r\n
+  s>     \r\n
+  s> makefile('rb', None)
+  s>     HTTP/1.1 200 Script output follows\r\n
+  s>     Server: testing stub value\r\n
+  s>     Date: $HTTP_DATE$\r\n
+  s>     Content-Type: application/mercurial-0.1\r\n
+  s>     Content-Length: 41\r\n
+  s>     \r\n
+  s>     0000000000000000000000000000000000000000\n
+  response: b'0000000000000000000000000000000000000000\n'
+
   $ killdaemons.py
+  $ enablehttpv2 empty
+  $ hg -R empty serve -p $HGPORT -d --pid-file hg.pid
+  $ cat hg.pid > $DAEMON_PIDS
+
+Client with HTTPv2 enabled automatically upgrades if the server supports it
+
+  $ hg --config experimental.httppeer.advertise-v2=true --verbose debugwireproto http://$LOCALIP:$HGPORT << EOF
+  > command heads
+  > EOF
+  s>     GET /?cmd=capabilities HTTP/1.1\r\n
+  s>     Accept-Encoding: identity\r\n
+  s>     vary: X-HgProto-1,X-HgUpgrade-1\r\n
+  s>     x-hgproto-1: cbor\r\n
+  s>     x-hgupgrade-1: exp-http-v2-0001\r\n
+  s>     accept: application/mercurial-0.1\r\n
+  s>     host: $LOCALIP:$HGPORT\r\n (glob)
+  s>     user-agent: Mercurial debugwireproto\r\n
+  s>     \r\n
+  s> makefile('rb', None)
+  s>     HTTP/1.1 200 OK\r\n
+  s>     Server: testing stub value\r\n
+  s>     Date: $HTTP_DATE$\r\n
+  s>     Content-Type: application/mercurial-cbor\r\n
+  s>     Content-Length: 879\r\n
+  s>     \r\n
+  s>     \xa3Dapis\xa1Pexp-http-v2-0001\xa2Hcommands\xa7Eheads\xa2Dargs\xa1Jpubliconly\xf4Kpermissions\x81DpullEknown\xa2Dargs\xa1Enodes\x81HdeadbeefKpermissions\x81DpullFlookup\xa2Dargs\xa1CkeyCfooKpermissions\x81DpullGpushkey\xa2Dargs\xa4CkeyCkeyCnewCnewColdColdInamespaceBnsKpermissions\x81DpushHlistkeys\xa2Dargs\xa1InamespaceBnsKpermissions\x81DpullIbranchmap\xa2Dargs\xa0Kpermissions\x81DpullLcapabilities\xa2Dargs\xa0Kpermissions\x81DpullKcompression\x82\xa1DnameDzstd\xa1DnameDzlibGapibaseDapi/Nv1capabilitiesY\x01\xcabatch branchmap $USUAL_BUNDLE2_CAPS_SERVER$ changegroupsubset compression=$BUNDLE2_COMPRESSIONS$ getbundle httpheader=1024 httpmediatype=0.1rx,0.1tx,0.2tx known lookup pushkey streamreqs=generaldelta,revlogv1 unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
+  sending heads command
+  s>     POST /api/exp-http-v2-0001/ro/heads HTTP/1.1\r\n
+  s>     Accept-Encoding: identity\r\n
+  s>     accept: application/mercurial-exp-framing-0003\r\n
+  s>     content-type: application/mercurial-exp-framing-0003\r\n
+  s>     content-length: 20\r\n
+  s>     host: $LOCALIP:$HGPORT\r\n (glob)
+  s>     user-agent: Mercurial debugwireproto\r\n
+  s>     \r\n
+  s>     \x0c\x00\x00\x01\x00\x01\x01\x11\xa1DnameEheads
+  s> makefile('rb', None)
+  s>     HTTP/1.1 200 OK\r\n
+  s>     Server: testing stub value\r\n
+  s>     Date: $HTTP_DATE$\r\n
+  s>     Content-Type: application/mercurial-exp-framing-0003\r\n
+  s>     Transfer-Encoding: chunked\r\n
+  s>     \r\n
+  s>     1e\r\n
+  s>     \x16\x00\x00\x01\x00\x02\x01F
+  s>     \x81T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00
+  s>     \r\n
+  received frame(size=22; request=1; stream=2; streamflags=stream-begin; type=bytes-response; flags=eos|cbor)
+  s>     0\r\n
+  s>     \r\n
+  response: [[b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00']]
+
+  $ killdaemons.py
diff --git a/mercurial/httppeer.py b/mercurial/httppeer.py
--- a/mercurial/httppeer.py
+++ b/mercurial/httppeer.py
@@ -29,6 +29,7 @@ 
     util,
     wireproto,
     wireprotoframing,
+    wireprototypes,
     wireprotov2server,
 )
 
@@ -311,7 +312,8 @@ 
 
     return res
 
-def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible):
+def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible,
+                           allowcbor=False):
     # record the url we got redirected to
     respurl = pycompat.bytesurl(resp.geturl())
     if respurl.endswith(qs):
@@ -339,8 +341,19 @@ 
             % (safeurl, proto or 'no content-type', resp.read(1024)))
 
     try:
-        version = proto.split('-', 1)[1]
-        version_info = tuple([int(n) for n in version.split('.')])
+        subtype = proto.split('-', 1)[1]
+
+        # Unless we end up supporting CBOR in the legacy wire protocol,
+        # this should ONLY be encountered for the initial capabilities
+        # request during handshake.
+        if subtype == 'cbor':
+            if allowcbor:
+                return respurl, proto, resp
+            else:
+                raise error.RepoError(_('unexpected CBOR response from '
+                                        'server'))
+
+        version_info = tuple([int(n) for n in subtype.split('.')])
     except ValueError:
         raise error.RepoError(_("'%s' sent a broken Content-Type "
                                 "header (%s)") % (safeurl, proto))
@@ -361,9 +374,9 @@ 
         resp = engine.decompressorreader(resp)
     else:
         raise error.RepoError(_("'%s' uses newer protocol %s") %
-                              (safeurl, version))
+                              (safeurl, subtype))
 
-    return respurl, resp
+    return respurl, proto, resp
 
 class httppeer(wireproto.wirepeer):
     def __init__(self, ui, path, url, opener, requestbuilder, caps):
@@ -416,8 +429,8 @@ 
 
         resp = sendrequest(self.ui, self._urlopener, req)
 
-        self._url, resp = parsev1commandresponse(self.ui, self._url, cu, qs,
-                                                 resp, _compressible)
+        self._url, ct, resp = parsev1commandresponse(self.ui, self._url, cu, qs,
+                                                     resp, _compressible)
 
         return resp
 
@@ -501,17 +514,18 @@ 
 
 # TODO implement interface for version 2 peers
 class httpv2peer(object):
-    def __init__(self, ui, repourl, opener):
+    def __init__(self, ui, repourl, apipath, opener, requestbuilder,
+                 apidescriptor):
         self.ui = ui
 
         if repourl.endswith('/'):
             repourl = repourl[:-1]
 
         self.url = repourl
+        self._apipath = apipath
         self._opener = opener
-        # This is an its own attribute to facilitate extensions overriding
-        # the default type.
-        self._requestbuilder = urlreq.request
+        self._requestbuilder = requestbuilder
+        self._descriptor = apidescriptor
 
     def close(self):
         pass
@@ -540,8 +554,7 @@ 
             'pull': 'ro',
         }[permission]
 
-        url = '%s/api/%s/%s/%s' % (self.url, wireprotov2server.HTTPV2,
-                                   permission, name)
+        url = '%s/%s/%s/%s' % (self.url, self._apipath, permission, name)
 
         # TODO this should be part of a generic peer for the frame-based
         # protocol.
@@ -597,28 +610,94 @@ 
 
         return results
 
+# Registry of API service names to metadata about peers that handle it.
+#
+# The following keys are meaningful:
+#
+# init
+#    Callable receiving (ui, repourl, servicepath, opener, requestbuilder,
+#                        apidescriptor) to create a peer.
+#
+# priority
+#    Integer priority for the service. If we could choose from multiple
+#    services, we choose the one with the highest priority.
+API_PEERS = {
+    wireprototypes.HTTPV2: {
+        'init': httpv2peer,
+        'priority': 50,
+    },
+}
+
 def performhandshake(ui, url, opener, requestbuilder):
     # The handshake is a request to the capabilities command.
 
     caps = None
     def capable(x):
         raise error.ProgrammingError('should not be called')
 
+    args = {}
+
+    # The client advertises support for newer protocols by adding an
+    # X-HgUpgrade-* header with a list of supported APIs and an
+    # X-HgProto-* header advertising which serializing formats it supports.
+    # We only support the HTTP version 2 transport and CBOR responses for
+    # now.
+    advertisev2 = ui.configbool('experimental', 'httppeer.advertise-v2')
+
+    if advertisev2:
+        args['headers'] = {
+            r'X-HgProto-1': r'cbor',
+        }
+
+        args['headers'].update(
+            encodevalueinheaders(' '.join(sorted(API_PEERS)),
+                                 'X-HgUpgrade',
+                                 # We don't know the header limit this early.
+                                 # So make it small.
+                                 1024))
+
     req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
                                            capable, url, 'capabilities',
-                                           {})
+                                           args)
 
     resp = sendrequest(ui, opener, req)
 
-    respurl, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
-                                           compressible=False)
+    respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
+                                               compressible=False,
+                                               allowcbor=advertisev2)
 
     try:
-        rawcaps = resp.read()
+        rawdata = resp.read()
     finally:
         resp.close()
 
-    return respurl, set(rawcaps.split())
+    if not ct.startswith('application/mercurial-'):
+        raise error.ProgrammingError('unexpected content-type: %s' % ct)
+
+    if advertisev2:
+        if ct == 'application/mercurial-cbor':
+            try:
+                info = cbor.loads(rawdata)
+            except cbor.CBORDecodeError:
+                raise error.Abort(_('error decoding CBOR from remote server'),
+                                  hint=_('try again and consider contacting '
+                                         'the server operator'))
+
+        # We got a legacy response. That's fine.
+        elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'):
+            info = {
+                'v1capabilities': set(rawdata.split())
+            }
+
+        else:
+            raise error.RepoError(
+                _('unexpected response type from server: %s') % ct)
+    else:
+        info = {
+            'v1capabilities': set(rawdata.split())
+        }
+
+    return respurl, info
 
 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
     """Construct an appropriate HTTP peer instance.
@@ -640,9 +719,33 @@ 
 
     opener = opener or urlmod.opener(ui, authinfo)
 
-    respurl, caps = performhandshake(ui, url, opener, requestbuilder)
+    respurl, info = performhandshake(ui, url, opener, requestbuilder)
+
+    # Given the intersection of APIs that both we and the server support,
+    # sort by their advertised priority and pick the first one.
+    #
+    # TODO consider making this request-based and interface driven. For
+    # example, the caller could say "I want a peer that does X." It's quite
+    # possible that not all peers would do that. Since we know the service
+    # capabilities, we could filter out services not meeting the
+    # requirements. Possibly by consulting the interfaces defined by the
+    # peer type.
+    apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys())
 
-    return httppeer(ui, path, respurl, opener, requestbuilder, caps)
+    preferredchoices = sorted(apipeerchoices,
+                              key=lambda x: API_PEERS[x]['priority'],
+                              reverse=True)
+
+    for service in preferredchoices:
+        apipath = '%s/%s' % (info['apibase'].rstrip('/'), service)
+
+        return API_PEERS[service]['init'](ui, respurl, apipath, opener,
+                                          requestbuilder,
+                                          info['apis'][service])
+
+    # Failed to construct an API peer. Fall back to legacy.
+    return httppeer(ui, path, respurl, opener, requestbuilder,
+                    info['v1capabilities'])
 
 def instance(ui, path, create):
     if create:
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -83,6 +83,7 @@ 
     vfs as vfsmod,
     wireprotoframing,
     wireprotoserver,
+    wireprototypes,
 )
 from .utils import (
     dateutil,
@@ -2910,7 +2911,9 @@ 
 
         if opts['peer'] == 'http2':
             ui.write(_('creating http peer for wire protocol version 2\n'))
-            peer = httppeer.httpv2peer(ui, path, opener)
+            peer = httppeer.httpv2peer(
+                ui, path, 'api/%s' % wireprototypes.HTTPV2,
+                opener, httppeer.urlreq.request, {})
         elif opts['peer'] == 'raw':
             ui.write(_('using raw connection to peer\n'))
             peer = None
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -538,6 +538,9 @@ 
 coreconfigitem('experimental', 'hook-track-tags',
     default=False,
 )
+coreconfigitem('experimental', 'httppeer.advertise-v2',
+    default=False,
+)
 coreconfigitem('experimental', 'httppostargs',
     default=False,
 )