Patchwork D10028: sidedata-exchange: add `wanted_sidedata` and `sidedata_computers` to repos

login
register
mail settings
Submitter phabricator
Date Feb. 19, 2021, 11:17 a.m.
Message ID <differential-rev-PHID-DREV-pkvvdtq2y54qxhdjim5x-req@mercurial-scm.org>
Download mbox | patch
Permalink /patch/48343/
State Superseded
Headers show

Comments

phabricator - Feb. 19, 2021, 11:17 a.m.
Alphare created this revision.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.

REVISION SUMMARY
  Each repo will advertise the sidedata categories it requires (categories being
  unique and canonical), and have a set of "computers", functions to generate
  sidedata from `(repo, revlog, rev, previous_sidedata)`, for a given category.
  The set of computers can be a superset of the set of the wanted categories, but
  not smaller: repos are expected to be coherent in their handling of sidedata.

REPOSITORY
  rHG Mercurial

BRANCH
  default

REVISION DETAIL
  https://phab.mercurial-scm.org/D10028

AFFECTED FILES
  mercurial/bundle2.py
  mercurial/changegroup.py
  mercurial/exchange.py
  mercurial/interfaces/repository.py
  mercurial/localrepo.py
  mercurial/metadata.py
  mercurial/statichttprepo.py
  tests/test-check-interfaces.py

CHANGE DETAILS




To: Alphare, #hg-reviewers
Cc: mercurial-patches, mercurial-devel

Patch

diff --git a/tests/test-check-interfaces.py b/tests/test-check-interfaces.py
--- a/tests/test-check-interfaces.py
+++ b/tests/test-check-interfaces.py
@@ -85,6 +85,7 @@ 
 class dummyrepo(object):
     def __init__(self):
         self.ui = uimod.ui()
+        self._wanted_sidedata = set()
 
     def filtered(self, name):
         pass
diff --git a/mercurial/statichttprepo.py b/mercurial/statichttprepo.py
--- a/mercurial/statichttprepo.py
+++ b/mercurial/statichttprepo.py
@@ -172,6 +172,7 @@ 
         self.names = namespaces.namespaces()
         self.filtername = None
         self._extrafilterid = None
+        self._wanted_sidedata = set()
 
         try:
             requirements = set(self.vfs.read(b'requires').splitlines())
diff --git a/mercurial/metadata.py b/mercurial/metadata.py
--- a/mercurial/metadata.py
+++ b/mercurial/metadata.py
@@ -18,6 +18,7 @@ 
 from . import (
     error,
     pycompat,
+    requirements as requirementsmod,
     util,
 )
 
@@ -804,6 +805,27 @@ 
     return encode_files_sidedata(files), files.has_copies_info
 
 
+def copies_sidedata_computer(repo, revlog, rev, existing_sidedata):
+    return _getsidedata(repo, rev)[0]
+
+
+def get_sidedata_spec_from_requirements(reqs):
+    wanted = set()
+    computers = {}
+
+    if requirementsmod.COPIESSDC_REQUIREMENT in reqs:
+        # We don't go through the extension helper methods, because this is
+        # created before extensions are run on `reposetup`.
+        wanted.add(pycompat.bytestr(sidedatamod.SD_FILES))
+        computers[b"changelog"] = {
+            pycompat.bytestr(sidedatamod.SD_FILES): (
+                (sidedatamod.SD_FILES,),
+                copies_sidedata_computer,
+            )
+        }
+    return wanted, computers
+
+
 def getsidedataadder(srcrepo, destrepo):
     use_w = srcrepo.ui.configbool(b'experimental', b'worker.repository-upgrade')
     if pycompat.iswindows or not use_w:
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -49,6 +49,7 @@ 
     match as matchmod,
     mergestate as mergestatemod,
     mergeutil,
+    metadata as metadatamod,
     namespaces,
     narrowspec,
     obsolete,
@@ -270,6 +271,11 @@ 
             caps = moderncaps.copy()
         self._repo = repo.filtered(b'served')
         self.ui = repo.ui
+
+        if repo._wanted_sidedata:
+            formatted = bundle2.format_remote_wanted_sidedata(repo)
+            caps.add(b'exp-wanted-sidedata=' + formatted)
+
         self._caps = repo._restrictcapabilities(caps)
 
     # Begin of _basepeer interface.
@@ -1393,6 +1399,12 @@ 
         if requirementsmod.COPIESSDC_REQUIREMENT in self.requirements:
             self.filecopiesmode = b'changeset-sidedata'
 
+        wanted, computers = metadatamod.get_sidedata_spec_from_requirements(
+            self.requirements
+        )
+        self._wanted_sidedata = wanted
+        self._sidedata_computers = computers
+
     def _getvfsward(self, origfunc):
         """build a ward for self.vfs"""
         rref = weakref.ref(self)
@@ -3327,6 +3339,22 @@ 
             fp.close()
         return self.pathto(fp.name[len(self.root) + 1 :])
 
+    def register_wanted_sidedata(self, category):
+        self._wanted_sidedata.add(pycompat.bytestr(category))
+
+    def register_sidedata_computer(self, kind, category, keys, computer):
+        if kind not in (b"changelog", b"manifest", b"filelog"):
+            msg = _(b"unexpected revlog kind '%s'.")
+            raise error.ProgrammingError(msg % kind)
+        category = pycompat.bytestr(category)
+        if category in self._sidedata_computers.get(kind, []):
+            msg = _(
+                b"cannot register a sidedata computer twice for category '%s'."
+            )
+            raise error.ProgrammingError(msg % category)
+        self._sidedata_computers.setdefault(kind, {})
+        self._sidedata_computers[kind][category] = (keys, computer)
+
 
 # used to avoid circular references so destructors work
 def aftertrans(files):
diff --git a/mercurial/interfaces/repository.py b/mercurial/interfaces/repository.py
--- a/mercurial/interfaces/repository.py
+++ b/mercurial/interfaces/repository.py
@@ -1832,6 +1832,12 @@ 
     def savecommitmessage(text):
         pass
 
+    def register_sidedata_computer(kind, category, keys, computer):
+        pass
+
+    def register_wanted_sidedata(category):
+        pass
+
 
 class completelocalrepository(
     ilocalrepositorymain, ilocalrepositoryfilestorage
diff --git a/mercurial/exchange.py b/mercurial/exchange.py
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -420,7 +420,18 @@ 
                 b'unbundle wire protocol command'
             )
         )
-
+    for category in sorted(bundle2.read_remote_wanted_sidedata(pushop.remote)):
+        for kind, computers in repo._sidedata_computers.items():
+            if computers.get(category):
+                break
+        else:
+            raise error.Abort(
+                _(
+                    b'cannot push: required sidedata category not supported'
+                    b" by this client: '%s'"
+                )
+                % pycompat.bytestr(category)
+            )
     # get lock as we might write phase data
     wlock = lock = None
     try:
@@ -865,8 +876,15 @@ 
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
+
+    remote_sidedata = bundle2.read_remote_wanted_sidedata(pushop.remote)
     cgstream = changegroup.makestream(
-        pushop.repo, pushop.outgoing, version, b'push'
+        pushop.repo,
+        pushop.outgoing,
+        version,
+        b'push',
+        bundlecaps=b2caps,
+        remote_sidedata=remote_sidedata,
     )
     cgpart = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
@@ -1607,6 +1625,21 @@ 
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
+    for category in repo._wanted_sidedata:
+        for kind, computers in repo._sidedata_computers.items():
+            if computers.get(category):
+                break
+        else:
+            # This should never happen since repos are supposed to be able to
+            # generate the sidedata they require.
+            raise error.ProgrammingError(
+                _(
+                    b'cannot pull: required sidedata category not supported'
+                    b" by this client: '%s'"
+                )
+                % pycompat.bytestr(category)
+            )
+
     pullop.trmanager = transactionmanager(repo, b'pull', remote.url())
     wlock = util.nullcontextmanager()
     if not bookmod.bookmarksinstore(repo):
@@ -1820,6 +1853,10 @@ 
             pullop.stepsdone.add(b'obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
 
+    remote_sidedata = bundle2.read_remote_wanted_sidedata(pullop.remote)
+    if remote_sidedata:
+        kwargs[b'remote_sidedata'] = remote_sidedata
+
     with pullop.remote.commandexecutor() as e:
         args = dict(kwargs)
         args[b'source'] = b'pull'
@@ -2388,6 +2425,8 @@ 
 
     if b'exp-sidedata-flag' in repo.requirements:
         part.addparam(b'exp-sidedata', b'1')
+        sidedata = bundle2.format_remote_wanted_sidedata(repo)
+        part.addparam(b'exp-wanted-sidedata', sidedata)
 
     if (
         kwargs.get('narrow', False)
diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -950,6 +950,7 @@ 
         if bundlecaps is None:
             bundlecaps = set()
         self._bundlecaps = bundlecaps
+        self._remote_sidedata = remote_sidedata or set()
         self._isshallow = shallow
         self._fullclnodes = fullnodes
 
diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -1810,6 +1810,26 @@ 
     return params
 
 
+def format_remote_wanted_sidedata(repo):
+    wanted = b""
+    if repo._wanted_sidedata:
+        wanted = b','.join(
+            pycompat.bytestr(c) for c in sorted(repo._wanted_sidedata)
+        )
+    return wanted
+
+
+def read_remote_wanted_sidedata(remote):
+    sidedata_categories = remote.capable(b'exp-wanted-sidedata')
+    return read_wanted_sidedata(sidedata_categories)
+
+
+def read_wanted_sidedata(formatted):
+    if formatted:
+        return set(formatted.split(b','))
+    return set()
+
+
 def addpartbundlestream2(bundler, repo, **kwargs):
     if not kwargs.get('stream', False):
         return
@@ -1955,6 +1975,7 @@ 
         b'version',
         b'nbchanges',
         b'exp-sidedata',
+        b'exp-wanted-sidedata',
         b'treemanifest',
         b'targetphase',
     ),
@@ -1997,6 +2018,10 @@ 
     targetphase = inpart.params.get(b'targetphase')
     if targetphase is not None:
         extrakwargs['targetphase'] = int(targetphase)
+
+    remote_sidedata = inpart.params.get(b'exp-wanted-sidedata')
+    extrakwargs['sidedata_categories'] = read_wanted_sidedata(remote_sidedata)
+
     ret = _processchangegroup(
         op,
         cg,
@@ -2557,5 +2582,7 @@ 
             part.addparam(b'treemanifest', b'1')
         if b'exp-sidedata-flag' in repo.requirements:
             part.addparam(b'exp-sidedata', b'1')
+            wanted = format_remote_wanted_sidedata(repo)
+            part.addparam(b'exp-wanted-sidedata', wanted)
 
     return bundler