Patchwork [6,of,9,sparse] sparse: move function for resolving sparse matcher into core

login
register
mail settings
Submitter Gregory Szorc
Date July 6, 2017, 9:54 p.m.
Message ID <f7f5f791854ec1ee1cc8.1499378062@ubuntu-vm-main>
Download mbox | patch
Permalink /patch/22057/
State Accepted
Headers show

Comments

Gregory Szorc - July 6, 2017, 9:54 p.m.
# HG changeset patch
# User Gregory Szorc <gregory.szorc@gmail.com>
# Date 1499377907 25200
#      Thu Jul 06 14:51:47 2017 -0700
# Node ID f7f5f791854ec1ee1cc8679d7cb1193f7d7f05b6
# Parent  fa855679d5ef4af9530f9500d1b08a74202aa35a
sparse: move function for resolving sparse matcher into core

As part of the move, the function arguments changed so revs are
passed as a list instead of *args. This allows us to use keyword
arguments properly.

Since the plan is to integrate sparse into core and have it
enabled by default, we need to prepare for a sparse matcher
to always be obtained and operated on. As part of the move,
we inserted code that returns an always matcher if sparse
isn't enabled. Some callers in the sparse extension take this
into account and conditionally perform matching depending on
whether the special always matcher is seen. I /think/ this
may have sped up some operations where the extension is
installed but no sparse config is activated.

One thing I'm ensure of in this code is whether os.path.dirname()
is semantically correct. os.posixpath.dirname() (which is
exported as pathutil.dirname) might be a better choise because
all patterns should be using posix directory separators (/)
instead of Windows (\). There's an inline comment that implies
Windows was tested. So hopefully it won't be a problem. We
can improve this in a follow-up. I've added a TODO to track it.

Like the matcher implementations, this code may one day be moved
to another module - perhaps merge.py - as it is tightly coupled with
working directory updates and could benefit from tighter
integration with that code. Again, our goal is to move to core
first then better integrate later.

Patch

diff --git a/hgext/sparse.py b/hgext/sparse.py
--- a/hgext/sparse.py
+++ b/hgext/sparse.py
@@ -75,7 +75,6 @@  certain files::
 from __future__ import absolute_import
 
 import collections
-import os
 
 from mercurial.i18n import _
 from mercurial.node import nullid
@@ -88,7 +87,6 @@  from mercurial import (
     extensions,
     hg,
     localrepo,
-    match as matchmod,
     merge as mergemod,
     registrar,
     sparse,
@@ -154,22 +152,23 @@  def _setupupdates(ui):
         actions, diverge, renamedelete = orig(repo, wctx, mctx, ancestors,
                                               branchmerge, *arg, **kwargs)
 
-        if not util.safehasattr(repo, 'sparsematch'):
+        oldrevs = [pctx.rev() for pctx in wctx.parents()]
+        oldsparsematch = sparse.matcher(repo, oldrevs)
+
+        if oldsparsematch.always():
             return actions, diverge, renamedelete
 
         files = set()
         prunedactions = {}
-        oldrevs = [pctx.rev() for pctx in wctx.parents()]
-        oldsparsematch = repo.sparsematch(*oldrevs)
 
         if branchmerge:
             # If we're merging, use the wctx filter, since we're merging into
             # the wctx.
-            sparsematch = repo.sparsematch(wctx.parents()[0].rev())
+            sparsematch = sparse.matcher(repo, [wctx.parents()[0].rev()])
         else:
             # If we're updating, use the target context's filter, since we're
             # moving to the target context.
-            sparsematch = repo.sparsematch(mctx.rev())
+            sparsematch = sparse.matcher(repo, [mctx.rev()])
 
         temporaryfiles = []
         for file, action in actions.iteritems():
@@ -237,7 +236,7 @@  def _setupupdates(ui):
 
         # If we're updating to a location, clean up any stale temporary includes
         # (ex: this happens during hg rebase --abort).
-        if not branchmerge and util.safehasattr(repo, 'sparsematch'):
+        if not branchmerge and util.safehasattr(repo, 'prunetemporaryincludes'):
             repo.prunetemporaryincludes()
         return results
 
@@ -256,7 +255,7 @@  def _setupcommit(ui):
         # profiles will only have data if sparse is enabled.
         if set(profiles) & set(ctx.files()):
             origstatus = repo.status()
-            origsparsematch = repo.sparsematch()
+            origsparsematch = sparse.matcher(repo)
             _refresh(repo.ui, repo, origstatus, origsparsematch, True)
 
         if util.safehasattr(repo, 'prunetemporaryincludes'):
@@ -273,7 +272,7 @@  def _setuplog(ui):
     def _logrevs(orig, repo, opts):
         revs = orig(repo, opts)
         if opts.get('sparse'):
-            sparsematch = repo.sparsematch()
+            sparsematch = sparse.matcher(repo)
             def ctxmatch(rev):
                 ctx = repo[rev]
                 return any(f for f in ctx.files() if sparsematch(f))
@@ -355,10 +354,11 @@  def _setupdirstate(ui):
         def __get__(self, obj, type=None):
             repo = obj.repo
             origignore = self.orig.__get__(obj)
-            if not util.safehasattr(repo, 'sparsematch'):
+
+            sparsematch = sparse.matcher(repo)
+            if sparsematch.always():
                 return origignore
 
-            sparsematch = repo.sparsematch()
             if self.sparsematch != sparsematch or self.origignore != origignore:
                 self.func = sparse.unionmatcher([
                     origignore, sparse.negatematcher(sparsematch)])
@@ -376,8 +376,8 @@  def _setupdirstate(ui):
 
     # dirstate.rebuild should not add non-matching files
     def _rebuild(orig, self, parent, allfiles, changedfiles=None):
-        if util.safehasattr(self.repo, 'sparsematch'):
-            matcher = self.repo.sparsematch()
+        matcher = sparse.matcher(self.repo)
+        if not matcher.always():
             allfiles = allfiles.matches(matcher)
             if changedfiles:
                 changedfiles = [f for f in changedfiles if matcher(f)]
@@ -398,9 +398,9 @@  def _setupdirstate(ui):
     for func in editfuncs:
         def _wrapper(orig, self, *args):
             repo = self.repo
-            if util.safehasattr(repo, 'sparsematch'):
+            sparsematch = sparse.matcher(repo)
+            if not sparsematch.always():
                 dirstate = repo.dirstate
-                sparsematch = repo.sparsematch()
                 for f in args:
                     if (f is not None and not sparsematch(f) and
                         f not in dirstate):
@@ -412,72 +412,6 @@  def _setupdirstate(ui):
 
 def _wraprepo(ui, repo):
     class SparseRepo(repo.__class__):
-        def sparsematch(self, *revs, **kwargs):
-            """Returns the sparse match function for the given revs.
-
-            If multiple revs are specified, the match function is the union
-            of all the revs.
-
-            `includetemp` is used to indicate if the temporarily included file
-            should be part of the matcher.
-            """
-            if not revs or revs == (None,):
-                revs = [self.changelog.rev(node) for node in
-                    self.dirstate.parents() if node != nullid]
-
-            includetemp = kwargs.get('includetemp', True)
-            signature = sparse.configsignature(self, includetemp=includetemp)
-
-            key = '%s %s' % (str(signature), ' '.join([str(r) for r in revs]))
-
-            result = self._sparsematchercache.get(key, None)
-            if result:
-                return result
-
-            matchers = []
-            for rev in revs:
-                try:
-                    includes, excludes, profiles = sparse.patternsforrev(
-                        self, rev)
-
-                    if includes or excludes:
-                        # Explicitly include subdirectories of includes so
-                        # status will walk them down to the actual include.
-                        subdirs = set()
-                        for include in includes:
-                            dirname = os.path.dirname(include)
-                            # basename is used to avoid issues with absolute
-                            # paths (which on Windows can include the drive).
-                            while os.path.basename(dirname):
-                                subdirs.add(dirname)
-                                dirname = os.path.dirname(dirname)
-
-                        matcher = matchmod.match(self.root, '', [],
-                            include=includes, exclude=excludes,
-                            default='relpath')
-                        if subdirs:
-                            matcher = sparse.forceincludematcher(matcher,
-                                                                 subdirs)
-                        matchers.append(matcher)
-                except IOError:
-                    pass
-
-            result = None
-            if not matchers:
-                result = matchmod.always(self.root, '')
-            elif len(matchers) == 1:
-                result = matchers[0]
-            else:
-                result = sparse.unionmatcher(matchers)
-
-            if kwargs.get('includetemp', True):
-                tempincludes = sparse.readtemporaryincludes(self)
-                result = sparse.forceincludematcher(result, tempincludes)
-
-            self._sparsematchercache[key] = result
-
-            return result
-
         def prunetemporaryincludes(self):
             if repo.vfs.exists('tempsparse'):
                 origstatus = self.status()
@@ -486,7 +420,7 @@  def _wraprepo(ui, repo):
                     # Still have pending changes. Don't bother trying to prune.
                     return
 
-                sparsematch = self.sparsematch(includetemp=False)
+                sparsematch = sparse.matcher(self, includetemp=False)
                 dirstate = self.dirstate
                 actions = []
                 dropped = []
@@ -613,7 +547,7 @@  def debugsparse(ui, repo, *pats, **opts)
             wlock = repo.wlock()
             fcounts = map(
                 len,
-                _refresh(ui, repo, repo.status(), repo.sparsematch(), force))
+                _refresh(ui, repo, repo.status(), sparse.matcher(repo), force))
             _verbose_output(ui, opts, 0, 0, 0, *fcounts)
         finally:
             wlock.release()
@@ -626,7 +560,7 @@  def _config(ui, repo, pats, opts, includ
     """
     wlock = repo.wlock()
     try:
-        oldsparsematch = repo.sparsematch()
+        oldsparsematch = sparse.matcher(repo)
 
         raw = repo.vfs.tryread('sparse')
         if raw:
@@ -726,7 +660,7 @@  def _import(ui, repo, files, opts, force
             excludecount = len(excludes - aexcludes)
 
             oldstatus = repo.status()
-            oldsparsematch = repo.sparsematch()
+            oldsparsematch = sparse.matcher(repo)
             sparse.writeconfig(repo, includes, excludes, profiles)
 
             try:
@@ -746,7 +680,7 @@  def _clear(ui, repo, files, force=False)
 
         if includes or excludes:
             oldstatus = repo.status()
-            oldsparsematch = repo.sparsematch()
+            oldsparsematch = sparse.matcher(repo)
             sparse.writeconfig(repo, set(), set(), profiles)
             _refresh(ui, repo, oldstatus, oldsparsematch, force)
 
@@ -764,7 +698,7 @@  def _refresh(ui, repo, origstatus, origs
     pending.update(modified)
     pending.update(added)
     pending.update(removed)
-    sparsematch = repo.sparsematch()
+    sparsematch = sparse.matcher(repo)
     abort = False
     for file in pending:
         if not sparsematch(file):
diff --git a/mercurial/sparse.py b/mercurial/sparse.py
--- a/mercurial/sparse.py
+++ b/mercurial/sparse.py
@@ -8,12 +8,14 @@ 
 from __future__ import absolute_import
 
 import hashlib
+import os
 
 from .i18n import _
 from .node import nullid
 from . import (
     error,
     match as matchmod,
+    pycompat,
     util,
 )
 
@@ -265,3 +267,70 @@  def hashmatcher(matcher):
     sha1 = hashlib.sha1()
     sha1.update(repr(matcher))
     return sha1.hexdigest()
+
+def matcher(repo, revs=None, includetemp=True):
+    """Obtain a matcher for sparse working directories for the given revs.
+
+    If multiple revisions are specified, the matcher is the union of all
+    revs.
+
+    ``includetemp`` indicates whether to use the temporary sparse profile.
+    """
+    # If sparse isn't enabled, sparse matcher matches everything.
+    if not enabled:
+        return matchmod.always(repo.root, '')
+
+    if not revs or revs == [None]:
+        revs = [repo.changelog.rev(node)
+                for node in repo.dirstate.parents() if node != nullid]
+
+    signature = configsignature(repo, includetemp=includetemp)
+
+    key = '%s %s' % (signature, ' '.join(map(pycompat.bytestr, revs)))
+
+    result = repo._sparsematchercache.get(key)
+    if result:
+        return result
+
+    matchers = []
+    for rev in revs:
+        try:
+            includes, excludes, profiles = patternsforrev(repo, rev)
+
+            if includes or excludes:
+                # Explicitly include subdirectories of includes so
+                # status will walk them down to the actual include.
+                subdirs = set()
+                for include in includes:
+                    # TODO consider using posix path functions here so Windows
+                    # \ directory separators don't come into play.
+                    dirname = os.path.dirname(include)
+                    # basename is used to avoid issues with absolute
+                    # paths (which on Windows can include the drive).
+                    while os.path.basename(dirname):
+                        subdirs.add(dirname)
+                        dirname = os.path.dirname(dirname)
+
+                matcher = matchmod.match(repo.root, '', [],
+                                         include=includes, exclude=excludes,
+                                         default='relpath')
+                if subdirs:
+                    matcher = forceincludematcher(matcher, subdirs)
+                matchers.append(matcher)
+        except IOError:
+            pass
+
+    if not matchers:
+        result = matchmod.always(repo.root, '')
+    elif len(matchers) == 1:
+        result = matchers[0]
+    else:
+        result = unionmatcher(matchers)
+
+    if includetemp:
+        tempincludes = readtemporaryincludes(repo)
+        result = forceincludematcher(result, tempincludes)
+
+    repo._sparsematchercache[key] = result
+
+    return result