Patchwork [7,of,9] verify: check directory manifests

login
register
mail settings
Submitter Martin von Zweigbergk
Date Feb. 12, 2016, 10:42 p.m.
Message ID <CAESOdVDTMQdw+00p4PM504L_YW5z1eUhkh2rXcJrk2uxt7+DDg@mail.gmail.com>
Download mbox | patch
Permalink /patch/13161/
State Superseded
Commit 7297e9e13a8a5132ba1bb78c51cf8ab962048df8
Headers show

Comments

Martin von Zweigbergk - Feb. 12, 2016, 10:42 p.m.
# HG changeset patch
# User Martin von Zweigbergk <martinvonz@google.com>
# Date 1454908404 28800
#      Sun Feb 07 21:13:24 2016 -0800
# Node ID 346182e9ee70ecfdd387630eb209f85a73f60996
# Parent  9535b8d9a1d1c3e3b47661d8a046a41af3581046
verify: check directory manifests
Pierre-Yves David - Feb. 13, 2016, 4:09 p.m.
On 02/12/2016 10:42 PM, Martin von Zweigbergk wrote:
> # HG changeset patch
> # User Martin von Zweigbergk <martinvonz@google.com>
> # Date 1454908404 28800
> #      Sun Feb 07 21:13:24 2016 -0800
> # Node ID 346182e9ee70ecfdd387630eb209f85a73f60996
> # Parent  9535b8d9a1d1c3e3b47661d8a046a41af3581046
> verify: check directory manifests

Pushed up to here, there was multiple small reject that I was able to 
fix up to here. Consider setting up a pull url next time you have a 
large series.

Patch 8 and 9 have very large reject, please resend.

Patch

diff --git a/mercurial/manifest.py b/mercurial/manifest.py
--- a/mercurial/manifest.py
+++ b/mercurial/manifest.py
@@ -325,6 +325,9 @@ 
     def iteritems(self):
         return (x[:2] for x in self._lm.iterentries())

+    def iterentries(self):
+        return self._lm.iterentries()
+
     def text(self, usemanifestv2=False):
         if usemanifestv2:
             return _textv2(self._lm.iterentries())
@@ -920,7 +923,8 @@ 
         return manifestdict(data)

     def dirlog(self, dir):
-        assert self._treeondisk
+        if dir:
+            assert self._treeondisk
         if dir not in self._dirlogcache:
             self._dirlogcache[dir] = manifest(self.opener, dir,
                                               self._dirlogcache)
@@ -945,6 +949,16 @@ 
         d = mdiff.patchtext(self.revdiff(self.deltaparent(r), r))
         return self._newmanifest(d)

+    def readshallowdelta(self, node):
+        if not self._treeondisk:
+            return self.readdelta(node)
+        if self._usemanifestv2:
+            raise error.Abort(
+                "readshallowdelta() not implemented for manifestv2")
+        r = self.rev(node)
+        d = mdiff.patchtext(self.revdiff(self.deltaparent(r), r))
+        return manifestdict(d)
+
     def readfast(self, node):
         '''use the faster of readdelta or read

diff --git a/mercurial/verify.py b/mercurial/verify.py
--- a/mercurial/verify.py
+++ b/mercurial/verify.py
@@ -197,46 +197,73 @@ 
         ui.progress(_('checking'), None)
         return mflinkrevs, filelinkrevs

-    def _verifymanifest(self, mflinkrevs):
+    def _verifymanifest(self, mflinkrevs, dir=""):
         repo = self.repo
         ui = self.ui
-        mf = self.repo.manifest
+        mf = self.repo.manifest.dirlog(dir)

-        ui.status(_("checking manifests\n"))
+        if not dir:
+            self.ui.status(_("checking manifests\n"))
+
         filenodes = {}
+        subdirnodes = {}
         seen = {}
         label = "manifest"
+        if dir:
+            label = dir
         if self.refersmf:
             # Do not check manifest if there are only changelog entries with
             # null manifests.
             self.checklog(mf, label, 0)
         total = len(mf)
         for i in mf:
-            ui.progress(_('checking'), i, total=total, unit=_('manifests'))
+            if not dir:
+                ui.progress(_('checking'), i, total=total, unit=_('manifests'))
             n = mf.node(i)
             lr = self.checkentry(mf, i, n, seen, mflinkrevs.get(n, []), label)
             if n in mflinkrevs:
                 del mflinkrevs[n]
+            elif dir:
+                self.err(lr, _("%s not in parent-directory manifest") %
+                         short(n), label)
             else:
                 self.err(lr, _("%s not in changesets") % short(n), label)

             try:
-                for f, fn in mf.readdelta(n).iteritems():
+                for f, fn, fl in mf.readshallowdelta(n).iterentries():
                     if not f:
-                        self.err(lr, _("file without name in manifest"))
-                    elif f != "/dev/null": # ignore this in very old repos
-                        if _validpath(repo, f):
-                            filenodes.setdefault(
-                                _normpath(f), {}).setdefault(fn, lr)
+                        self.err(lr, _("entry without name in manifest"))
+                    elif f == "/dev/null":  # ignore this in very old repos
+                        continue
+                    fullpath = dir + _normpath(f)
+                    if not _validpath(repo, fullpath):
+                        continue
+                    if fl == 't':
+                        subdirnodes.setdefault(fullpath + '/', {}).setdefault(
+                            fn, []).append(lr)
+                    else:
+                        filenodes.setdefault(fullpath, {}).setdefault(fn, lr)
             except Exception as inst:
                 self.exc(lr, _("reading delta %s") % short(n), inst, label)
-        ui.progress(_('checking'), None)
+        if not dir:
+            ui.progress(_('checking'), None)

         if self.havemf:
             for c, m in sorted([(c, m) for m in mflinkrevs
                         for c in mflinkrevs[m]]):
-                self.err(c, _("changeset refers to unknown revision %s") %
-                         short(m), label)
+                if dir:
+                    self.err(c, _("parent-directory manifest refers
to unknown "
+                                  "revision %s") % short(m), label)
+                else:
+                    self.err(c, _("changeset refers to unknown revision %s") %
+                             short(m), label)
+
+        if not dir and subdirnodes:
+            self.ui.status(_("checking directory manifests\n"))
+        for subdir, linkrevs in subdirnodes.iteritems():
+            subdirfilenodes = self._verifymanifest(linkrevs, subdir)
+            for f, onefilenodes in subdirfilenodes.iteritems():
+                filenodes.setdefault(f, {}).update(onefilenodes)

         return filenodes

diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t
--- a/tests/test-treemanifest.t
+++ b/tests/test-treemanifest.t
@@ -469,6 +469,7 @@ 
   $ hg verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -498,6 +499,56 @@ 
   adding meta/b/foo/apple/bees/00manifest.i
   16 items added, 0 removed from fncache

+Back up the recently added revlogs
+  $ cp -r .hg/store .hg/store-newcopy
+
+Verify reports missing dirlog
+  $ rm .hg/store/meta/b/00manifest.*
+  $ hg verify
+  checking changesets
+  checking manifests
+  checking directory manifests
+   0: empty or missing b/
+   b/@0: parent-directory manifest refers to unknown revision 67688a370455
+   b/@1: parent-directory manifest refers to unknown revision f38e85d334c5
+   b/@2: parent-directory manifest refers to unknown revision 99c9792fd4b0
+  crosschecking files in changesets and manifests
+   b/bar/fruits.txt@0: in changeset but not in manifest
+   b/bar/orange/fly/gnat.py@0: in changeset but not in manifest
+   b/bar/orange/fly/housefly.txt@0: in changeset but not in manifest
+   b/foo/apple/bees/flower.py@0: in changeset but not in manifest
+  checking files
+  8 files, 3 changesets, 10 total revisions
+  8 integrity errors encountered!
+  (first damaged changeset appears to be 0)
+  [1]
+  $ cp -rT .hg/store-newcopy .hg/store
+
+Verify reports missing dirlog entry
+  $ mv -f .hg/store-copy/meta/b/00manifest.* .hg/store/meta/b/
+  $ hg verify
+  checking changesets
+  checking manifests
+  checking directory manifests
+   b/@1: parent-directory manifest refers to unknown revision f38e85d334c5
+   b/@2: parent-directory manifest refers to unknown revision 99c9792fd4b0
+   b/bar/@?: rev 1 points to unexpected changeset 1
+   b/bar/@?: 5e03c4ee5e4a not in parent-directory manifest
+   b/bar/@?: rev 2 points to unexpected changeset 2
+   b/bar/@?: 1b16940d66d6 not in parent-directory manifest
+   b/bar/orange/@?: rev 1 points to unexpected changeset 2
+   (expected None)
+   b/bar/orange/fly/@?: rev 1 points to unexpected changeset 2
+   (expected None)
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+  2 warnings encountered!
+  8 integrity errors encountered!
+  (first damaged changeset appears to be 1)
+  [1]
+  $ cp -rT .hg/store-newcopy .hg/store
+
 Test cloning a treemanifest repo over http.
   $ hg serve -p $HGPORT2 -d --pid-file=hg.pid --errorlog=errors.log
   $ cat hg.pid >> $DAEMON_PIDS
@@ -542,6 +593,7 @@ 
   $ hg verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -586,6 +638,7 @@ 
   $ hg -R local-clone-basicstore verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -595,6 +648,7 @@ 
   $ hg -R local-clone-encodedstore verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -604,6 +658,7 @@ 
   $ hg -R local-clone-fncachestore verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -619,6 +674,7 @@ 
   $ hg -R stream-clone-basicstore verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -634,6 +690,7 @@ 
   $ hg -R stream-clone-encodedstore verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -649,6 +706,7 @@ 
   $ hg -R stream-clone-fncachestore verify
   checking changesets
   checking manifests
+  checking directory manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions