Submitter | Martin von Zweigbergk |
---|---|
Date | Feb. 5, 2016, 9:17 p.m. |
Message ID | <10d703242604ae8e9ecd.1454707046@waste.org> |
Download | mbox | patch |
Permalink | /patch/13020/ |
State | Accepted |
Headers | show |
Comments
On Fri, Feb 05, 2016 at 03:17:26PM -0600, Martin von Zweigbergk wrote: > # HG changeset patch > # User Martin von Zweigbergk <martinvonz@google.com> > # Date 1454603647 28800 > # Thu Feb 04 08:34:07 2016 -0800 > # Node ID 10d703242604ae8e9ecd505e4322153333b191a3 > # Parent 30acd1be0f83c981cf90f1b03666910f6a2b39d7 > treemanifests: fix streaming clone Queued these. Thanks for cleaning up my mess. > > Similar to the previous patch, the .hg/store/meta/ directory does not > get copied when when using "hg clone --uncompressed". Fix by including > "meta/" in store.datafiles(). This seems safe to do, as there are only > a few users of this method. "hg manifest" already filters the paths by > "data/" prefix. The calls from largefiles also seem safe. The use in > verify needs updating to prevent it from mistaking dirlogs for > orphaned filelogs. That change is included in this patch. > > Since the dirlogs will now be in the fncache when using fncachestore, > let's also update debugrebuildfncache(). That will also allow any > existing treemanifest repos to get their dirlogs into the fncache. > > Also update test-treemanifest.t to use an a directory name that > requires dot-encoding and uppercase-encoding so we test that the path > encoding works. > > diff --git a/mercurial/repair.py b/mercurial/repair.py > --- a/mercurial/repair.py > +++ b/mercurial/repair.py > @@ -273,6 +273,16 @@ > > ui.progress(_('changeset'), None) > > + if 'treemanifest' in repo: # safe but unnecessary otherwise > + for dir in util.dirs(seenfiles): > + i = 'meta/%s/00manifest.i' % dir > + d = 'meta/%s/00manifest.d' % dir > + > + if repo.store._exists(i): > + newentries.add(i) > + if repo.store._exists(d): > + newentries.add(d) > + > addcount = len(newentries - oldentries) > removecount = len(oldentries - newentries) > for p in sorted(oldentries - newentries): > diff --git a/mercurial/store.py b/mercurial/store.py > --- a/mercurial/store.py > +++ b/mercurial/store.py > @@ -330,7 +330,7 @@ > return l > > def datafiles(self): > - return self._walk('data', True) > + return self._walk('data', True) + self._walk('meta', True) > > def topfiles(self): > # yield manifest before changelog > @@ -378,7 +378,7 @@ > self.opener = self.vfs > > def datafiles(self): > - for a, b, size in self._walk('data', True): > + for a, b, size in super(encodedstore, self).datafiles(): > try: > a = decodefilename(a) > except KeyError: > @@ -460,7 +460,8 @@ > self.encode = encode > > def __call__(self, path, mode='r', *args, **kw): > - if mode not in ('r', 'rb') and path.startswith('data/'): > + if mode not in ('r', 'rb') and (path.startswith('data/') or > + path.startswith('meta/')): > self.fncache.add(path) > return self.vfs(self.encode(path), mode, *args, **kw) > > diff --git a/mercurial/verify.py b/mercurial/verify.py > --- a/mercurial/verify.py > +++ b/mercurial/verify.py > @@ -284,7 +284,7 @@ > for f, f2, size in repo.store.datafiles(): > if not f: > self.err(None, _("cannot decode filename '%s'") % f2) > - elif size > 0 or not revlogv1: > + elif (size > 0 or not revlogv1) and f.startswith('data/'): > storefiles.add(_normpath(f)) > > files = sorted(set(filenodes) | set(filelinkrevs)) > diff --git a/tests/run-tests.py b/tests/run-tests.py > --- a/tests/run-tests.py > +++ b/tests/run-tests.py > @@ -719,6 +719,8 @@ > (br':%d\b' % self._startport, b':$HGPORT'), > (br':%d\b' % (self._startport + 1), b':$HGPORT1'), > (br':%d\b' % (self._startport + 2), b':$HGPORT2'), > + (br':%d\b' % (self._startport + 2), b':$HGPORT3'), > + (br':%d\b' % (self._startport + 2), b':$HGPORT4'), > (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$', > br'\1 (glob)'), > ] > @@ -741,6 +743,8 @@ > env["HGPORT"] = str(self._startport) > env["HGPORT1"] = str(self._startport + 1) > env["HGPORT2"] = str(self._startport + 2) > + env["HGPORT3"] = str(self._startport + 3) > + env["HGPORT4"] = str(self._startport + 4) > env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc') > env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids') > env["HGEDITOR"] = ('"' + sys.executable + '"' > diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t > --- a/tests/test-treemanifest.t > +++ b/tests/test-treemanifest.t > @@ -367,7 +367,7 @@ > $ hg --config experimental.treemanifest=True init deeprepo > $ cd deeprepo > > - $ mkdir a > + $ mkdir .A > $ mkdir b > $ mkdir b/bar > $ mkdir b/bar/orange > @@ -376,8 +376,8 @@ > $ mkdir b/foo/apple > $ mkdir b/foo/apple/bees > > - $ touch a/one.txt > - $ touch a/two.txt > + $ touch .A/one.txt > + $ touch .A/two.txt > $ touch b/bar/fruits.txt > $ touch b/bar/orange/fly/gnat.py > $ touch b/bar/orange/fly/housefly.txt > @@ -393,8 +393,8 @@ > Test files from the root. > > $ hg files -r . > - a/one.txt (glob) > - a/two.txt (glob) > + .A/one.txt (glob) > + .A/two.txt (glob) > b/bar/fruits.txt (glob) > b/bar/orange/fly/gnat.py (glob) > b/bar/orange/fly/housefly.txt (glob) > @@ -412,7 +412,7 @@ > > Test files for a subdirectory. > > - $ rm -r .hg/store/meta/a > + $ rm -r .hg/store/meta/~2e_a > $ hg files -r . b > b/bar/fruits.txt (glob) > b/bar/orange/fly/gnat.py (glob) > @@ -422,7 +422,7 @@ > > Test files with just includes and excludes. > > - $ rm -r .hg/store/meta/a > + $ rm -r .hg/store/meta/~2e_a > $ rm -r .hg/store/meta/b/bar/orange/fly > $ rm -r .hg/store/meta/b/foo/apple/bees > $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees > @@ -431,7 +431,7 @@ > > Test files for a subdirectory, excluding a directory within it. > > - $ rm -r .hg/store/meta/a > + $ rm -r .hg/store/meta/~2e_a > $ rm -r .hg/store/meta/b/foo > $ hg files -r . -X path:b/foo b > b/bar/fruits.txt (glob) > @@ -442,7 +442,7 @@ > Test files for a sub directory, including only a directory within it, and > including an unrelated directory. > > - $ rm -r .hg/store/meta/a > + $ rm -r .hg/store/meta/~2e_a > $ rm -r .hg/store/meta/b/foo > $ hg files -r . -I path:b/bar/orange -I path:a b > b/bar/orange/fly/gnat.py (glob) > @@ -452,7 +452,7 @@ > Test files for a pattern, including a directory, and excluding a directory > within that. > > - $ rm -r .hg/store/meta/a > + $ rm -r .hg/store/meta/~2e_a > $ rm -r .hg/store/meta/b/foo > $ rm -r .hg/store/meta/b/bar/orange > $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange > @@ -488,8 +488,6 @@ > Tree manifest revlogs exist. > $ find deepclone/.hg/store/meta | sort > deepclone/.hg/store/meta > - deepclone/.hg/store/meta/a > - deepclone/.hg/store/meta/a/00manifest.i > deepclone/.hg/store/meta/b > deepclone/.hg/store/meta/b/00manifest.i > deepclone/.hg/store/meta/b/bar > @@ -504,6 +502,8 @@ > deepclone/.hg/store/meta/b/foo/apple/00manifest.i > deepclone/.hg/store/meta/b/foo/apple/bees > deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i > + deepclone/.hg/store/meta/~2e_a > + deepclone/.hg/store/meta/~2e_a/00manifest.i > Verify passes. > $ cd deepclone > $ hg verify > @@ -525,8 +525,12 @@ > added 3 changesets with 10 changes to 8 files > updating to branch default > 8 files updated, 0 files merged, 0 files removed, 0 files unresolved > - $ grep store deeprepo-basicstore/.hg/requires > + $ cd deeprepo-basicstore > + $ grep store .hg/requires > [1] > + $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log > + $ cat hg.pid >> $DAEMON_PIDS > + $ cd .. > $ hg clone --config format.usefncache=False \ > > --config experimental.changegroup3=True \ > > http://localhost:$HGPORT2 deeprepo-encodedstore > @@ -537,8 +541,12 @@ > added 3 changesets with 10 changes to 8 files > updating to branch default > 8 files updated, 0 files merged, 0 files removed, 0 files unresolved > - $ grep fncache deeprepo-encodedstore/.hg/requires > + $ cd deeprepo-encodedstore > + $ grep fncache .hg/requires > [1] > + $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log > + $ cat hg.pid >> $DAEMON_PIDS > + $ cd .. > > Local clone with basicstore > $ hg clone -U deeprepo-basicstore local-clone-basicstore > @@ -566,3 +574,55 @@ > crosschecking files in changesets and manifests > checking files > 8 files, 3 changesets, 10 total revisions > + > +Stream clone with basicstore > + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ > + > http://localhost:$HGPORT3 stream-clone-basicstore > + streaming all changes > + 18 files to transfer, * of data (glob) > + transferred * in * seconds (*) (glob) > + searching for changes > + no changes found > + $ hg -R stream-clone-basicstore verify > + checking changesets > + checking manifests > + crosschecking files in changesets and manifests > + checking files > + 8 files, 3 changesets, 10 total revisions > + > +Stream clone with encodedstore > + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ > + > http://localhost:$HGPORT4 stream-clone-encodedstore > + streaming all changes > + 18 files to transfer, * of data (glob) > + transferred * in * seconds (*) (glob) > + searching for changes > + no changes found > + $ hg -R stream-clone-encodedstore verify > + checking changesets > + checking manifests > + crosschecking files in changesets and manifests > + checking files > + 8 files, 3 changesets, 10 total revisions > + > +Stream clone with fncachestore > + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ > + > http://localhost:$HGPORT2 stream-clone-fncachestore > + streaming all changes > + 18 files to transfer, * of data (glob) > + transferred * in * seconds (*) (glob) > + searching for changes > + no changes found > + $ hg -R stream-clone-fncachestore verify > + checking changesets > + checking manifests > + crosschecking files in changesets and manifests > + checking files > + 8 files, 3 changesets, 10 total revisions > + > +Packed bundle > + $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg > + writing 3349 bytes for 18 files > + bundle requirements: generaldelta, revlogv1, treemanifest > + $ hg debugbundle --spec repo-packed.hg > + none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest > _______________________________________________ > Mercurial-devel mailing list > Mercurial-devel@mercurial-scm.org > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Patch
diff --git a/mercurial/repair.py b/mercurial/repair.py --- a/mercurial/repair.py +++ b/mercurial/repair.py @@ -273,6 +273,16 @@ ui.progress(_('changeset'), None) + if 'treemanifest' in repo: # safe but unnecessary otherwise + for dir in util.dirs(seenfiles): + i = 'meta/%s/00manifest.i' % dir + d = 'meta/%s/00manifest.d' % dir + + if repo.store._exists(i): + newentries.add(i) + if repo.store._exists(d): + newentries.add(d) + addcount = len(newentries - oldentries) removecount = len(oldentries - newentries) for p in sorted(oldentries - newentries): diff --git a/mercurial/store.py b/mercurial/store.py --- a/mercurial/store.py +++ b/mercurial/store.py @@ -330,7 +330,7 @@ return l def datafiles(self): - return self._walk('data', True) + return self._walk('data', True) + self._walk('meta', True) def topfiles(self): # yield manifest before changelog @@ -378,7 +378,7 @@ self.opener = self.vfs def datafiles(self): - for a, b, size in self._walk('data', True): + for a, b, size in super(encodedstore, self).datafiles(): try: a = decodefilename(a) except KeyError: @@ -460,7 +460,8 @@ self.encode = encode def __call__(self, path, mode='r', *args, **kw): - if mode not in ('r', 'rb') and path.startswith('data/'): + if mode not in ('r', 'rb') and (path.startswith('data/') or + path.startswith('meta/')): self.fncache.add(path) return self.vfs(self.encode(path), mode, *args, **kw) diff --git a/mercurial/verify.py b/mercurial/verify.py --- a/mercurial/verify.py +++ b/mercurial/verify.py @@ -284,7 +284,7 @@ for f, f2, size in repo.store.datafiles(): if not f: self.err(None, _("cannot decode filename '%s'") % f2) - elif size > 0 or not revlogv1: + elif (size > 0 or not revlogv1) and f.startswith('data/'): storefiles.add(_normpath(f)) files = sorted(set(filenodes) | set(filelinkrevs)) diff --git a/tests/run-tests.py b/tests/run-tests.py --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -719,6 +719,8 @@ (br':%d\b' % self._startport, b':$HGPORT'), (br':%d\b' % (self._startport + 1), b':$HGPORT1'), (br':%d\b' % (self._startport + 2), b':$HGPORT2'), + (br':%d\b' % (self._startport + 2), b':$HGPORT3'), + (br':%d\b' % (self._startport + 2), b':$HGPORT4'), (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$', br'\1 (glob)'), ] @@ -741,6 +743,8 @@ env["HGPORT"] = str(self._startport) env["HGPORT1"] = str(self._startport + 1) env["HGPORT2"] = str(self._startport + 2) + env["HGPORT3"] = str(self._startport + 3) + env["HGPORT4"] = str(self._startport + 4) env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc') env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids') env["HGEDITOR"] = ('"' + sys.executable + '"' diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t --- a/tests/test-treemanifest.t +++ b/tests/test-treemanifest.t @@ -367,7 +367,7 @@ $ hg --config experimental.treemanifest=True init deeprepo $ cd deeprepo - $ mkdir a + $ mkdir .A $ mkdir b $ mkdir b/bar $ mkdir b/bar/orange @@ -376,8 +376,8 @@ $ mkdir b/foo/apple $ mkdir b/foo/apple/bees - $ touch a/one.txt - $ touch a/two.txt + $ touch .A/one.txt + $ touch .A/two.txt $ touch b/bar/fruits.txt $ touch b/bar/orange/fly/gnat.py $ touch b/bar/orange/fly/housefly.txt @@ -393,8 +393,8 @@ Test files from the root. $ hg files -r . - a/one.txt (glob) - a/two.txt (glob) + .A/one.txt (glob) + .A/two.txt (glob) b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) b/bar/orange/fly/housefly.txt (glob) @@ -412,7 +412,7 @@ Test files for a subdirectory. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ hg files -r . b b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) @@ -422,7 +422,7 @@ Test files with just includes and excludes. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/bar/orange/fly $ rm -r .hg/store/meta/b/foo/apple/bees $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees @@ -431,7 +431,7 @@ Test files for a subdirectory, excluding a directory within it. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ hg files -r . -X path:b/foo b b/bar/fruits.txt (glob) @@ -442,7 +442,7 @@ Test files for a sub directory, including only a directory within it, and including an unrelated directory. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ hg files -r . -I path:b/bar/orange -I path:a b b/bar/orange/fly/gnat.py (glob) @@ -452,7 +452,7 @@ Test files for a pattern, including a directory, and excluding a directory within that. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ rm -r .hg/store/meta/b/bar/orange $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange @@ -488,8 +488,6 @@ Tree manifest revlogs exist. $ find deepclone/.hg/store/meta | sort deepclone/.hg/store/meta - deepclone/.hg/store/meta/a - deepclone/.hg/store/meta/a/00manifest.i deepclone/.hg/store/meta/b deepclone/.hg/store/meta/b/00manifest.i deepclone/.hg/store/meta/b/bar @@ -504,6 +502,8 @@ deepclone/.hg/store/meta/b/foo/apple/00manifest.i deepclone/.hg/store/meta/b/foo/apple/bees deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i + deepclone/.hg/store/meta/~2e_a + deepclone/.hg/store/meta/~2e_a/00manifest.i Verify passes. $ cd deepclone $ hg verify @@ -525,8 +525,12 @@ added 3 changesets with 10 changes to 8 files updating to branch default 8 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ grep store deeprepo-basicstore/.hg/requires + $ cd deeprepo-basicstore + $ grep store .hg/requires [1] + $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log + $ cat hg.pid >> $DAEMON_PIDS + $ cd .. $ hg clone --config format.usefncache=False \ > --config experimental.changegroup3=True \ > http://localhost:$HGPORT2 deeprepo-encodedstore @@ -537,8 +541,12 @@ added 3 changesets with 10 changes to 8 files updating to branch default 8 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ grep fncache deeprepo-encodedstore/.hg/requires + $ cd deeprepo-encodedstore + $ grep fncache .hg/requires [1] + $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log + $ cat hg.pid >> $DAEMON_PIDS + $ cd .. Local clone with basicstore $ hg clone -U deeprepo-basicstore local-clone-basicstore @@ -566,3 +574,55 @@ crosschecking files in changesets and manifests checking files 8 files, 3 changesets, 10 total revisions + +Stream clone with basicstore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT3 stream-clone-basicstore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-basicstore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Stream clone with encodedstore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT4 stream-clone-encodedstore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-encodedstore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Stream clone with fncachestore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT2 stream-clone-fncachestore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-fncachestore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Packed bundle + $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg + writing 3349 bytes for 18 files + bundle requirements: generaldelta, revlogv1, treemanifest + $ hg debugbundle --spec repo-packed.hg + none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest