From patchwork Fri Oct 1 07:20:26 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: D11520: dirstate-v2: Add support when Rust is not enabled From: phabricator X-Patchwork-Id: 49844 Message-Id: To: Phabricator Cc: mercurial-devel@mercurial-scm.org Date: Fri, 1 Oct 2021 07:20:26 +0000 SimonSapin created this revision. Herald added a reviewer: hg-reviewers. Herald added a subscriber: mercurial-patches. REVISION SUMMARY This wires into `dirstatemap` the parser and serializer added in previous changesets. The memory representation is still the same, with a flat `dict` for `DirstateItem`s and another one for copy sources. Serialization always creates a new dirstate-v2 data file and does not support (when Rust is not enabled) appending to an existing one, since we don’t keep track of which tree nodes are new or modified. Instead the tree is reconstructed during serialization. REPOSITORY rHG Mercurial BRANCH default REVISION DETAIL https://phab.mercurial-scm.org/D11520 AFFECTED FILES mercurial/dirstate.py mercurial/dirstatemap.py mercurial/localrepo.py tests/test-dirstate-race.t tests/test-dirstate-race2.t tests/test-dirstate.t tests/test-hgignore.t tests/test-permissions.t tests/test-purge.t tests/test-status.t tests/test-symlinks.t CHANGE DETAILS To: SimonSapin, #hg-reviewers Cc: mercurial-patches, mercurial-devel diff --git a/tests/test-symlinks.t b/tests/test-symlinks.t --- a/tests/test-symlinks.t +++ b/tests/test-symlinks.t @@ -3,7 +3,6 @@ #testcases dirstate-v1 dirstate-v2 #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif diff --git a/tests/test-status.t b/tests/test-status.t --- a/tests/test-status.t +++ b/tests/test-status.t @@ -1,13 +1,6 @@ #testcases dirstate-v1 dirstate-v2 -#if no-rust - $ hg init repo0 --config format.exp-dirstate-v2=1 - abort: dirstate v2 format requested by config but not supported (requires Rust extensions) - [255] -#endif - #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif @@ -743,7 +736,7 @@ if also listing unknowns. The tree-based dirstate and status algorithm fix this: -#if symlink no-dirstate-v1 +#if symlink no-dirstate-v1 rust $ cd .. $ hg init issue6335 @@ -759,11 +752,11 @@ ? bar/a ? foo - $ hg status -c # incorrect output with `dirstate-v1` + $ hg status -c # incorrect output without the Rust implementation $ hg status -cu ? bar/a ? foo - $ hg status -d # incorrect output with `dirstate-v1` + $ hg status -d # incorrect output without the Rust implementation ! foo/a $ hg status -du ! foo/a @@ -910,7 +903,7 @@ I B.hs I ignored-folder/ctest.hs -#if dirstate-v2 +#if rust dirstate-v2 Check read_dir caching diff --git a/tests/test-purge.t b/tests/test-purge.t --- a/tests/test-purge.t +++ b/tests/test-purge.t @@ -1,7 +1,6 @@ #testcases dirstate-v1 dirstate-v2 #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif diff --git a/tests/test-permissions.t b/tests/test-permissions.t --- a/tests/test-permissions.t +++ b/tests/test-permissions.t @@ -3,7 +3,6 @@ #testcases dirstate-v1 dirstate-v2 #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif diff --git a/tests/test-hgignore.t b/tests/test-hgignore.t --- a/tests/test-hgignore.t +++ b/tests/test-hgignore.t @@ -1,7 +1,6 @@ #testcases dirstate-v1 dirstate-v2 #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif @@ -397,9 +396,10 @@ #endif -#if dirstate-v2 +#if dirstate-v2 rust Check the hash of ignore patterns written in the dirstate +This is an optimization that is only relevant when using the Rust extensions $ hg status > /dev/null $ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1 diff --git a/tests/test-dirstate.t b/tests/test-dirstate.t --- a/tests/test-dirstate.t +++ b/tests/test-dirstate.t @@ -1,7 +1,6 @@ #testcases dirstate-v1 dirstate-v2 #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif diff --git a/tests/test-dirstate-race2.t b/tests/test-dirstate-race2.t --- a/tests/test-dirstate-race2.t +++ b/tests/test-dirstate-race2.t @@ -1,7 +1,6 @@ #testcases dirstate-v1 dirstate-v2 #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif diff --git a/tests/test-dirstate-race.t b/tests/test-dirstate-race.t --- a/tests/test-dirstate-race.t +++ b/tests/test-dirstate-race.t @@ -1,7 +1,6 @@ #testcases dirstate-v1 dirstate-v2 #if dirstate-v2 -#require rust $ echo '[format]' >> $HGRCPATH $ echo 'exp-dirstate-v2=1' >> $HGRCPATH #endif diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py --- a/mercurial/localrepo.py +++ b/mercurial/localrepo.py @@ -917,9 +917,6 @@ # Start with all requirements supported by this file. supported = set(localrepository._basesupported) - if dirstate.SUPPORTS_DIRSTATE_V2: - supported.add(requirementsmod.DIRSTATE_V2_REQUIREMENT) - # Execute ``featuresetupfuncs`` entries if they belong to an extension # relevant to this ui instance. modules = {m.__name__ for n, m in extensions.extensions(ui)} @@ -1266,6 +1263,7 @@ requirementsmod.NODEMAP_REQUIREMENT, bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT, requirementsmod.SHARESAFE_REQUIREMENT, + requirementsmod.DIRSTATE_V2_REQUIREMENT, } _basesupported = supportedformats | { requirementsmod.STORE_REQUIREMENT, @@ -3609,15 +3607,7 @@ # experimental config: format.exp-dirstate-v2 # Keep this logic in sync with `has_dirstate_v2()` in `tests/hghave.py` if ui.configbool(b'format', b'exp-dirstate-v2'): - if dirstate.SUPPORTS_DIRSTATE_V2: - requirements.add(requirementsmod.DIRSTATE_V2_REQUIREMENT) - else: - raise error.Abort( - _( - b"dirstate v2 format requested by config " - b"but not supported (requires Rust extensions)" - ) - ) + requirements.add(requirementsmod.DIRSTATE_V2_REQUIREMENT) # experimental config: format.exp-use-copies-side-data-changeset if ui.configbool(b'format', b'exp-use-copies-side-data-changeset'): diff --git a/mercurial/dirstatemap.py b/mercurial/dirstatemap.py --- a/mercurial/dirstatemap.py +++ b/mercurial/dirstatemap.py @@ -36,7 +36,111 @@ rangemask = 0x7FFFFFFF -class dirstatemap(object): +class dirstatemapcommon(object): + """ + Methods that are idertical for both implementations of the dirstatemap + class, with and without Rust extensions enabled. + """ + + def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2): + self._ui = ui + self._opener = opener + self._root = root + self._filename = b'dirstate' + self._nodelen = 20 # Also update Rust code when changing this! + self._nodeconstants = nodeconstants + self._use_dirstate_v2 = use_dirstate_v2 + self._parents = None + self._dirtyparents = False + self._docket = None + + # for consistent view between _pl() and _read() invocations + self._pendingmode = None + + def _opendirstatefile(self): + fp, mode = txnutil.trypending(self._root, self._opener, self._filename) + if self._pendingmode is not None and self._pendingmode != mode: + fp.close() + raise error.Abort( + _(b'working directory state may be changed parallelly') + ) + self._pendingmode = mode + return fp + + def _readdirstatefile(self, size=-1): + try: + with self._opendirstatefile() as fp: + return fp.read(size) + except IOError as err: + if err.errno != errno.ENOENT: + raise + # File doesn't exist, so the current state is empty + return b'' + + @property + def docket(self): + if not self._docket: + if not self._use_dirstate_v2: + raise error.ProgrammingError( + b'dirstate only has a docket in v2 format' + ) + self._docket = docketmod.DirstateDocket.parse( + self._readdirstatefile(), self._nodeconstants + ) + return self._docket + + def parents(self): + if not self._parents: + if self._use_dirstate_v2: + self._parents = self.docket.parents + else: + read_len = self._nodelen * 2 + st = self._readdirstatefile(read_len) + l = len(st) + if l == read_len: + self._parents = ( + st[: self._nodelen], + st[self._nodelen : 2 * self._nodelen], + ) + elif l == 0: + self._parents = ( + self._nodeconstants.nullid, + self._nodeconstants.nullid, + ) + else: + raise error.Abort( + _(b'working directory state appears damaged!') + ) + return self._parents + + def write_no_append(self, tr, st, meta, packed): + old_docket = self.docket + new_docket = docketmod.DirstateDocket.with_new_uuid( + self.parents(), len(packed), meta + ) + data_filename = new_docket.data_filename() + if tr: + tr.add(data_filename, 0) + self._opener.write(data_filename, packed) + # Write the new docket after the new data file has been + # written. Because `st` was opened with `atomictemp=True`, + # the actual `.hg/dirstate` file is only affected on close. + st.write(new_docket.serialize()) + st.close() + # Remove the old data file after the new docket pointing to + # the new data file was written. + if old_docket.uuid: + data_filename = old_docket.data_filename() + unlink = lambda _tr=None: self._opener.unlink(data_filename) + if tr: + category = b"dirstate-v2-clean-" + old_docket.uuid + tr.addpostclose(category, unlink) + else: + unlink() + self._docket = new_docket + + +class dirstatemap(dirstatemapcommon): """Map encapsulating the dirstate's contents. The dirstate contains the following state: @@ -70,23 +174,6 @@ denormalized form that they appear as in the dirstate. """ - def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2): - self._ui = ui - self._opener = opener - self._root = root - self._filename = b'dirstate' - self._nodelen = 20 - self._nodeconstants = nodeconstants - assert ( - not use_dirstate_v2 - ), "should have detected unsupported requirement" - - self._parents = None - self._dirtyparents = False - - # for consistent view between _pl() and _read() invocations - self._pendingmode = None - @propertycache def _map(self): self._map = {} @@ -351,46 +438,6 @@ def _alldirs(self): return pathutil.dirs(self._map) - def _opendirstatefile(self): - fp, mode = txnutil.trypending(self._root, self._opener, self._filename) - if self._pendingmode is not None and self._pendingmode != mode: - fp.close() - raise error.Abort( - _(b'working directory state may be changed parallelly') - ) - self._pendingmode = mode - return fp - - def parents(self): - if not self._parents: - try: - fp = self._opendirstatefile() - st = fp.read(2 * self._nodelen) - fp.close() - except IOError as err: - if err.errno != errno.ENOENT: - raise - # File doesn't exist, so the current state is empty - st = b'' - - l = len(st) - if l == self._nodelen * 2: - self._parents = ( - st[: self._nodelen], - st[self._nodelen : 2 * self._nodelen], - ) - elif l == 0: - self._parents = ( - self._nodeconstants.nullid, - self._nodeconstants.nullid, - ) - else: - raise error.Abort( - _(b'working directory state appears damaged!') - ) - - return self._parents - def setparents(self, p1, p2, fold_p2=False): self._parents = (p1, p2) self._dirtyparents = True @@ -411,19 +458,17 @@ self._opener.join(self._filename) ) - try: - fp = self._opendirstatefile() - try: - st = fp.read() - finally: - fp.close() - except IOError as err: - if err.errno != errno.ENOENT: - raise - return + if self._use_dirstate_v2: + if not self.docket.uuid: + return + st = self._opener.read(self.docket.data_filename()) + else: + st = self._readdirstatefile() + if not st: return + # TODO: adjust this estimate for dirstate-v2 if util.safehasattr(parsers, b'dict_new_presized'): # Make an estimate of the number of files in the dirstate based on # its size. This trades wasting some memory for avoiding costly @@ -445,8 +490,14 @@ # parsing the dirstate. # # (we cannot decorate the function directly since it is in a C module) - parse_dirstate = util.nogc(parsers.parse_dirstate) - p = parse_dirstate(self._map, self.copymap, st) + if self._use_dirstate_v2: + p = self.docket.parents + meta = self.docket.tree_metadata + parse_dirstate = util.nogc(v2.parse_dirstate) + parse_dirstate(self._map, self.copymap, st, meta) + else: + parse_dirstate = util.nogc(parsers.parse_dirstate) + p = parse_dirstate(self._map, self.copymap, st) if not self._dirtyparents: self.setparents(*p) @@ -455,11 +506,19 @@ self.__getitem__ = self._map.__getitem__ self.get = self._map.get - def write(self, _tr, st, now): - st.write( - parsers.pack_dirstate(self._map, self.copymap, self.parents(), now) - ) - st.close() + def write(self, tr, st, now): + if not self._use_dirstate_v2: + st.write( + parsers.pack_dirstate( + self._map, self.copymap, self.parents(), now + ) + ) + st.close() + self._dirtyparents = False + return + + packed, meta = v2.pack_dirstate(self._map, self.copymap, now) + self.write_no_append(tr, st, meta, packed) self._dirtyparents = False @propertycache @@ -476,23 +535,15 @@ return f +# When Rust is enabled, define a different implementation of +# the `dirstatemap` class. if rustmod is not None: - class dirstatemap(object): + class dirstatemap(dirstatemapcommon): def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2): - self._use_dirstate_v2 = use_dirstate_v2 - self._nodeconstants = nodeconstants - self._ui = ui - self._opener = opener - self._root = root - self._filename = b'dirstate' - self._nodelen = 20 # Also update Rust code when changing this! - self._parents = None - self._dirtyparents = False - self._docket = None - - # for consistent view between _pl() and _read() invocations - self._pendingmode = None + super(dirstatemap, self).__init__( + ui, opener, root, nodeconstants, use_dirstate_v2 + ) def addfile( self, @@ -693,28 +744,6 @@ # forward for python2,3 compat iteritems = items - def _opendirstatefile(self): - fp, mode = txnutil.trypending( - self._root, self._opener, self._filename - ) - if self._pendingmode is not None and self._pendingmode != mode: - fp.close() - raise error.Abort( - _(b'working directory state may be changed parallelly') - ) - self._pendingmode = mode - return fp - - def _readdirstatefile(self, size=-1): - try: - with self._opendirstatefile() as fp: - return fp.read(size) - except IOError as err: - if err.errno != errno.ENOENT: - raise - # File doesn't exist, so the current state is empty - return b'' - def setparents(self, p1, p2, fold_p2=False): self._parents = (p1, p2) self._dirtyparents = True @@ -754,43 +783,6 @@ ) return copies - def parents(self): - if not self._parents: - if self._use_dirstate_v2: - self._parents = self.docket.parents - else: - read_len = self._nodelen * 2 - st = self._readdirstatefile(read_len) - l = len(st) - if l == read_len: - self._parents = ( - st[: self._nodelen], - st[self._nodelen : 2 * self._nodelen], - ) - elif l == 0: - self._parents = ( - self._nodeconstants.nullid, - self._nodeconstants.nullid, - ) - else: - raise error.Abort( - _(b'working directory state appears damaged!') - ) - - return self._parents - - @property - def docket(self): - if not self._docket: - if not self._use_dirstate_v2: - raise error.ProgrammingError( - b'dirstate only has a docket in v2 format' - ) - self._docket = docketmod.DirstateDocket.parse( - self._readdirstatefile(), self._nodeconstants - ) - return self._docket - @propertycache def _rustmap(self): """ @@ -853,30 +845,7 @@ st.write(docket.serialize()) st.close() else: - old_docket = self.docket - new_docket = docketmod.DirstateDocket.with_new_uuid( - self.parents(), len(packed), meta - ) - data_filename = new_docket.data_filename() - if tr: - tr.add(data_filename, 0) - self._opener.write(data_filename, packed) - # Write the new docket after the new data file has been - # written. Because `st` was opened with `atomictemp=True`, - # the actual `.hg/dirstate` file is only affected on close. - st.write(new_docket.serialize()) - st.close() - # Remove the old data file after the new docket pointing to - # the new data file was written. - if old_docket.uuid: - data_filename = old_docket.data_filename() - unlink = lambda _tr=None: self._opener.unlink(data_filename) - if tr: - category = b"dirstate-v2-clean-" + old_docket.uuid - tr.addpostclose(category, unlink) - else: - unlink() - self._docket = new_docket + self.write_no_append(tr, st, meta, packed) # Reload from the newly-written file util.clearcachedproperty(self, b"_rustmap") self._dirtyparents = False diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py --- a/mercurial/dirstate.py +++ b/mercurial/dirstate.py @@ -39,8 +39,6 @@ parsers = policy.importmod('parsers') rustmod = policy.importrust('dirstate') -SUPPORTS_DIRSTATE_V2 = rustmod is not None - propertycache = util.propertycache filecache = scmutil.filecache _rangemask = dirstatemap.rangemask