Patchwork D11520: dirstate-v2: Add support when Rust is not enabled

login
register
mail settings
Submitter phabricator
Date Oct. 1, 2021, 7:20 a.m.
Message ID <differential-rev-PHID-DREV-uk5ui6wimhnj3zi33o6d-req@mercurial-scm.org>
Download mbox | patch
Permalink /patch/49844/
State Superseded
Headers show

Comments

phabricator - Oct. 1, 2021, 7:20 a.m.
SimonSapin created this revision.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.

REVISION SUMMARY
  This wires into `dirstatemap` the parser and serializer added in previous
  changesets. The memory representation is still the same, with a flat `dict`
  for `DirstateItem`s and another one for copy sources. Serialization always
  creates a new dirstate-v2 data file and does not support (when Rust is not
  enabled) appending to an existing one, since we don’t keep track of which
  tree nodes are new or modified. Instead the tree is reconstructed during
  serialization.

REPOSITORY
  rHG Mercurial

BRANCH
  default

REVISION DETAIL
  https://phab.mercurial-scm.org/D11520

AFFECTED FILES
  mercurial/dirstate.py
  mercurial/dirstatemap.py
  mercurial/localrepo.py
  tests/test-dirstate-race.t
  tests/test-dirstate-race2.t
  tests/test-dirstate.t
  tests/test-hgignore.t
  tests/test-permissions.t
  tests/test-purge.t
  tests/test-status.t
  tests/test-symlinks.t

CHANGE DETAILS




To: SimonSapin, #hg-reviewers
Cc: mercurial-patches, mercurial-devel

Patch

diff --git a/tests/test-symlinks.t b/tests/test-symlinks.t
--- a/tests/test-symlinks.t
+++ b/tests/test-symlinks.t
@@ -3,7 +3,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
diff --git a/tests/test-status.t b/tests/test-status.t
--- a/tests/test-status.t
+++ b/tests/test-status.t
@@ -1,13 +1,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
-#if no-rust
-  $ hg init repo0 --config format.exp-dirstate-v2=1
-  abort: dirstate v2 format requested by config but not supported (requires Rust extensions)
-  [255]
-#endif
-
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
@@ -743,7 +736,7 @@ 
 if also listing unknowns.
 The tree-based dirstate and status algorithm fix this:
 
-#if symlink no-dirstate-v1
+#if symlink no-dirstate-v1 rust
 
   $ cd ..
   $ hg init issue6335
@@ -759,11 +752,11 @@ 
   ? bar/a
   ? foo
 
-  $ hg status -c  # incorrect output with `dirstate-v1`
+  $ hg status -c  # incorrect output without the Rust implementation
   $ hg status -cu
   ? bar/a
   ? foo
-  $ hg status -d  # incorrect output with `dirstate-v1`
+  $ hg status -d  # incorrect output without the Rust implementation
   ! foo/a
   $ hg status -du
   ! foo/a
@@ -910,7 +903,7 @@ 
   I B.hs
   I ignored-folder/ctest.hs
 
-#if dirstate-v2
+#if rust dirstate-v2
 
 Check read_dir caching
 
diff --git a/tests/test-purge.t b/tests/test-purge.t
--- a/tests/test-purge.t
+++ b/tests/test-purge.t
@@ -1,7 +1,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
diff --git a/tests/test-permissions.t b/tests/test-permissions.t
--- a/tests/test-permissions.t
+++ b/tests/test-permissions.t
@@ -3,7 +3,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
diff --git a/tests/test-hgignore.t b/tests/test-hgignore.t
--- a/tests/test-hgignore.t
+++ b/tests/test-hgignore.t
@@ -1,7 +1,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
@@ -397,9 +396,10 @@ 
 
 #endif
 
-#if dirstate-v2
+#if dirstate-v2 rust
 
 Check the hash of ignore patterns written in the dirstate
+This is an optimization that is only relevant when using the Rust extensions
 
   $ hg status > /dev/null
   $ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1
diff --git a/tests/test-dirstate.t b/tests/test-dirstate.t
--- a/tests/test-dirstate.t
+++ b/tests/test-dirstate.t
@@ -1,7 +1,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
diff --git a/tests/test-dirstate-race2.t b/tests/test-dirstate-race2.t
--- a/tests/test-dirstate-race2.t
+++ b/tests/test-dirstate-race2.t
@@ -1,7 +1,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
diff --git a/tests/test-dirstate-race.t b/tests/test-dirstate-race.t
--- a/tests/test-dirstate-race.t
+++ b/tests/test-dirstate-race.t
@@ -1,7 +1,6 @@ 
 #testcases dirstate-v1 dirstate-v2
 
 #if dirstate-v2
-#require rust
   $ echo '[format]' >> $HGRCPATH
   $ echo 'exp-dirstate-v2=1' >> $HGRCPATH
 #endif
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -917,9 +917,6 @@ 
     # Start with all requirements supported by this file.
     supported = set(localrepository._basesupported)
 
-    if dirstate.SUPPORTS_DIRSTATE_V2:
-        supported.add(requirementsmod.DIRSTATE_V2_REQUIREMENT)
-
     # Execute ``featuresetupfuncs`` entries if they belong to an extension
     # relevant to this ui instance.
     modules = {m.__name__ for n, m in extensions.extensions(ui)}
@@ -1266,6 +1263,7 @@ 
         requirementsmod.NODEMAP_REQUIREMENT,
         bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT,
         requirementsmod.SHARESAFE_REQUIREMENT,
+        requirementsmod.DIRSTATE_V2_REQUIREMENT,
     }
     _basesupported = supportedformats | {
         requirementsmod.STORE_REQUIREMENT,
@@ -3609,15 +3607,7 @@ 
     # experimental config: format.exp-dirstate-v2
     # Keep this logic in sync with `has_dirstate_v2()` in `tests/hghave.py`
     if ui.configbool(b'format', b'exp-dirstate-v2'):
-        if dirstate.SUPPORTS_DIRSTATE_V2:
-            requirements.add(requirementsmod.DIRSTATE_V2_REQUIREMENT)
-        else:
-            raise error.Abort(
-                _(
-                    b"dirstate v2 format requested by config "
-                    b"but not supported (requires Rust extensions)"
-                )
-            )
+        requirements.add(requirementsmod.DIRSTATE_V2_REQUIREMENT)
 
     # experimental config: format.exp-use-copies-side-data-changeset
     if ui.configbool(b'format', b'exp-use-copies-side-data-changeset'):
diff --git a/mercurial/dirstatemap.py b/mercurial/dirstatemap.py
--- a/mercurial/dirstatemap.py
+++ b/mercurial/dirstatemap.py
@@ -36,7 +36,111 @@ 
 rangemask = 0x7FFFFFFF
 
 
-class dirstatemap(object):
+class dirstatemapcommon(object):
+    """
+    Methods that are idertical for both implementations of the dirstatemap
+    class, with and without Rust extensions enabled.
+    """
+
+    def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
+        self._ui = ui
+        self._opener = opener
+        self._root = root
+        self._filename = b'dirstate'
+        self._nodelen = 20  # Also update Rust code when changing this!
+        self._nodeconstants = nodeconstants
+        self._use_dirstate_v2 = use_dirstate_v2
+        self._parents = None
+        self._dirtyparents = False
+        self._docket = None
+
+        # for consistent view between _pl() and _read() invocations
+        self._pendingmode = None
+
+    def _opendirstatefile(self):
+        fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
+        if self._pendingmode is not None and self._pendingmode != mode:
+            fp.close()
+            raise error.Abort(
+                _(b'working directory state may be changed parallelly')
+            )
+        self._pendingmode = mode
+        return fp
+
+    def _readdirstatefile(self, size=-1):
+        try:
+            with self._opendirstatefile() as fp:
+                return fp.read(size)
+        except IOError as err:
+            if err.errno != errno.ENOENT:
+                raise
+            # File doesn't exist, so the current state is empty
+            return b''
+
+    @property
+    def docket(self):
+        if not self._docket:
+            if not self._use_dirstate_v2:
+                raise error.ProgrammingError(
+                    b'dirstate only has a docket in v2 format'
+                )
+            self._docket = docketmod.DirstateDocket.parse(
+                self._readdirstatefile(), self._nodeconstants
+            )
+        return self._docket
+
+    def parents(self):
+        if not self._parents:
+            if self._use_dirstate_v2:
+                self._parents = self.docket.parents
+            else:
+                read_len = self._nodelen * 2
+                st = self._readdirstatefile(read_len)
+                l = len(st)
+                if l == read_len:
+                    self._parents = (
+                        st[: self._nodelen],
+                        st[self._nodelen : 2 * self._nodelen],
+                    )
+                elif l == 0:
+                    self._parents = (
+                        self._nodeconstants.nullid,
+                        self._nodeconstants.nullid,
+                    )
+                else:
+                    raise error.Abort(
+                        _(b'working directory state appears damaged!')
+                    )
+        return self._parents
+
+    def write_no_append(self, tr, st, meta, packed):
+        old_docket = self.docket
+        new_docket = docketmod.DirstateDocket.with_new_uuid(
+            self.parents(), len(packed), meta
+        )
+        data_filename = new_docket.data_filename()
+        if tr:
+            tr.add(data_filename, 0)
+        self._opener.write(data_filename, packed)
+        # Write the new docket after the new data file has been
+        # written. Because `st` was opened with `atomictemp=True`,
+        # the actual `.hg/dirstate` file is only affected on close.
+        st.write(new_docket.serialize())
+        st.close()
+        # Remove the old data file after the new docket pointing to
+        # the new data file was written.
+        if old_docket.uuid:
+            data_filename = old_docket.data_filename()
+            unlink = lambda _tr=None: self._opener.unlink(data_filename)
+            if tr:
+                category = b"dirstate-v2-clean-" + old_docket.uuid
+                tr.addpostclose(category, unlink)
+            else:
+                unlink()
+        self._docket = new_docket
+
+
+class dirstatemap(dirstatemapcommon):
     """Map encapsulating the dirstate's contents.
 
     The dirstate contains the following state:
@@ -70,23 +174,6 @@ 
       denormalized form that they appear as in the dirstate.
     """
 
-    def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
-        self._ui = ui
-        self._opener = opener
-        self._root = root
-        self._filename = b'dirstate'
-        self._nodelen = 20
-        self._nodeconstants = nodeconstants
-        assert (
-            not use_dirstate_v2
-        ), "should have detected unsupported requirement"
-
-        self._parents = None
-        self._dirtyparents = False
-
-        # for consistent view between _pl() and _read() invocations
-        self._pendingmode = None
-
     @propertycache
     def _map(self):
         self._map = {}
@@ -351,46 +438,6 @@ 
     def _alldirs(self):
         return pathutil.dirs(self._map)
 
-    def _opendirstatefile(self):
-        fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
-        if self._pendingmode is not None and self._pendingmode != mode:
-            fp.close()
-            raise error.Abort(
-                _(b'working directory state may be changed parallelly')
-            )
-        self._pendingmode = mode
-        return fp
-
-    def parents(self):
-        if not self._parents:
-            try:
-                fp = self._opendirstatefile()
-                st = fp.read(2 * self._nodelen)
-                fp.close()
-            except IOError as err:
-                if err.errno != errno.ENOENT:
-                    raise
-                # File doesn't exist, so the current state is empty
-                st = b''
-
-            l = len(st)
-            if l == self._nodelen * 2:
-                self._parents = (
-                    st[: self._nodelen],
-                    st[self._nodelen : 2 * self._nodelen],
-                )
-            elif l == 0:
-                self._parents = (
-                    self._nodeconstants.nullid,
-                    self._nodeconstants.nullid,
-                )
-            else:
-                raise error.Abort(
-                    _(b'working directory state appears damaged!')
-                )
-
-        return self._parents
-
     def setparents(self, p1, p2, fold_p2=False):
         self._parents = (p1, p2)
         self._dirtyparents = True
@@ -411,19 +458,17 @@ 
             self._opener.join(self._filename)
         )
 
-        try:
-            fp = self._opendirstatefile()
-            try:
-                st = fp.read()
-            finally:
-                fp.close()
-        except IOError as err:
-            if err.errno != errno.ENOENT:
-                raise
-            return
+        if self._use_dirstate_v2:
+            if not self.docket.uuid:
+                return
+            st = self._opener.read(self.docket.data_filename())
+        else:
+            st = self._readdirstatefile()
+
         if not st:
             return
 
+        # TODO: adjust this estimate for dirstate-v2
         if util.safehasattr(parsers, b'dict_new_presized'):
             # Make an estimate of the number of files in the dirstate based on
             # its size. This trades wasting some memory for avoiding costly
@@ -445,8 +490,14 @@ 
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
-        parse_dirstate = util.nogc(parsers.parse_dirstate)
-        p = parse_dirstate(self._map, self.copymap, st)
+        if self._use_dirstate_v2:
+            p = self.docket.parents
+            meta = self.docket.tree_metadata
+            parse_dirstate = util.nogc(v2.parse_dirstate)
+            parse_dirstate(self._map, self.copymap, st, meta)
+        else:
+            parse_dirstate = util.nogc(parsers.parse_dirstate)
+            p = parse_dirstate(self._map, self.copymap, st)
         if not self._dirtyparents:
             self.setparents(*p)
 
@@ -455,11 +506,19 @@ 
         self.__getitem__ = self._map.__getitem__
         self.get = self._map.get
 
-    def write(self, _tr, st, now):
-        st.write(
-            parsers.pack_dirstate(self._map, self.copymap, self.parents(), now)
-        )
-        st.close()
+    def write(self, tr, st, now):
+        if not self._use_dirstate_v2:
+            st.write(
+                parsers.pack_dirstate(
+                    self._map, self.copymap, self.parents(), now
+                )
+            )
+            st.close()
+            self._dirtyparents = False
+            return
+
+        packed, meta = v2.pack_dirstate(self._map, self.copymap, now)
+        self.write_no_append(tr, st, meta, packed)
         self._dirtyparents = False
 
     @propertycache
@@ -476,23 +535,15 @@ 
         return f
 
 
+# When Rust is enabled, define a different implementation of
+# the `dirstatemap` class.
 if rustmod is not None:
 
-    class dirstatemap(object):
+    class dirstatemap(dirstatemapcommon):
         def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
-            self._use_dirstate_v2 = use_dirstate_v2
-            self._nodeconstants = nodeconstants
-            self._ui = ui
-            self._opener = opener
-            self._root = root
-            self._filename = b'dirstate'
-            self._nodelen = 20  # Also update Rust code when changing this!
-            self._parents = None
-            self._dirtyparents = False
-            self._docket = None
-
-            # for consistent view between _pl() and _read() invocations
-            self._pendingmode = None
+            super(dirstatemap, self).__init__(
+                ui, opener, root, nodeconstants, use_dirstate_v2
+            )
 
         def addfile(
             self,
@@ -693,28 +744,6 @@ 
         # forward for python2,3 compat
         iteritems = items
 
-        def _opendirstatefile(self):
-            fp, mode = txnutil.trypending(
-                self._root, self._opener, self._filename
-            )
-            if self._pendingmode is not None and self._pendingmode != mode:
-                fp.close()
-                raise error.Abort(
-                    _(b'working directory state may be changed parallelly')
-                )
-            self._pendingmode = mode
-            return fp
-
-        def _readdirstatefile(self, size=-1):
-            try:
-                with self._opendirstatefile() as fp:
-                    return fp.read(size)
-            except IOError as err:
-                if err.errno != errno.ENOENT:
-                    raise
-                # File doesn't exist, so the current state is empty
-                return b''
-
         def setparents(self, p1, p2, fold_p2=False):
             self._parents = (p1, p2)
             self._dirtyparents = True
@@ -754,43 +783,6 @@ 
                         )
             return copies
 
-        def parents(self):
-            if not self._parents:
-                if self._use_dirstate_v2:
-                    self._parents = self.docket.parents
-                else:
-                    read_len = self._nodelen * 2
-                    st = self._readdirstatefile(read_len)
-                    l = len(st)
-                    if l == read_len:
-                        self._parents = (
-                            st[: self._nodelen],
-                            st[self._nodelen : 2 * self._nodelen],
-                        )
-                    elif l == 0:
-                        self._parents = (
-                            self._nodeconstants.nullid,
-                            self._nodeconstants.nullid,
-                        )
-                    else:
-                        raise error.Abort(
-                            _(b'working directory state appears damaged!')
-                        )
-
-            return self._parents
-
-        @property
-        def docket(self):
-            if not self._docket:
-                if not self._use_dirstate_v2:
-                    raise error.ProgrammingError(
-                        b'dirstate only has a docket in v2 format'
-                    )
-                self._docket = docketmod.DirstateDocket.parse(
-                    self._readdirstatefile(), self._nodeconstants
-                )
-            return self._docket
-
         @propertycache
         def _rustmap(self):
             """
@@ -853,30 +845,7 @@ 
                 st.write(docket.serialize())
                 st.close()
             else:
-                old_docket = self.docket
-                new_docket = docketmod.DirstateDocket.with_new_uuid(
-                    self.parents(), len(packed), meta
-                )
-                data_filename = new_docket.data_filename()
-                if tr:
-                    tr.add(data_filename, 0)
-                self._opener.write(data_filename, packed)
-                # Write the new docket after the new data file has been
-                # written. Because `st` was opened with `atomictemp=True`,
-                # the actual `.hg/dirstate` file is only affected on close.
-                st.write(new_docket.serialize())
-                st.close()
-                # Remove the old data file after the new docket pointing to
-                # the new data file was written.
-                if old_docket.uuid:
-                    data_filename = old_docket.data_filename()
-                    unlink = lambda _tr=None: self._opener.unlink(data_filename)
-                    if tr:
-                        category = b"dirstate-v2-clean-" + old_docket.uuid
-                        tr.addpostclose(category, unlink)
-                    else:
-                        unlink()
-                self._docket = new_docket
+                self.write_no_append(tr, st, meta, packed)
             # Reload from the newly-written file
             util.clearcachedproperty(self, b"_rustmap")
             self._dirtyparents = False
diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py
--- a/mercurial/dirstate.py
+++ b/mercurial/dirstate.py
@@ -39,8 +39,6 @@ 
 parsers = policy.importmod('parsers')
 rustmod = policy.importrust('dirstate')
 
-SUPPORTS_DIRSTATE_V2 = rustmod is not None
-
 propertycache = util.propertycache
 filecache = scmutil.filecache
 _rangemask = dirstatemap.rangemask