Patchwork [4,of,4,stable] largefiles: update in two steps, handle interrupted updates better

login
register
mail settings
Submitter Mads Kiilerich
Date Nov. 7, 2013, 12:59 a.m.
Message ID <bf9d390341502cc5bba5.1383785946@localhost.localdomain>
Download mbox | patch
Permalink /patch/2867/
State Accepted
Commit 8a021cd38719a4d4729027e44b5e0d9ea1144174
Headers show

Comments

Mads Kiilerich - Nov. 7, 2013, 12:59 a.m.
# HG changeset patch
# User Mads Kiilerich <madski@unity3d.com>
# Date 1383785800 -3600
#      Thu Nov 07 01:56:40 2013 +0100
# Branch stable
# Node ID bf9d390341502cc5bba579ad698e32d8f6795487
# Parent  b6c3191618e2dc151347d307678d213049992456
largefiles: update in two steps, handle interrupted updates better

An update would try to fetch any missing largefiles after having updated normal
files and standins. That could fail or be interrupted and would leave the
working directory in a state where the largefiles not only were missing but
also were scheduled for remove ... and where the old largefile was left in
place.

Instead we now remove old largefiles before starting to download and update
missing largefiles.
Matt Mackall - Nov. 16, 2013, 8:36 p.m.
On Thu, 2013-11-07 at 01:59 +0100, Mads Kiilerich wrote:
> # HG changeset patch
> # User Mads Kiilerich <madski@unity3d.com>
> # Date 1383785800 -3600
> #      Thu Nov 07 01:56:40 2013 +0100
> # Branch stable
> # Node ID bf9d390341502cc5bba579ad698e32d8f6795487
> # Parent  b6c3191618e2dc151347d307678d213049992456
> largefiles: update in two steps, handle interrupted updates better

Queued for stable. I'd generally prefer that cleanups (even obviously
preliminaries for bugfixes) be deferred to default.

Patch

diff --git a/hgext/largefiles/lfcommands.py b/hgext/largefiles/lfcommands.py
--- a/hgext/largefiles/lfcommands.py
+++ b/hgext/largefiles/lfcommands.py
@@ -438,45 +438,27 @@  def updatelfiles(ui, repo, filelist=None
         if filelist is not None:
             lfiles = [f for f in lfiles if f in filelist]
 
-        if lfiles:
-            if printmessage:
-                ui.status(_('getting changed largefiles\n'))
-            cachelfiles(ui, repo, None, lfiles)
-
+        update = {}
         updated, removed = 0, 0
         for lfile in lfiles:
-            # updates a single largefile and copies the state of its standin from
-            # the repository's dirstate to its state in the lfdirstate.
             abslfile = repo.wjoin(lfile)
             absstandin = repo.wjoin(lfutil.standin(lfile))
             if os.path.exists(absstandin):
                 if (os.path.exists(absstandin + '.orig') and
                     os.path.exists(abslfile)):
                     shutil.copyfile(abslfile, abslfile + '.orig')
-                update1 = 0
                 expecthash = lfutil.readstandin(repo, lfile)
                 if (expecthash != '' and
                     (not os.path.exists(abslfile) or
                      expecthash != lfutil.hashfile(abslfile))):
-                    if not lfutil.copyfromcache(repo, expecthash, lfile):
-                        # use normallookup() to allocate entry in largefiles
-                        # dirstate, because lack of it misleads
-                        # lfilesrepo.status() into recognition that such cache
-                        # missing files are REMOVED.
-                        if lfile not in repo[None]: # not switched to normal file
-                            util.unlinkpath(abslfile, ignoremissing=True)
-                        lfdirstate.normallookup(lfile)
-                        continue # don't try to set the mode
-                    else:
-                        # Synchronize largefile dirstate to the last modified
-                        # time of the file
-                        lfdirstate.normal(lfile)
-                    update1 = 1
-                mode = os.stat(absstandin).st_mode
-                if mode != os.stat(abslfile).st_mode:
-                    os.chmod(abslfile, mode)
-                    update1 = 1
-                updated += update1
+                    if lfile not in repo[None]: # not switched to normal file
+                        util.unlinkpath(abslfile, ignoremissing=True)
+                    # use normallookup() to allocate entry in largefiles
+                    # dirstate, because lack of it misleads
+                    # lfilesrepo.status() into recognition that such cache
+                    # missing files are REMOVED.
+                    lfdirstate.normallookup(lfile)
+                    update[lfile] = expecthash
             else:
                 # Remove lfiles for which the standin is deleted, unless the
                 # lfile is added to the repository again. This happens when a
@@ -486,6 +468,40 @@  def updatelfiles(ui, repo, filelist=None
                     repo.dirstate.normalize(lfile) not in repo[None]):
                     util.unlinkpath(abslfile)
                     removed += 1
+
+        # largefile processing might be slow and be interrupted - be prepared
+        lfdirstate.write()
+
+        if lfiles:
+            if printmessage:
+                ui.status(_('getting changed largefiles\n'))
+            cachelfiles(ui, repo, None, lfiles)
+
+        for lfile in lfiles:
+            update1 = 0
+
+            expecthash = update.get(lfile)
+            if expecthash:
+                if not lfutil.copyfromcache(repo, expecthash, lfile):
+                    # failed ... but already removed and set to normallookup
+                    continue
+                # Synchronize largefile dirstate to the last modified
+                # time of the file
+                lfdirstate.normal(lfile)
+                update1 = 1
+
+            # copy the state of largefile standin from the repository's
+            # dirstate to its state in the lfdirstate.
+            abslfile = repo.wjoin(lfile)
+            absstandin = repo.wjoin(lfutil.standin(lfile))
+            if os.path.exists(absstandin):
+                mode = os.stat(absstandin).st_mode
+                if mode != os.stat(abslfile).st_mode:
+                    os.chmod(abslfile, mode)
+                    update1 = 1
+
+            updated += update1
+
             state = repo.dirstate[lfutil.standin(lfile)]
             if state == 'n':
                 # When rebasing, we need to synchronize the standin and the
diff --git a/tests/test-largefiles-small-disk.t b/tests/test-largefiles-small-disk.t
--- a/tests/test-largefiles-small-disk.t
+++ b/tests/test-largefiles-small-disk.t
@@ -64,3 +64,4 @@  makes copies instead of hardlinks:
 The largefile is not created in .hg/largefiles:
 
   $ ls bob/.hg/largefiles
+  dirstate