Patchwork [11,of,13] largefiles: stat all largefiles in one batch before downloading

login
register
mail settings
Submitter Mads Kiilerich
Date April 16, 2013, 2:43 a.m.
Message ID <376ec3c2b2ae87d1328f.1366080208@mk-desktop>
Download mbox | patch
Permalink /patch/1341/
State Accepted
Commit 9d33d6e0d442df89d514859306dda3a1a0bd2749
Headers show

Comments

Mads Kiilerich - April 16, 2013, 2:43 a.m.
# HG changeset patch
# User Mads Kiilerich <madski@unity3d.com>
# Date 1366061863 -7200
#      Mon Apr 15 23:37:43 2013 +0200
# Node ID 376ec3c2b2ae87d1328fa2541b7ecf3acf6bbaf9
# Parent  d316529de6215c8f51f916d3202198c16a503ea4
largefiles: stat all largefiles in one batch before downloading

This avoids a lot of expensive roundtrips to remote repositories ... but might
be slightly slower for local operations.

This will also change some aborts on missing files to warnings. That will in
some situations make it possible to continue working on a repository with
missing largefiles.

Patch

diff --git a/hgext/largefiles/basestore.py b/hgext/largefiles/basestore.py
--- a/hgext/largefiles/basestore.py
+++ b/hgext/largefiles/basestore.py
@@ -62,12 +62,19 @@ 
         util.makedirs(lfutil.storepath(self.repo, ''))
 
         at = 0
+        available = self.exists(set(hash for (_filename, hash) in files))
         for filename, hash in files:
             ui.progress(_('getting largefiles'), at, unit='lfile',
                 total=len(files))
             at += 1
             ui.note(_('getting %s:%s\n') % (filename, hash))
 
+            if not available.get(hash):
+                ui.warn(_('%s: largefile %s not available from %s\n')
+                        % (filename, hash, self.url))
+                missing.append(filename)
+                continue
+
             storefilename = lfutil.storepath(self.repo, hash)
             tmpfile = util.atomictempfile(storefilename + '.tmp',
                                           createmode=self.repo.store.createmode)
diff --git a/hgext/largefiles/remotestore.py b/hgext/largefiles/remotestore.py
--- a/hgext/largefiles/remotestore.py
+++ b/hgext/largefiles/remotestore.py
@@ -47,16 +47,6 @@ 
                 fd.close()
 
     def _getfile(self, tmpfile, filename, hash):
-        # quit if the largefile isn't there
-        stat = self._stat([hash])[hash]
-        if stat == 1:
-            raise util.Abort(_('remotestore: largefile %s is invalid') % hash)
-        elif stat == 2:
-            raise util.Abort(_('remotestore: largefile %s is missing') % hash)
-        elif stat != 0:
-            raise RuntimeError('error getting file: unexpected response from '
-                               'statlfile (%r)' % stat)
-
         try:
             chunks = self._get(hash)
         except urllib2.HTTPError, e:
diff --git a/hgext/largefiles/wirestore.py b/hgext/largefiles/wirestore.py
--- a/hgext/largefiles/wirestore.py
+++ b/hgext/largefiles/wirestore.py
@@ -26,8 +26,9 @@ 
         return self.remote.getlfile(hash)
 
     def _stat(self, hashes):
-        '''For each hash, return 2 if the largefile is missing, 1 if it has a
-        mismatched checksum, or 0 if it is in good condition'''
+        '''For each hash, return 0 if it is available, other values if not.
+        It is usually 2 if the largefile is missing, but might be 1 the server
+        has a corrupted copy.'''
         batch = self.remote.batch()
         futures = {}
         for hash in hashes:
diff --git a/tests/test-largefiles-cache.t b/tests/test-largefiles-cache.t
--- a/tests/test-largefiles-cache.t
+++ b/tests/test-largefiles-cache.t
@@ -47,7 +47,7 @@ 
 
   $ hg update -r0
   getting changed largefiles
-  error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
+  large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg status
@@ -64,7 +64,7 @@ 
 
   $ hg update -r0
   getting changed largefiles
-  error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
+  large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg status
diff --git a/tests/test-largefiles.t b/tests/test-largefiles.t
--- a/tests/test-largefiles.t
+++ b/tests/test-largefiles.t
@@ -1306,7 +1306,7 @@ 
   $ rm ${USERCACHE}/7838695e10da2bb75ac1156565f40a2595fa2fa0
   $ hg up -r 6
   getting changed largefiles
-  error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+  large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
   1 largefiles updated, 2 removed
   4 files updated, 0 files merged, 2 files removed, 0 files unresolved
   $ rm normal3
@@ -1327,7 +1327,7 @@ 
   ! normal3
   $ hg up -Cr.
   getting changed largefiles
-  error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+  large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg st
@@ -1349,7 +1349,7 @@ 
   4 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   getting changed largefiles
-  error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+  large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
   1 largefiles updated, 0 removed
 
   $ hg rollback -q
@@ -1773,8 +1773,11 @@ 
   $ mv empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 .
   $ hg -R http-clone up --config largefiles.usercache=http-clone-usercache
   getting changed largefiles
-  abort: remotestore: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 is missing
-  [255]
+  f1: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 not available from http://localhost:$HGPORT2/
+  0 largefiles updated, 0 removed
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ hg -R http-clone st
+  ! f1
   $ hg -R http-clone up -Cqr null
 
 largefiles pulled on update - a largefile corrupted on the server:
@@ -1811,9 +1814,9 @@ 
   getting changed largefiles
   using http://localhost:$HGPORT2/
   sending capabilities command
+  sending batch command
   getting largefiles: 0/1 lfile (0.00%)
   getting f1:02a439e5c31c526465ab1a0ca1f431f76b827b90
-  sending batch command
   sending getlfile command
   found 02a439e5c31c526465ab1a0ca1f431f76b827b90 in store
   1 largefiles updated, 0 removed
diff --git a/tests/test-lfconvert.t b/tests/test-lfconvert.t
--- a/tests/test-lfconvert.t
+++ b/tests/test-lfconvert.t
@@ -343,7 +343,7 @@ 
   $ rm largefiles-repo/.hg/largefiles/*
   $ hg lfconvert --to-normal issue3519 normalized3519
   initializing destination normalized3519
-  error getting id 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 from url file:$TESTTMP/largefiles-repo for file large: can't get file locally (glob)
+  large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:$TESTTMP/largefiles-repo
   abort: missing largefile 'large' from revision d4892ec57ce212905215fad1d9018f56b99202ad
   [255]