Patchwork [1,of,2,v2] largefiles: always use filechunkiter when iterating files

login
register
mail settings
Submitter Mads Kiilerich
Date Oct. 14, 2016, 1:08 a.m.
Message ID <b7889580507c3d1d40e6.1476407304@localhost.localdomain>
Download mbox | patch
Permalink /patch/17074/
State Accepted
Headers show

Comments

Mads Kiilerich - Oct. 14, 2016, 1:08 a.m.
# HG changeset patch
# User Mads Kiilerich <madski@unity3d.com>
# Date 1476267738 -7200
#      Wed Oct 12 12:22:18 2016 +0200
# Node ID b7889580507c3d1d40e61904c7a2c2aba335c6cd
# Parent  c0410814002f467c24ef07ce73850ba15b306f8e
largefiles: always use filechunkiter when iterating files

Before, we would sometimes use the default iterator over large files. That
iterator is line based and would add extra buffering and use odd chunk sizes
which could give some overhead.

copyandhash can't just apply a filechunkiter as it sometimes is passed a
genuine generator when downloading remotely.

Patch

diff --git a/hgext/largefiles/lfutil.py b/hgext/largefiles/lfutil.py
--- a/hgext/largefiles/lfutil.py
+++ b/hgext/largefiles/lfutil.py
@@ -231,7 +231,8 @@  def copyfromcache(repo, hash, filename):
     # don't use atomic writes in the working copy.
     with open(path, 'rb') as srcfd:
         with wvfs(filename, 'wb') as destfd:
-            gothash = copyandhash(srcfd, destfd)
+            gothash = copyandhash(
+                util.filechunkiter(srcfd), destfd)
     if gothash != hash:
         repo.ui.warn(_('%s: data corruption in %s with hash %s\n')
                      % (filename, path, gothash))
diff --git a/hgext/largefiles/localstore.py b/hgext/largefiles/localstore.py
--- a/hgext/largefiles/localstore.py
+++ b/hgext/largefiles/localstore.py
@@ -10,6 +10,7 @@ 
 from __future__ import absolute_import
 
 from mercurial.i18n import _
+from mercurial import util
 
 from . import (
     basestore,
@@ -42,7 +43,8 @@  class localstore(basestore.basestore):
             raise basestore.StoreError(filename, hash, self.url,
                 _("can't get file locally"))
         with open(path, 'rb') as fd:
-            return lfutil.copyandhash(fd, tmpfile)
+            return lfutil.copyandhash(
+                util.filechunkiter(fd), tmpfile)
 
     def _verifyfiles(self, contents, filestocheck):
         failed = False
diff --git a/hgext/largefiles/remotestore.py b/hgext/largefiles/remotestore.py
--- a/hgext/largefiles/remotestore.py
+++ b/hgext/largefiles/remotestore.py
@@ -118,7 +118,7 @@  class remotestore(basestore.basestore):
         raise NotImplementedError('abstract method')
 
     def _get(self, hash):
-        '''Get file with the given hash from the remote store.'''
+        '''Get a iterator for content with the given hash.'''
         raise NotImplementedError('abstract method')
 
     def _stat(self, hashes):