Patchwork D6084: packaging: split downloading code into own module

login
register
mail settings
Submitter phabricator
Date March 10, 2019, 1:51 a.m.
Message ID <2bae62e37c9d1c01f5a9fae183356601@localhost.localdomain>
Download mbox | patch
Permalink /patch/39182/
State Not Applicable
Headers show

Comments

phabricator - March 10, 2019, 1:51 a.m.
This revision was automatically updated to reflect the committed changes.
Closed by commit rHGc2237fe1359e: packaging: split downloading code into own module (authored by indygreg, committed by ).

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D6084?vs=14391&id=14439

REVISION DETAIL
  https://phab.mercurial-scm.org/D6084

AFFECTED FILES
  contrib/packaging/hgpackaging/downloads.py
  contrib/packaging/hgpackaging/util.py
  contrib/packaging/inno/build.py
  tests/test-check-code.t

CHANGE DETAILS




To: indygreg, #hg-reviewers
Cc: mercurial-devel

Patch

diff --git a/tests/test-check-code.t b/tests/test-check-code.t
--- a/tests/test-check-code.t
+++ b/tests/test-check-code.t
@@ -12,6 +12,7 @@ 
   > -X hgext/fsmonitor/pywatchman \
   > -X mercurial/thirdparty \
   > | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false
+  Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob)
   Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob)
   Skipping i18n/polib.py it has no-che?k-code (glob)
diff --git a/contrib/packaging/inno/build.py b/contrib/packaging/inno/build.py
--- a/contrib/packaging/inno/build.py
+++ b/contrib/packaging/inno/build.py
@@ -87,8 +87,10 @@ 
     for finding the Python 2.7 toolchain. So, we require the environment
     to already be configured with an active toolchain.
     """
+    from hgpackaging.downloads import (
+        download_entry,
+    )
     from hgpackaging.util import (
-        download_entry,
         extract_tar_to_directory,
         extract_zip_to_directory,
     )
diff --git a/contrib/packaging/hgpackaging/util.py b/contrib/packaging/hgpackaging/util.py
--- a/contrib/packaging/hgpackaging/util.py
+++ b/contrib/packaging/hgpackaging/util.py
@@ -7,117 +7,11 @@ 
 
 # no-check-code because Python 3 native.
 
-import gzip

-import hashlib

 import pathlib
 import tarfile
-import urllib.request

 import zipfile
 
 
-def hash_path(p: pathlib.Path):

-    h = hashlib.sha256()

-

-    with p.open('rb') as fh:

-        while True:

-            chunk = fh.read(65536)

-            if not chunk:

-                break

-

-            h.update(chunk)

-

-    return h.hexdigest()

-

-

-class IntegrityError(Exception):

-    """Represents an integrity error when downloading a URL."""

-

-

-def secure_download_stream(url, size, sha256):

-    """Securely download a URL to a stream of chunks.

-

-    If the integrity of the download fails, an IntegrityError is

-    raised.

-    """

-    h = hashlib.sha256()

-    length = 0

-

-    with urllib.request.urlopen(url) as fh:

-        if not url.endswith('.gz') and fh.info().get('Content-Encoding') == 'gzip':

-            fh = gzip.GzipFile(fileobj=fh)

-

-        while True:

-            chunk = fh.read(65536)

-            if not chunk:

-                break

-

-            h.update(chunk)

-            length += len(chunk)

-

-            yield chunk

-

-    digest = h.hexdigest()

-

-    if length != size:

-        raise IntegrityError('size mismatch on %s: wanted %d; got %d' % (

-            url, size, length))

-

-    if digest != sha256:

-        raise IntegrityError('sha256 mismatch on %s: wanted %s; got %s' % (

-            url, sha256, digest))

-

-

-def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str):

-    """Download a URL to a filesystem path, possibly with verification."""

-

-    # We download to a temporary file and rename at the end so there's

-    # no chance of the final file being partially written or containing

-    # bad data.

-    print('downloading %s to %s' % (url, path))

-

-    if path.exists():

-        good = True

-

-        if path.stat().st_size != size:

-            print('existing file size is wrong; removing')

-            good = False

-

-        if good:

-            if hash_path(path) != sha256:

-                print('existing file hash is wrong; removing')

-                good = False

-

-        if good:

-            print('%s exists and passes integrity checks' % path)

-            return

-

-        path.unlink()

-

-    tmp = path.with_name('%s.tmp' % path.name)

-

-    try:

-        with tmp.open('wb') as fh:

-            for chunk in secure_download_stream(url, size, sha256):

-                fh.write(chunk)

-    except IntegrityError:

-        tmp.unlink()

-        raise

-

-    tmp.rename(path)

-    print('successfully downloaded %s' % url)

-

-

-def download_entry(entry: dict, dest_path: pathlib.Path, local_name=None) -> pathlib.Path:

-    url = entry['url']

-

-    local_name = local_name or url[url.rindex('/') + 1:]

-

-    local_path = dest_path / local_name

-    download_to_path(url, local_path, entry['size'], entry['sha256'])

-

-    return local_path

-

-

 def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path):
     with tarfile.open(source, 'r') as tf:
         tf.extractall(dest)
diff --git a/contrib/packaging/hgpackaging/util.py b/contrib/packaging/hgpackaging/downloads.py
copy from contrib/packaging/hgpackaging/util.py
copy to contrib/packaging/hgpackaging/downloads.py
--- a/contrib/packaging/hgpackaging/util.py
+++ b/contrib/packaging/hgpackaging/downloads.py
@@ -1,4 +1,4 @@ 
-# util.py - Common packaging utility code.

+# downloads.py - Code for downloading dependencies.

 #
 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
 #
@@ -10,9 +10,7 @@ 
 import gzip
 import hashlib
 import pathlib
-import tarfile

 import urllib.request
-import zipfile

 
 
 def hash_path(p: pathlib.Path):
@@ -116,13 +114,3 @@ 
     download_to_path(url, local_path, entry['size'], entry['sha256'])
 
     return local_path
-

-

-def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path):

-    with tarfile.open(source, 'r') as tf:

-        tf.extractall(dest)

-

-

-def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path):

-    with zipfile.ZipFile(source, 'r') as zf:

-        zf.extractall(dest)