Patchwork [3,of,6,import-refactor,V3] setup: refactor handling of modules with C/Python implementations

login
register
mail settings
Submitter Gregory Szorc
Date Dec. 4, 2015, 5:51 a.m.
Message ID <c5cea2ce781221a79169.1449208269@ubuntu-main>
Download mbox | patch
Permalink /patch/11801/
State Accepted
Delegated to: Yuya Nishihara
Headers show

Comments

Gregory Szorc - Dec. 4, 2015, 5:51 a.m.
# HG changeset patch
# User Gregory Szorc <gregory.szorc@gmail.com>
# Date 1449208092 28800
#      Thu Dec 03 21:48:12 2015 -0800
# Node ID c5cea2ce781221a791697d0f8c98c24ca083c071
# Parent  e7f2cafddb61f893011f19927778eec255df4805
setup: refactor handling of modules with C/Python implementations

Previously, .py files under mercurial/pure/ were copied to mercurial/*
during installation if we were performing a pure Python installation.

Now that the new import hooks and module load policy are in place, this
hackery from the past is no longer necessary.

With this patch, we stop copying modules from mercurial/pure/* to
mercurial/*. Instead, we preserve the files at their original
hierarchy, mirroring the source repository structure.

In addition, we always install the pure modules. Before, we would only
include the pure modules in the distribution/installation if the
install-time settings requested a pure Python installation. The upside
of this change is that CPython and PyPy can run from the same Mercurial
installation, making packaging and distribution of Mercurial simpler.

The inclusion of pure Python modules in the installation sounds
risky, as it could lead to inadvertent loading of non-C modules.
This shouldn't be a problem. The default module load policy is "C
only" (or at least will be shortly) and the only way to load pure
modules from an installation is if a) pure installation was requested
b) the HGMODULELOADPOLICY overrides the requirement for C modules.

The default module load policy as defined in source is a special string
whose default value from the checkout is equivalent to the "C only"
policy (again, not exactly the state right now). For pure
installations, this default policy is not appropriate and will not
work. This patch adds support for rewriting __init__.py during
installation to reflect the module load policy that should be in
place accoding to the installation settings. For default CPython
installs, the value in the source file will change but there will
be no functional change. For pure installations, the default policy
will be set to "py," allowing them to work without having to set
environment variables.
Yuya Nishihara - Dec. 4, 2015, 1:02 p.m.
On Thu, 03 Dec 2015 21:51:09 -0800, Gregory Szorc wrote:
> # HG changeset patch
> # User Gregory Szorc <gregory.szorc@gmail.com>
> # Date 1449208092 28800
> #      Thu Dec 03 21:48:12 2015 -0800
> # Node ID c5cea2ce781221a791697d0f8c98c24ca083c071
> # Parent  e7f2cafddb61f893011f19927778eec255df4805
> setup: refactor handling of modules with C/Python implementations

> --- a/setup.py
> +++ b/setup.py
> @@ -308,36 +308,45 @@ class hgbuildpy(build_py):
>      if convert2to3:
>          fixer_names = sorted(set(getfixers("lib2to3.fixes") +
>                                   getfixers("hgfixes")))
>  
>      def finalize_options(self):
>          build_py.finalize_options(self)
>  
>          if self.distribution.pure:
> -            if self.py_modules is None:
> -                self.py_modules = []
> -            for ext in self.distribution.ext_modules:
> -                if ext.name.startswith("mercurial."):
> -                    self.py_modules.append("mercurial.pure.%s" % ext.name[10:])
>              self.distribution.ext_modules = []
>          else:
>              h = os.path.join(get_python_inc(), 'Python.h')
>              if not os.path.exists(h):
>                  raise SystemExit('Python headers are required to build '
>                                   'Mercurial but weren\'t found in %s' % h)
>  
> -    def find_modules(self):
> -        modules = build_py.find_modules(self)
> -        for module in modules:
> -            if module[0] == "mercurial.pure":
> -                if module[1] != "__init__":
> -                    yield ("mercurial", module[1], module[2])
> -            else:
> -                yield module
> +    def run(self):
> +        if self.distribution.pure:
> +            modulepolicy = 'py'
> +        else:
> +            modulepolicy = 'c'
> +
> +        realcopyfile = file_util.copy_file
> +        def replacemodulepolicycopy(*args, **kwargs):
> +            dst, copied = realcopyfile(*args, **kwargs)
> +
> +            if dst.endswith('__init__.py'):
> +                content = open(dst, 'rb').read()
> +                content = content.replace(b'@MODULELOADPOLICY@',
> +                                          modulepolicy.encode(libdir_escape))
> +                with open(dst, 'wb') as fh:
> +                    fh.write(content)

"make local" overwrites mercurial/__init__.py. I'll make a couple of tweaks
on this patch and push the series to the clowncopter.

 - fix the overwrite issue
 - return dst, copied
 - override self.copy_file instead of patching file_util.copy_file

Patch

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -308,36 +308,45 @@  class hgbuildpy(build_py):
     if convert2to3:
         fixer_names = sorted(set(getfixers("lib2to3.fixes") +
                                  getfixers("hgfixes")))
 
     def finalize_options(self):
         build_py.finalize_options(self)
 
         if self.distribution.pure:
-            if self.py_modules is None:
-                self.py_modules = []
-            for ext in self.distribution.ext_modules:
-                if ext.name.startswith("mercurial."):
-                    self.py_modules.append("mercurial.pure.%s" % ext.name[10:])
             self.distribution.ext_modules = []
         else:
             h = os.path.join(get_python_inc(), 'Python.h')
             if not os.path.exists(h):
                 raise SystemExit('Python headers are required to build '
                                  'Mercurial but weren\'t found in %s' % h)
 
-    def find_modules(self):
-        modules = build_py.find_modules(self)
-        for module in modules:
-            if module[0] == "mercurial.pure":
-                if module[1] != "__init__":
-                    yield ("mercurial", module[1], module[2])
-            else:
-                yield module
+    def run(self):
+        if self.distribution.pure:
+            modulepolicy = 'py'
+        else:
+            modulepolicy = 'c'
+
+        realcopyfile = file_util.copy_file
+        def replacemodulepolicycopy(*args, **kwargs):
+            dst, copied = realcopyfile(*args, **kwargs)
+
+            if dst.endswith('__init__.py'):
+                content = open(dst, 'rb').read()
+                content = content.replace(b'@MODULELOADPOLICY@',
+                                          modulepolicy.encode(libdir_escape))
+                with open(dst, 'wb') as fh:
+                    fh.write(content)
+
+        file_util.copy_file = replacemodulepolicycopy
+        try:
+            build_py.run(self)
+        finally:
+            file_util.copy_file = realcopyfile
 
 class buildhgextindex(Command):
     description = 'generate prebuilt index of hgext (for frozen package)'
     user_options = []
     _indexfilename = 'hgext/__index__.py'
 
     def initialize_options(self):
         pass
@@ -473,16 +482,17 @@  cmdclass = {'build': hgbuild,
             'build_py': hgbuildpy,
             'build_hgextindex': buildhgextindex,
             'install_lib': hginstalllib,
             'install_scripts': hginstallscripts,
             'build_hgexe': buildhgexe,
             }
 
 packages = ['mercurial', 'mercurial.hgweb', 'mercurial.httpclient',
+            'mercurial.pure',
             'hgext', 'hgext.convert', 'hgext.highlight', 'hgext.zeroconf',
             'hgext.largefiles']
 
 common_depends = ['mercurial/util.h']
 
 osutil_ldflags = []
 
 if sys.platform == 'darwin':