Patchwork [4,of,4,v2] mpatch: implement cffi version of mpatch

login
register
mail settings
Submitter Maciej Fijalkowski
Date June 23, 2016, 6:57 a.m.
Message ID <ec1da6e456500f9ccc67.1466665031@brick.arcode.com>
Download mbox | patch
Permalink /patch/15578/
State Changes Requested
Headers show

Comments

Maciej Fijalkowski - June 23, 2016, 6:57 a.m.
# HG changeset patch
# User Maciej Fijalkowski <fijall@gmail.com>
# Date 1466664974 -7200
#      Thu Jun 23 08:56:14 2016 +0200
# Node ID ec1da6e456500f9ccc6707eaf4ca4b0b9210a3de
# Parent  42e8ffc4adb694a2a0a4787e129aa4e7d0296631
mpatch: implement cffi version of mpatch

This is a first go at working, but not 100% efficient version of mpatch
for PyPy that uses cffi. Additional improvements might follow, but it
already makes mercurial far more usable under pypy.
Yuya Nishihara - June 27, 2016, 2:05 p.m.
On Thu, 23 Jun 2016 08:57:11 +0200, Maciej Fijalkowski wrote:
> # HG changeset patch
> # User Maciej Fijalkowski <fijall@gmail.com>
> # Date 1466664974 -7200
> #      Thu Jun 23 08:56:14 2016 +0200
> # Node ID ec1da6e456500f9ccc6707eaf4ca4b0b9210a3de
> # Parent  42e8ffc4adb694a2a0a4787e129aa4e7d0296631
> mpatch: implement cffi version of mpatch

> +ffi.set_source("_mpatch_cffi", open(mpatch_c).read(),
> +               include_dirs=["mercurial"])

Maybe we'd better put the _mpatch_cffi module under mercurial package?

> -struct flist* cpy_get_next_item(void* bins, ssize_t pos)
> +struct flist *cpy_get_item(void *bins, ssize_t pos)

Nit: this can be a local function. And this hunk should belong to the patch 2.

> +        def patches(text, bins):
> +            lgt = len(bins)
> +            all = []
> +            if not lgt:
> +                return text
> +            arg = (all, bins)
> +            patch = lib.fold(ffi.new_handle(arg), lib.cffi_get_next_item, 0,
> +                             lgt)
> +            if not patch:
> +                raise mpatchError("cannot decode chunk")
> +            outlen = lib.calcsize(len(text), patch)
> +            if outlen < 0:
> +                lib.lfree(patch)
> +                raise mpatchError("inconsistency detected")
> +            buf = ffi.new("char[]", outlen)
> +            if lib.apply(buf, text, len(text), patch) < 0:
> +                lib.lfree(patch)
> +                raise mpatchError("error applying patches")
> +            res = ffi.buffer(buf, outlen)[:]
> +            lib.lfree(patch)

I don't know how CFFI works, but if we can use try-finally, I'll do

    if not patch:
        raise mpatchError("cannot decode chunk")
    try:
        ...
    finally:
        lib.lfree(patch)

Patch

diff -r 42e8ffc4adb6 -r ec1da6e45650 mercurial/__init__.py
--- a/mercurial/__init__.py	Tue Jun 07 15:35:58 2016 +0200
+++ b/mercurial/__init__.py	Thu Jun 23 08:56:14 2016 +0200
@@ -19,6 +19,7 @@ 
 __all__ = []
 
 modulepolicy = policy.policy
+policynoc = policy.policynoc
 
 # Modules that have both Python and C implementations. See also the
 # set of .py files under mercurial/pure/.
diff -r 42e8ffc4adb6 -r ec1da6e45650 mercurial/build_mpatch_cffi.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/build_mpatch_cffi.py	Thu Jun 23 08:56:14 2016 +0200
@@ -0,0 +1,34 @@ 
+
+from __future__ import absolute_import
+
+import os
+import cffi
+
+ffi = cffi.FFI()
+mpatch_c = os.path.join(os.path.dirname(__file__), 'mpatch.c')
+ffi.set_source("_mpatch_cffi", open(mpatch_c).read(),
+               include_dirs=["mercurial"])
+ffi.cdef("""
+
+struct frag {
+       int start, end, len;
+       const char *data;
+};
+
+struct flist {
+       struct frag *base, *head, *tail;
+};
+
+extern "Python" struct flist* cffi_get_next_item(void*, ssize_t);
+
+int decode(const char *bin, size_t len, struct flist** res);
+struct flist *combine(struct flist *a, struct flist *b);
+ssize_t calcsize(size_t len, struct flist *l);
+void lfree(struct flist *a);
+static int apply(char *buf, const char *orig, size_t len, struct flist *l);
+struct flist *fold(void *bins, struct flist* (*get_next_item)(void*, ssize_t),
+                   ssize_t start, ssize_t end);
+""")
+
+if __name__ == '__main__':
+    ffi.compile()
diff -r 42e8ffc4adb6 -r ec1da6e45650 mercurial/mpatch.h
--- a/mercurial/mpatch.h	Tue Jun 07 15:35:58 2016 +0200
+++ b/mercurial/mpatch.h	Thu Jun 23 08:56:14 2016 +0200
@@ -8,21 +8,21 @@ 
 #define MPATCH_ERR_INVALID_PATCH -3
 
 struct frag {
-       int start, end, len;
-       const char *data;
+	int start, end, len;
+	const char *data;
 };
 
 struct flist {
-       struct frag *base, *head, *tail;
+	struct frag *base, *head, *tail;
 };
 
-int decode(const char *bin, ssize_t len, struct flist** res);
+int decode(const char *bin, ssize_t len, struct flist **res);
 struct flist *combine(struct flist *a, struct flist *b);
 ssize_t calcsize(ssize_t len, struct flist *l);
 void lfree(struct flist *a);
 int apply(char *buf, const char *orig, ssize_t len, struct flist *l);
-ssize_t _patchedsize(long orig, char* bin, ssize_t patchlen);
+ssize_t _patchedsize(long orig, char *bin, ssize_t patchlen);
 struct flist *fold(void *bins, struct flist* (*get_next_item)(void*, ssize_t),
-	               ssize_t start, ssize_t end);
+					ssize_t start, ssize_t end);
 
-#endif // _HG_MPATCH_H_
\ No newline at end of file
+#endif
diff -r 42e8ffc4adb6 -r ec1da6e45650 mercurial/mpatch_module.c
--- a/mercurial/mpatch_module.c	Tue Jun 07 15:35:58 2016 +0200
+++ b/mercurial/mpatch_module.c	Thu Jun 23 08:56:14 2016 +0200
@@ -10,7 +10,7 @@ 
 
 static PyObject *mpatch_Error;
 
-struct flist* cpy_get_next_item(void* bins, ssize_t pos)
+struct flist *cpy_get_item(void *bins, ssize_t pos)
 {
 	const char *buffer;
 	struct flist *res;
@@ -52,7 +52,7 @@ 
 	if (PyObject_AsCharBuffer(text, &in, &inlen))
 		return NULL;
 
-	patch = fold(bins, cpy_get_next_item, 0, len);
+	patch = fold(bins, cpy_get_item, 0, len);
 	if (!patch) {
 		return NULL;
 	}
diff -r 42e8ffc4adb6 -r ec1da6e45650 mercurial/pure/mpatch.py
--- a/mercurial/pure/mpatch.py	Tue Jun 07 15:35:58 2016 +0200
+++ b/mercurial/pure/mpatch.py	Thu Jun 23 08:56:14 2016 +0200
@@ -9,8 +9,10 @@ 
 
 import struct
 
-from . import pycompat
+from . import policy, pycompat
 stringio = pycompat.stringio
+modulepolicy = policy.policy
+policynocffi = policy.policynocffi
 
 class mpatchError(Exception):
     """error raised when a delta cannot be decoded
@@ -125,3 +127,43 @@ 
 
     outlen += orig - last
     return outlen
+
+if modulepolicy not in policynocffi:
+    try:
+        from _mpatch_cffi import ffi, lib
+    except ImportError:
+        if modulepolicy == 'cffi': # strict cffi import
+            raise
+    else:
+        @ffi.def_extern()
+        def cffi_get_next_item(arg, pos):
+            all, bins = ffi.from_handle(arg)
+            container = ffi.new("struct flist*[1]")
+            to_pass = ffi.new("char[]", str(bins[pos]))
+            all.append(to_pass)
+            r = lib.decode(to_pass, len(to_pass) - 1, container)
+            if r < 0:
+                return ffi.NULL
+            return container[0]
+
+        def patches(text, bins):
+            lgt = len(bins)
+            all = []
+            if not lgt:
+                return text
+            arg = (all, bins)
+            patch = lib.fold(ffi.new_handle(arg), lib.cffi_get_next_item, 0,
+                             lgt)
+            if not patch:
+                raise mpatchError("cannot decode chunk")
+            outlen = lib.calcsize(len(text), patch)
+            if outlen < 0:
+                lib.lfree(patch)
+                raise mpatchError("inconsistency detected")
+            buf = ffi.new("char[]", outlen)
+            if lib.apply(buf, text, len(text), patch) < 0:
+                lib.lfree(patch)
+                raise mpatchError("error applying patches")
+            res = ffi.buffer(buf, outlen)[:]
+            lib.lfree(patch)
+            return res
diff -r 42e8ffc4adb6 -r ec1da6e45650 setup.py
--- a/setup.py	Tue Jun 07 15:35:58 2016 +0200
+++ b/setup.py	Thu Jun 23 08:56:14 2016 +0200
@@ -262,7 +262,8 @@ 
 
 
 class hgdist(Distribution):
-    pure = ispypy
+    pure = False
+    cffi = ispypy
 
     global_options = Distribution.global_options + \
                      [('pure', None, "use pure (slow) Python "
@@ -316,6 +317,10 @@ 
 
         if self.distribution.pure:
             self.distribution.ext_modules = []
+        elif self.distribution.cffi:
+            from mercurial import build_mpatch_cffi
+            exts = [build_mpatch_cffi.ffi.distutils_extension()]
+            self.distribution.ext_modules = exts
         else:
             h = os.path.join(get_python_inc(), 'Python.h')
             if not os.path.exists(h):