Patchwork [04,of,10,V9] bookmarks: introduce binary encoding

login
register
mail settings
Submitter Stanislau Hlebik
Date Nov. 13, 2016, 10:30 a.m.
Message ID <8b6ab3b8df3aae90ac72.1479033015@dev1918.lla1.facebook.com>
Download mbox | patch
Permalink /patch/17535/
State Changes Requested
Headers show

Comments

Stanislau Hlebik - Nov. 13, 2016, 10:30 a.m.
# HG changeset patch
# User Stanislau Hlebik <stash@fb.com>
# Date 1479032456 28800
#      Sun Nov 13 02:20:56 2016 -0800
# Branch stable
# Node ID 8b6ab3b8df3aae90ac72d7fa4603e7d5b4ab55e9
# Parent  0917ce41b232a5fc2a6e503cbad15879e037a2f9
bookmarks: introduce binary encoding

Bookmarks binary encoding will be used for `bookmarks` bundle2 part.
The format is: <4 bytes - bookmark size, big-endian><bookmark>
               <1 byte - 0 if node is empty 1 otherwise><20 bytes node>
BookmarksEncodeError and BookmarksDecodeError maybe thrown if input is
incorrect.
Gregory Szorc - Nov. 16, 2016, 4:23 a.m.
On Sun, Nov 13, 2016 at 2:30 AM, Stanislau Hlebik <stash@fb.com> wrote:

> # HG changeset patch
> # User Stanislau Hlebik <stash@fb.com>
> # Date 1479032456 28800
> #      Sun Nov 13 02:20:56 2016 -0800
> # Branch stable
> # Node ID 8b6ab3b8df3aae90ac72d7fa4603e7d5b4ab55e9
> # Parent  0917ce41b232a5fc2a6e503cbad15879e037a2f9
> bookmarks: introduce binary encoding
>
> Bookmarks binary encoding will be used for `bookmarks` bundle2 part.
> The format is: <4 bytes - bookmark size, big-endian><bookmark>
>                <1 byte - 0 if node is empty 1 otherwise><20 bytes node>
> BookmarksEncodeError and BookmarksDecodeError maybe thrown if input is
> incorrect.
>

I'm pretty happy with this series up until and including this patch.


>
> diff --git a/mercurial/bookmarks.py b/mercurial/bookmarks.py
> --- a/mercurial/bookmarks.py
> +++ b/mercurial/bookmarks.py
> @@ -8,7 +8,9 @@
>  from __future__ import absolute_import
>
>  import errno
> +import io
>  import os
> +import struct
>
>  from .i18n import _
>  from .node import (
> @@ -23,6 +25,71 @@
>      util,
>  )
>
> +_NONEMPTYNODE = struct.pack('?', False)
> +_EMPTYNODE = struct.pack('?', True)
> +
> +def _unpackbookmarksize(stream):
> +    """Returns 0 if stream is empty.
> +    """
> +
> +    intstruct = struct.Struct('>i')
> +    expectedsize = intstruct.size
> +    encodedbookmarksize = stream.read(expectedsize)
> +    if not encodedbookmarksize:
> +        return 0
> +    if len(encodedbookmarksize) != expectedsize:
> +        raise ValueError(
> +            _('cannot decode bookmark size: '
> +              'expected size: %d, actual size: %d') %
> +            (expectedsize, len(encodedbookmarksize)))
> +    return intstruct.unpack(encodedbookmarksize)[0]
> +
> +def encodebookmarks(bookmarks):
> +    """Encodes bookmark to node mapping to the byte string.
> +
> +    Format: <4 bytes - bookmark size><bookmark>
> +            <1 byte - 0 if node is empty 1 otherwise><20 bytes node>
> +    Node may be 20 byte binary string or empty
> +    """
> +
> +    intstruct = struct.Struct('>i')
> +    for bookmark, node in sorted(bookmarks.iteritems()):
> +        yield intstruct.pack(len(bookmark))
> +        yield encoding.fromlocal(bookmark)
> +        if node:
> +            if len(node) != 20:
> +                raise ValueError(_('node must be 20 or bytes long'))
> +            yield _NONEMPTYNODE
> +            yield node
> +        else:
> +            yield _EMPTYNODE
> +
> +def decodebookmarks(buf):
> +    """Decodes buffer into bookmark to node mapping.
> +
> +    Node is either 20 bytes or empty.
> +    """
> +
> +    stream = io.BytesIO(buf)
> +    bookmarks = {}
> +    bookmarksize = _unpackbookmarksize(stream)
> +    boolstruct = struct.Struct('?')
> +    while bookmarksize:
> +        bookmark = stream.read(bookmarksize)
> +        if len(bookmark) != bookmarksize:
> +            raise ValueError(
> +                _('cannot decode bookmark: expected size: %d, '
> +                'actual size: %d') % (bookmarksize, len(bookmark)))
> +        bookmark = encoding.tolocal(bookmark)
> +        packedemptynodeflag = stream.read(boolstruct.size)
> +        emptynode = boolstruct.unpack(packedemptynodeflag)[0]
> +        node = ''
> +        if not emptynode:
> +            node = stream.read(20)
> +        bookmarks[bookmark] = node
> +        bookmarksize = _unpackbookmarksize(stream)
> +    return bookmarks
> +
>  def _getbkfile(repo):
>      """Hook so that extensions that mess with the store can hook bm
> storage.
>
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>

Patch

diff --git a/mercurial/bookmarks.py b/mercurial/bookmarks.py
--- a/mercurial/bookmarks.py
+++ b/mercurial/bookmarks.py
@@ -8,7 +8,9 @@ 
 from __future__ import absolute_import
 
 import errno
+import io
 import os
+import struct
 
 from .i18n import _
 from .node import (
@@ -23,6 +25,71 @@ 
     util,
 )
 
+_NONEMPTYNODE = struct.pack('?', False)
+_EMPTYNODE = struct.pack('?', True)
+
+def _unpackbookmarksize(stream):
+    """Returns 0 if stream is empty.
+    """
+
+    intstruct = struct.Struct('>i')
+    expectedsize = intstruct.size
+    encodedbookmarksize = stream.read(expectedsize)
+    if not encodedbookmarksize:
+        return 0
+    if len(encodedbookmarksize) != expectedsize:
+        raise ValueError(
+            _('cannot decode bookmark size: '
+              'expected size: %d, actual size: %d') %
+            (expectedsize, len(encodedbookmarksize)))
+    return intstruct.unpack(encodedbookmarksize)[0]
+
+def encodebookmarks(bookmarks):
+    """Encodes bookmark to node mapping to the byte string.
+
+    Format: <4 bytes - bookmark size><bookmark>
+            <1 byte - 0 if node is empty 1 otherwise><20 bytes node>
+    Node may be 20 byte binary string or empty
+    """
+
+    intstruct = struct.Struct('>i')
+    for bookmark, node in sorted(bookmarks.iteritems()):
+        yield intstruct.pack(len(bookmark))
+        yield encoding.fromlocal(bookmark)
+        if node:
+            if len(node) != 20:
+                raise ValueError(_('node must be 20 or bytes long'))
+            yield _NONEMPTYNODE
+            yield node
+        else:
+            yield _EMPTYNODE
+
+def decodebookmarks(buf):
+    """Decodes buffer into bookmark to node mapping.
+
+    Node is either 20 bytes or empty.
+    """
+
+    stream = io.BytesIO(buf)
+    bookmarks = {}
+    bookmarksize = _unpackbookmarksize(stream)
+    boolstruct = struct.Struct('?')
+    while bookmarksize:
+        bookmark = stream.read(bookmarksize)
+        if len(bookmark) != bookmarksize:
+            raise ValueError(
+                _('cannot decode bookmark: expected size: %d, '
+                'actual size: %d') % (bookmarksize, len(bookmark)))
+        bookmark = encoding.tolocal(bookmark)
+        packedemptynodeflag = stream.read(boolstruct.size)
+        emptynode = boolstruct.unpack(packedemptynodeflag)[0]
+        node = ''
+        if not emptynode:
+            node = stream.read(20)
+        bookmarks[bookmark] = node
+        bookmarksize = _unpackbookmarksize(stream)
+    return bookmarks
+
 def _getbkfile(repo):
     """Hook so that extensions that mess with the store can hook bm storage.