new file mode 100644
@@ -0,0 +1,779 @@
+# Copyright 2014-present Facebook, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# * Neither the name Facebook nor the names of its contributors may be used to
+# endorse or promote products derived from this software without specific
+# prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import errno
+import math
+import socket
+import subprocess
+import time
+
+# Sometimes it's really hard to get Python extensions to compile,
+# so fall back to a pure Python implementation.
+try:
+ import bser
+except ImportError:
+ import pybser as bser
+
+import capabilities
+
+if os.name == 'nt':
+ import ctypes
+ import ctypes.wintypes
+
+ wintypes = ctypes.wintypes
+ GENERIC_READ = 0x80000000
+ GENERIC_WRITE = 0x40000000
+ FILE_FLAG_OVERLAPPED = 0x40000000
+ OPEN_EXISTING = 3
+ INVALID_HANDLE_VALUE = -1
+ FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000
+ FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100
+ FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200
+ WAIT_TIMEOUT = 0x00000102
+ WAIT_OBJECT_0 = 0x00000000
+ ERROR_IO_PENDING = 997
+
+ class OVERLAPPED(ctypes.Structure):
+ _fields_ = [
+ ("Internal", wintypes.ULONG), ("InternalHigh", wintypes.ULONG),
+ ("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD),
+ ("hEvent", wintypes.HANDLE)
+ ]
+
+ def __init__(self):
+ self.Offset = 0
+ self.OffsetHigh = 0
+ self.hEvent = 0
+
+ LPDWORD = ctypes.POINTER(wintypes.DWORD)
+
+ CreateFile = ctypes.windll.kernel32.CreateFileA
+ CreateFile.argtypes = [wintypes.LPSTR, wintypes.DWORD, wintypes.DWORD,
+ wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD,
+ wintypes.HANDLE]
+ CreateFile.restype = wintypes.HANDLE
+
+ CloseHandle = ctypes.windll.kernel32.CloseHandle
+ CloseHandle.argtypes = [wintypes.HANDLE]
+ CloseHandle.restype = wintypes.BOOL
+
+ ReadFile = ctypes.windll.kernel32.ReadFile
+ ReadFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
+ LPDWORD, ctypes.POINTER(OVERLAPPED)]
+ ReadFile.restype = wintypes.BOOL
+
+ WriteFile = ctypes.windll.kernel32.WriteFile
+ WriteFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
+ LPDWORD, ctypes.POINTER(OVERLAPPED)]
+ WriteFile.restype = wintypes.BOOL
+
+ GetLastError = ctypes.windll.kernel32.GetLastError
+ GetLastError.argtypes = []
+ GetLastError.restype = wintypes.DWORD
+
+ FormatMessage = ctypes.windll.kernel32.FormatMessageA
+ FormatMessage.argtypes = [wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD,
+ wintypes.DWORD, ctypes.POINTER(wintypes.LPSTR),
+ wintypes.DWORD, wintypes.LPVOID]
+ FormatMessage.restype = wintypes.DWORD
+
+ LocalFree = ctypes.windll.kernel32.LocalFree
+
+ GetOverlappedResultEx = ctypes.windll.kernel32.GetOverlappedResultEx
+ GetOverlappedResultEx.argtypes = [wintypes.HANDLE,
+ ctypes.POINTER(OVERLAPPED), LPDWORD,
+ wintypes.DWORD, wintypes.BOOL]
+ GetOverlappedResultEx.restype = wintypes.BOOL
+
+ CancelIoEx = ctypes.windll.kernel32.CancelIoEx
+ CancelIoEx.argtypes = [wintypes.HANDLE, ctypes.POINTER(OVERLAPPED)]
+ CancelIoEx.restype = wintypes.BOOL
+
+# 2 bytes marker, 1 byte int size, 8 bytes int64 value
+sniff_len = 13
+
+# This is a helper for debugging the client.
+_debugging = False
+if _debugging:
+
+ def log(fmt, *args):
+ print('[%s] %s' %
+ (time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()),
+ fmt % args[:]))
+else:
+
+ def log(fmt, *args):
+ pass
+
+
+class WatchmanError(Exception):
+ pass
+
+
+class SocketTimeout(WatchmanError):
+ """A specialized exception raised for socket timeouts during communication to/from watchman.
+ This makes it easier to implement non-blocking loops as callers can easily distinguish
+ between a routine timeout and an actual error condition.
+
+ Note that catching WatchmanError will also catch this as it is a super-class, so backwards
+ compatibility in exception handling is preserved.
+ """
+
+
+class CommandError(WatchmanError):
+ """error returned by watchman
+
+ self.msg is the message returned by watchman.
+ """
+
+ def __init__(self, msg, cmd=None):
+ self.msg = msg
+ self.cmd = cmd
+ super(CommandError, self).__init__('watchman command error: %s' % msg)
+
+ def setCommand(self, cmd):
+ self.cmd = cmd
+
+ def __str__(self):
+ if self.cmd:
+ return '%s, while executing %s' % (self.msg, self.cmd)
+ return self.msg
+
+
+class Transport(object):
+ """ communication transport to the watchman server """
+ buf = None
+
+ def close(self):
+ """ tear it down """
+ raise NotImplementedError()
+
+ def readBytes(self, size):
+ """ read size bytes """
+ raise NotImplementedError()
+
+ def write(self, buf):
+ """ write some data """
+ raise NotImplementedError()
+
+ def setTimeout(self, value):
+ pass
+
+ def readLine(self):
+ """ read a line
+ Maintains its own buffer, callers of the transport should not mix
+ calls to readBytes and readLine.
+ """
+ if self.buf is None:
+ self.buf = []
+
+ # Buffer may already have a line if we've received unilateral
+ # response(s) from the server
+ if len(self.buf) == 1 and "\n" in self.buf[0]:
+ (line, b) = self.buf[0].split("\n", 1)
+ self.buf = [b]
+ return line
+
+ while True:
+ b = self.readBytes(4096)
+ if "\n" in b:
+ result = ''.join(self.buf)
+ (line, b) = b.split("\n", 1)
+ self.buf = [b]
+ return result + line
+ self.buf.append(b)
+
+
+class Codec(object):
+ """ communication encoding for the watchman server """
+ transport = None
+
+ def __init__(self, transport):
+ self.transport = transport
+
+ def receive(self):
+ raise NotImplementedError()
+
+ def send(self, *args):
+ raise NotImplementedError()
+
+ def setTimeout(self, value):
+ self.transport.setTimeout(value)
+
+
+class UnixSocketTransport(Transport):
+ """ local unix domain socket transport """
+ sock = None
+
+ def __init__(self, sockpath, timeout):
+ self.sockpath = sockpath
+ self.timeout = timeout
+
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ try:
+ sock.settimeout(self.timeout)
+ sock.connect(self.sockpath)
+ self.sock = sock
+ except socket.error as e:
+ raise WatchmanError('unable to connect to %s: %s' %
+ (self.sockpath, e))
+
+ def close(self):
+ self.sock.close()
+ self.sock = None
+
+ def setTimeout(self, value):
+ self.timeout = value
+ self.sock.settimeout(self.timeout)
+
+ def readBytes(self, size):
+ try:
+ buf = [self.sock.recv(size)]
+ if not buf[0]:
+ raise WatchmanError('empty watchman response')
+ return buf[0]
+ except socket.timeout:
+ raise SocketTimeout('timed out waiting for response')
+
+ def write(self, data):
+ try:
+ self.sock.sendall(data)
+ except socket.timeout:
+ raise SocketTimeout('timed out sending query command')
+
+
+class WindowsNamedPipeTransport(Transport):
+ """ connect to a named pipe """
+
+ def __init__(self, sockpath, timeout):
+ self.sockpath = sockpath
+ self.timeout = int(math.ceil(timeout * 1000))
+ self._iobuf = None
+
+ self.pipe = CreateFile(sockpath, GENERIC_READ | GENERIC_WRITE, 0, None,
+ OPEN_EXISTING, FILE_FLAG_OVERLAPPED, None)
+
+ if self.pipe == INVALID_HANDLE_VALUE:
+ self.pipe = None
+ self._raise_win_err('failed to open pipe %s' % sockpath,
+ GetLastError())
+
+ def _win32_strerror(self, err):
+ """ expand a win32 error code into a human readable message """
+
+ # FormatMessage will allocate memory and assign it here
+ buf = ctypes.c_char_p()
+ FormatMessage(
+ FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER
+ | FORMAT_MESSAGE_IGNORE_INSERTS, None, err, 0, buf, 0, None)
+ try:
+ return buf.value
+ finally:
+ LocalFree(buf)
+
+ def _raise_win_err(self, msg, err):
+ raise IOError('%s win32 error code: %d %s' %
+ (msg, err, self._win32_strerror(err)))
+
+ def close(self):
+ if self.pipe:
+ CloseHandle(self.pipe)
+ self.pipe = None
+
+ def readBytes(self, size):
+ """ A read can block for an unbounded amount of time, even if the
+ kernel reports that the pipe handle is signalled, so we need to
+ always perform our reads asynchronously
+ """
+
+ # try to satisfy the read from any buffered data
+ if self._iobuf:
+ if size >= len(self._iobuf):
+ res = self._iobuf
+ self.buf = None
+ return res
+ res = self._iobuf[:size]
+ self._iobuf = self._iobuf[size:]
+ return res
+
+ # We need to initiate a read
+ buf = ctypes.create_string_buffer(size)
+ olap = OVERLAPPED()
+
+ log('made read buff of size %d', size)
+
+ # ReadFile docs warn against sending in the nread parameter for async
+ # operations, so we always collect it via GetOverlappedResultEx
+ immediate = ReadFile(self.pipe, buf, size, None, olap)
+
+ if not immediate:
+ err = GetLastError()
+ if err != ERROR_IO_PENDING:
+ self._raise_win_err('failed to read %d bytes' % size,
+ GetLastError())
+
+ nread = wintypes.DWORD()
+ if not GetOverlappedResultEx(self.pipe, olap, nread,
+ 0 if immediate else self.timeout, True):
+ err = GetLastError()
+ CancelIoEx(self.pipe, olap)
+
+ if err == WAIT_TIMEOUT:
+ log('GetOverlappedResultEx timedout')
+ raise SocketTimeout('timed out after waiting %dms for read' %
+ self.timeout)
+
+ log('GetOverlappedResultEx reports error %d', err)
+ self._raise_win_err('error while waiting for read', err)
+
+ nread = nread.value
+ if nread == 0:
+ # Docs say that named pipes return 0 byte when the other end did
+ # a zero byte write. Since we don't ever do that, the only
+ # other way this shows up is if the client has gotten in a weird
+ # state, so let's bail out
+ CancelIoEx(self.pipe, olap)
+ raise IOError('Async read yielded 0 bytes; unpossible!')
+
+ # Holds precisely the bytes that we read from the prior request
+ buf = buf[:nread]
+
+ returned_size = min(nread, size)
+ if returned_size == nread:
+ return buf
+
+ # keep any left-overs around for a later read to consume
+ self._iobuf = buf[returned_size:]
+ return buf[:returned_size]
+
+ def write(self, data):
+ olap = OVERLAPPED()
+ immediate = WriteFile(self.pipe, ctypes.c_char_p(data), len(data),
+ None, olap)
+
+ if not immediate:
+ err = GetLastError()
+ if err != ERROR_IO_PENDING:
+ self._raise_win_err('failed to write %d bytes' % len(data),
+ GetLastError())
+
+ # Obtain results, waiting if needed
+ nwrote = wintypes.DWORD()
+ if GetOverlappedResultEx(self.pipe, olap, nwrote, 0 if immediate else
+ self.timeout, True):
+ return nwrote.value
+
+ err = GetLastError()
+
+ # It's potentially unsafe to allow the write to continue after
+ # we unwind, so let's make a best effort to avoid that happening
+ CancelIoEx(self.pipe, olap)
+
+ if err == WAIT_TIMEOUT:
+ raise SocketTimeout('timed out after waiting %dms for write' %
+ self.timeout)
+ self._raise_win_err('error while waiting for write of %d bytes' %
+ len(data), err)
+
+
+class CLIProcessTransport(Transport):
+ """ open a pipe to the cli to talk to the service
+ This intended to be used only in the test harness!
+
+ The CLI is an oddball because we only support JSON input
+ and cannot send multiple commands through the same instance,
+ so we spawn a new process for each command.
+
+ We disable server spawning for this implementation, again, because
+ it is intended to be used only in our test harness. You really
+ should not need to use the CLI transport for anything real.
+
+ While the CLI can output in BSER, our Transport interface doesn't
+ support telling this instance that it should do so. That effectively
+ limits this implementation to JSON input and output only at this time.
+
+ It is the responsibility of the caller to set the send and
+ receive codecs appropriately.
+ """
+ proc = None
+ closed = True
+
+ def __init__(self, sockpath, timeout):
+ self.sockpath = sockpath
+ self.timeout = timeout
+
+ def close(self):
+ if self.proc:
+ self.proc.kill()
+ self.proc = None
+
+ def _connect(self):
+ if self.proc:
+ return self.proc
+ args = [
+ 'watchman',
+ '--sockname={}'.format(self.sockpath),
+ '--logfile=/BOGUS',
+ '--statefile=/BOGUS',
+ '--no-spawn',
+ '--no-local',
+ '--no-pretty',
+ '-j',
+ ]
+ self.proc = subprocess.Popen(args,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE)
+ return self.proc
+
+ def readBytes(self, size):
+ self._connect()
+ res = self.proc.stdout.read(size)
+ if res == '':
+ raise WatchmanError('EOF on CLI process transport')
+ return res
+
+ def write(self, data):
+ if self.closed:
+ self.closed = False
+ self.proc = None
+ self._connect()
+ res = self.proc.stdin.write(data)
+ self.proc.stdin.close()
+ self.closed = True
+ return res
+
+
+class BserCodec(Codec):
+ """ use the BSER encoding. This is the default, preferred codec """
+
+ def _loads(self, response):
+ return bser.loads(response)
+
+ def receive(self):
+ buf = [self.transport.readBytes(sniff_len)]
+ if not buf[0]:
+ raise WatchmanError('empty watchman response')
+
+ elen = bser.pdu_len(buf[0])
+
+ rlen = len(buf[0])
+ while elen > rlen:
+ buf.append(self.transport.readBytes(elen - rlen))
+ rlen += len(buf[-1])
+
+ response = ''.join(buf)
+ try:
+ res = self._loads(response)
+ return res
+ except ValueError as e:
+ raise WatchmanError('watchman response decode error: %s' % e)
+
+ def send(self, *args):
+ cmd = bser.dumps(*args)
+ self.transport.write(cmd)
+
+
+class ImmutableBserCodec(BserCodec):
+ """ use the BSER encoding, decoding values using the newer
+ immutable object support """
+
+ def _loads(self, response):
+ return bser.loads(response, False)
+
+
+class JsonCodec(Codec):
+ """ Use json codec. This is here primarily for testing purposes """
+ json = None
+
+ def __init__(self, transport):
+ super(JsonCodec, self).__init__(transport)
+ # optional dep on json, only if JsonCodec is used
+ import json
+ self.json = json
+
+ def receive(self):
+ line = self.transport.readLine()
+ try:
+ return self.json.loads(line)
+ except Exception as e:
+ print(e, line)
+ raise
+
+ def send(self, *args):
+ cmd = self.json.dumps(*args)
+ self.transport.write(cmd + "\n")
+
+
+class client(object):
+ """ Handles the communication with the watchman service """
+ sockpath = None
+ transport = None
+ sendCodec = None
+ recvCodec = None
+ sendConn = None
+ recvConn = None
+ subs = {} # Keyed by subscription name
+ sub_by_root = {} # Keyed by root, then by subscription name
+ logs = [] # When log level is raised
+ unilateral = ['log', 'subscription']
+ tport = None
+ useImmutableBser = None
+
+ def __init__(self,
+ sockpath=None,
+ timeout=1.0,
+ transport=None,
+ sendEncoding=None,
+ recvEncoding=None,
+ useImmutableBser=False):
+ self.sockpath = sockpath
+ self.timeout = timeout
+ self.useImmutableBser = useImmutableBser
+
+ transport = transport or os.getenv('WATCHMAN_TRANSPORT') or 'local'
+ if transport == 'local' and os.name == 'nt':
+ self.transport = WindowsNamedPipeTransport
+ elif transport == 'local':
+ self.transport = UnixSocketTransport
+ elif transport == 'cli':
+ self.transport = CLIProcessTransport
+ if sendEncoding is None:
+ sendEncoding = 'json'
+ if recvEncoding is None:
+ recvEncoding = sendEncoding
+ else:
+ raise WatchmanError('invalid transport %s' % transport)
+
+ sendEncoding = sendEncoding or os.getenv('WATCHMAN_ENCODING') or 'bser'
+ recvEncoding = recvEncoding or os.getenv('WATCHMAN_ENCODING') or 'bser'
+
+ self.recvCodec = self._parseEncoding(recvEncoding)
+ self.sendCodec = self._parseEncoding(sendEncoding)
+
+ def _parseEncoding(self, enc):
+ if enc == 'bser':
+ if self.useImmutableBser:
+ return ImmutableBserCodec
+ return BserCodec
+ elif enc == 'json':
+ return JsonCodec
+ else:
+ raise WatchmanError('invalid encoding %s' % enc)
+
+ def _hasprop(self, result, name):
+ if self.useImmutableBser:
+ return hasattr(result, name)
+ return name in result
+
+ def _resolvesockname(self):
+ # if invoked via a trigger, watchman will set this env var; we
+ # should use it unless explicitly set otherwise
+ path = os.getenv('WATCHMAN_SOCK')
+ if path:
+ return path
+
+ cmd = ['watchman', '--output-encoding=bser', 'get-sockname']
+ try:
+ p = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ close_fds=os.name != 'nt')
+ except OSError as e:
+ raise WatchmanError('"watchman" executable not in PATH (%s)', e)
+
+ stdout, stderr = p.communicate()
+ exitcode = p.poll()
+
+ if exitcode:
+ raise WatchmanError("watchman exited with code %d" % exitcode)
+
+ result = bser.loads(stdout)
+ if 'error' in result:
+ raise WatchmanError('get-sockname error: %s' % result['error'])
+
+ return result['sockname']
+
+ def _connect(self):
+ """ establish transport connection """
+
+ if self.recvConn:
+ return
+
+ if self.sockpath is None:
+ self.sockpath = self._resolvesockname()
+
+ self.tport = self.transport(self.sockpath, self.timeout)
+ self.sendConn = self.sendCodec(self.tport)
+ self.recvConn = self.recvCodec(self.tport)
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ if self.tport:
+ self.tport.close()
+ self.tport = None
+ self.recvConn = None
+ self.sendConn = None
+
+ def receive(self):
+ """ receive the next PDU from the watchman service
+
+ If the client has activated subscriptions or logs then
+ this PDU may be a unilateral PDU sent by the service to
+ inform the client of a log event or subscription change.
+
+ It may also simply be the response portion of a request
+ initiated by query.
+
+ There are clients in production that subscribe and call
+ this in a loop to retrieve all subscription responses,
+ so care should be taken when making changes here.
+ """
+
+ self._connect()
+ result = self.recvConn.receive()
+ if self._hasprop(result, 'error'):
+ raise CommandError(result['error'])
+
+ if self._hasprop(result, 'log'):
+ self.logs.append(result['log'])
+
+ if self._hasprop(result, 'subscription'):
+ sub = result['subscription']
+ if not (sub in self.subs):
+ self.subs[sub] = []
+ self.subs[sub].append(result)
+
+ # also accumulate in {root,sub} keyed store
+ root = os.path.normcase(result['root'])
+ if not root in self.sub_by_root:
+ self.sub_by_root[root] = {}
+ if not sub in self.sub_by_root[root]:
+ self.sub_by_root[root][sub] = []
+ self.sub_by_root[root][sub].append(result)
+
+ return result
+
+ def isUnilateralResponse(self, res):
+ for k in self.unilateral:
+ if k in res:
+ return True
+ return False
+
+ def getLog(self, remove=True):
+ """ Retrieve buffered log data
+
+ If remove is true the data will be removed from the buffer.
+ Otherwise it will be left in the buffer
+ """
+ res = self.logs
+ if remove:
+ self.logs = []
+ return res
+
+ def getSubscription(self, name, remove=True, root=None):
+ """ Retrieve the data associated with a named subscription
+
+ If remove is True (the default), the subscription data is removed
+ from the buffer. Otherwise the data is returned but left in
+ the buffer.
+
+ Returns None if there is no data associated with `name`
+
+ If root is not None, then only return the subscription
+ data that matches both root and name. When used in this way,
+ remove processing impacts both the unscoped and scoped stores
+ for the subscription data.
+ """
+
+ if root is not None:
+ if not root in self.sub_by_root:
+ return None
+ if not name in self.sub_by_root[root]:
+ return None
+ sub = self.sub_by_root[root][name]
+ if remove:
+ del self.sub_by_root[root][name]
+ # don't let this grow unbounded
+ if name in self.subs:
+ del self.subs[name]
+ return sub
+
+ if not (name in self.subs):
+ return None
+ sub = self.subs[name]
+ if remove:
+ del self.subs[name]
+ return sub
+
+ def query(self, *args):
+ """ Send a query to the watchman service and return the response
+
+ This call will block until the response is returned.
+ If any unilateral responses are sent by the service in between
+ the request-response they will be buffered up in the client object
+ and NOT returned via this method.
+ """
+
+ log('calling client.query')
+ self._connect()
+ try:
+ self.sendConn.send(args)
+
+ res = self.receive()
+ while self.isUnilateralResponse(res):
+ res = self.receive()
+
+ return res
+ except CommandError as ex:
+ ex.setCommand(args)
+ raise ex
+
+ def capabilityCheck(self, optional=None, required=None):
+ """ Perform a server capability check """
+ res = self.query('version', {
+ 'optional': optional or [],
+ 'required': required or []
+ })
+
+ if not self._hasprop(res, 'capabilities'):
+ # Server doesn't support capabilities, so we need to
+ # synthesize the results based on the version
+ capabilities.synthesize(res, optional)
+ if 'error' in res:
+ raise CommandError(res['error'])
+
+ return res
+
+ def setTimeout(self, value):
+ self.recvConn.setTimeout(value)
+ self.sendConn.setTimeout(value)
+
+# no-check-code -- this is a 3rd party library
new file mode 100644
@@ -0,0 +1,950 @@
+/*
+Copyright (c) 2013-2015, Facebook, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+ endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include <Python.h>
+#ifdef _MSC_VER
+#define inline __inline
+#include <stdint.h>
+#endif
+
+/* Return the smallest size int that can store the value */
+#define INT_SIZE(x) (((x) == ((int8_t)x)) ? 1 : \
+ ((x) == ((int16_t)x)) ? 2 : \
+ ((x) == ((int32_t)x)) ? 4 : 8)
+
+#define BSER_ARRAY 0x00
+#define BSER_OBJECT 0x01
+#define BSER_STRING 0x02
+#define BSER_INT8 0x03
+#define BSER_INT16 0x04
+#define BSER_INT32 0x05
+#define BSER_INT64 0x06
+#define BSER_REAL 0x07
+#define BSER_TRUE 0x08
+#define BSER_FALSE 0x09
+#define BSER_NULL 0x0a
+#define BSER_TEMPLATE 0x0b
+#define BSER_SKIP 0x0c
+
+// An immutable object representation of BSER_OBJECT.
+// Rather than build a hash table, key -> value are obtained
+// by walking the list of keys to determine the offset into
+// the values array. The assumption is that the number of
+// array elements will be typically small (~6 for the top
+// level query result and typically 3-5 for the file entries)
+// so that the time overhead for this is small compared to
+// using a proper hash table. Even with this simplistic
+// approach, this is still faster for the mercurial use case
+// as it helps to eliminate creating N other objects to
+// represent the stat information in the hgwatchman extension
+typedef struct {
+ PyObject_HEAD
+ PyObject *keys; // tuple of field names
+ PyObject *values; // tuple of values
+} bserObject;
+
+static Py_ssize_t bserobj_tuple_length(PyObject *o) {
+ bserObject *obj = (bserObject*)o;
+
+ return PySequence_Length(obj->keys);
+}
+
+static PyObject *bserobj_tuple_item(PyObject *o, Py_ssize_t i) {
+ bserObject *obj = (bserObject*)o;
+
+ return PySequence_GetItem(obj->values, i);
+}
+
+static PySequenceMethods bserobj_sq = {
+ bserobj_tuple_length, /* sq_length */
+ 0, /* sq_concat */
+ 0, /* sq_repeat */
+ bserobj_tuple_item, /* sq_item */
+ 0, /* sq_ass_item */
+ 0, /* sq_contains */
+ 0, /* sq_inplace_concat */
+ 0 /* sq_inplace_repeat */
+};
+
+static void bserobj_dealloc(PyObject *o) {
+ bserObject *obj = (bserObject*)o;
+
+ Py_CLEAR(obj->keys);
+ Py_CLEAR(obj->values);
+ PyObject_Del(o);
+}
+
+static PyObject *bserobj_getattrro(PyObject *o, PyObject *name) {
+ bserObject *obj = (bserObject*)o;
+ Py_ssize_t i, n;
+ const char *namestr;
+
+ if (PyIndex_Check(name)) {
+ i = PyNumber_AsSsize_t(name, PyExc_IndexError);
+ if (i == -1 && PyErr_Occurred()) {
+ return NULL;
+ }
+ return PySequence_GetItem(obj->values, i);
+ }
+
+ // hack^Wfeature to allow mercurial to use "st_size" to reference "size"
+ namestr = PyString_AsString(name);
+ if (!strncmp(namestr, "st_", 3)) {
+ namestr += 3;
+ }
+
+ n = PyTuple_GET_SIZE(obj->keys);
+ for (i = 0; i < n; i++) {
+ const char *item_name = NULL;
+ PyObject *key = PyTuple_GET_ITEM(obj->keys, i);
+
+ item_name = PyString_AsString(key);
+ if (!strcmp(item_name, namestr)) {
+ return PySequence_GetItem(obj->values, i);
+ }
+ }
+ PyErr_Format(PyExc_AttributeError,
+ "bserobject has no attribute '%.400s'", namestr);
+ return NULL;
+}
+
+static PyMappingMethods bserobj_map = {
+ bserobj_tuple_length, /* mp_length */
+ bserobj_getattrro, /* mp_subscript */
+ 0 /* mp_ass_subscript */
+};
+
+PyTypeObject bserObjectType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ "bserobj_tuple", /* tp_name */
+ sizeof(bserObject), /* tp_basicsize */
+ 0, /* tp_itemsize */
+ bserobj_dealloc, /* tp_dealloc */
+ 0, /* tp_print */
+ 0, /* tp_getattr */
+ 0, /* tp_setattr */
+ 0, /* tp_compare */
+ 0, /* tp_repr */
+ 0, /* tp_as_number */
+ &bserobj_sq, /* tp_as_sequence */
+ &bserobj_map, /* tp_as_mapping */
+ 0, /* tp_hash */
+ 0, /* tp_call */
+ 0, /* tp_str */
+ bserobj_getattrro, /* tp_getattro */
+ 0, /* tp_setattro */
+ 0, /* tp_as_buffer */
+ Py_TPFLAGS_DEFAULT, /* tp_flags */
+ "bserobj tuple", /* tp_doc */
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
+ 0, /* tp_methods */
+ 0, /* tp_members */
+ 0, /* tp_getset */
+ 0, /* tp_base */
+ 0, /* tp_dict */
+ 0, /* tp_descr_get */
+ 0, /* tp_descr_set */
+ 0, /* tp_dictoffset */
+ 0, /* tp_init */
+ 0, /* tp_alloc */
+ 0, /* tp_new */
+};
+
+
+static PyObject *bser_loads_recursive(const char **ptr, const char *end,
+ int mutable);
+
+static const char bser_true = BSER_TRUE;
+static const char bser_false = BSER_FALSE;
+static const char bser_null = BSER_NULL;
+static const char bser_string_hdr = BSER_STRING;
+static const char bser_array_hdr = BSER_ARRAY;
+static const char bser_object_hdr = BSER_OBJECT;
+
+static inline uint32_t next_power_2(uint32_t n)
+{
+ n |= (n >> 16);
+ n |= (n >> 8);
+ n |= (n >> 4);
+ n |= (n >> 2);
+ n |= (n >> 1);
+ return n + 1;
+}
+
+// A buffer we use for building up the serialized result
+struct bser_buffer {
+ char *buf;
+ int wpos, allocd;
+};
+typedef struct bser_buffer bser_t;
+
+static int bser_append(bser_t *bser, const char *data, uint32_t len)
+{
+ int newlen = next_power_2(bser->wpos + len);
+ if (newlen > bser->allocd) {
+ char *nbuf = realloc(bser->buf, newlen);
+ if (!nbuf) {
+ return 0;
+ }
+
+ bser->buf = nbuf;
+ bser->allocd = newlen;
+ }
+
+ memcpy(bser->buf + bser->wpos, data, len);
+ bser->wpos += len;
+ return 1;
+}
+
+static int bser_init(bser_t *bser)
+{
+ bser->allocd = 8192;
+ bser->wpos = 0;
+ bser->buf = malloc(bser->allocd);
+
+ if (!bser->buf) {
+ return 0;
+ }
+
+ // Leave room for the serialization header, which includes
+ // our overall length. To make things simpler, we'll use an
+ // int32 for the header
+#define EMPTY_HEADER "\x00\x01\x05\x00\x00\x00\x00"
+ bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER)-1);
+
+ return 1;
+}
+
+static void bser_dtor(bser_t *bser)
+{
+ free(bser->buf);
+ bser->buf = NULL;
+}
+
+static int bser_long(bser_t *bser, int64_t val)
+{
+ int8_t i8;
+ int16_t i16;
+ int32_t i32;
+ int64_t i64;
+ char sz;
+ int size = INT_SIZE(val);
+ char *iptr;
+
+ switch (size) {
+ case 1:
+ sz = BSER_INT8;
+ i8 = (int8_t)val;
+ iptr = (char*)&i8;
+ break;
+ case 2:
+ sz = BSER_INT16;
+ i16 = (int16_t)val;
+ iptr = (char*)&i16;
+ break;
+ case 4:
+ sz = BSER_INT32;
+ i32 = (int32_t)val;
+ iptr = (char*)&i32;
+ break;
+ case 8:
+ sz = BSER_INT64;
+ i64 = (int64_t)val;
+ iptr = (char*)&i64;
+ break;
+ default:
+ PyErr_SetString(PyExc_RuntimeError,
+ "Cannot represent this long value!?");
+ return 0;
+ }
+
+ if (!bser_append(bser, &sz, sizeof(sz))) {
+ return 0;
+ }
+
+ return bser_append(bser, iptr, size);
+}
+
+static int bser_string(bser_t *bser, PyObject *sval)
+{
+ char *buf = NULL;
+ Py_ssize_t len;
+ int res;
+ PyObject *utf = NULL;
+
+ if (PyUnicode_Check(sval)) {
+ utf = PyUnicode_AsEncodedString(sval, "utf-8", "ignore");
+ sval = utf;
+ }
+
+ res = PyString_AsStringAndSize(sval, &buf, &len);
+ if (res == -1) {
+ res = 0;
+ goto out;
+ }
+
+ if (!bser_append(bser, &bser_string_hdr, sizeof(bser_string_hdr))) {
+ res = 0;
+ goto out;
+ }
+
+ if (!bser_long(bser, len)) {
+ res = 0;
+ goto out;
+ }
+
+ if (len > UINT32_MAX) {
+ PyErr_Format(PyExc_ValueError, "string too big");
+ res = 0;
+ goto out;
+ }
+
+ res = bser_append(bser, buf, (uint32_t)len);
+
+out:
+ if (utf) {
+ Py_DECREF(utf);
+ }
+
+ return res;
+}
+
+static int bser_recursive(bser_t *bser, PyObject *val)
+{
+ if (PyBool_Check(val)) {
+ if (val == Py_True) {
+ return bser_append(bser, &bser_true, sizeof(bser_true));
+ }
+ return bser_append(bser, &bser_false, sizeof(bser_false));
+ }
+
+ if (val == Py_None) {
+ return bser_append(bser, &bser_null, sizeof(bser_null));
+ }
+
+ if (PyInt_Check(val)) {
+ return bser_long(bser, PyInt_AS_LONG(val));
+ }
+
+ if (PyLong_Check(val)) {
+ return bser_long(bser, PyLong_AsLongLong(val));
+ }
+
+ if (PyString_Check(val) || PyUnicode_Check(val)) {
+ return bser_string(bser, val);
+ }
+
+
+ if (PyFloat_Check(val)) {
+ double dval = PyFloat_AS_DOUBLE(val);
+ char sz = BSER_REAL;
+
+ if (!bser_append(bser, &sz, sizeof(sz))) {
+ return 0;
+ }
+
+ return bser_append(bser, (char*)&dval, sizeof(dval));
+ }
+
+ if (PyList_Check(val)) {
+ Py_ssize_t i, len = PyList_GET_SIZE(val);
+
+ if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) {
+ return 0;
+ }
+
+ if (!bser_long(bser, len)) {
+ return 0;
+ }
+
+ for (i = 0; i < len; i++) {
+ PyObject *ele = PyList_GET_ITEM(val, i);
+
+ if (!bser_recursive(bser, ele)) {
+ return 0;
+ }
+ }
+
+ return 1;
+ }
+
+ if (PyTuple_Check(val)) {
+ Py_ssize_t i, len = PyTuple_GET_SIZE(val);
+
+ if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) {
+ return 0;
+ }
+
+ if (!bser_long(bser, len)) {
+ return 0;
+ }
+
+ for (i = 0; i < len; i++) {
+ PyObject *ele = PyTuple_GET_ITEM(val, i);
+
+ if (!bser_recursive(bser, ele)) {
+ return 0;
+ }
+ }
+
+ return 1;
+ }
+
+ if (PyMapping_Check(val)) {
+ Py_ssize_t len = PyMapping_Length(val);
+ Py_ssize_t pos = 0;
+ PyObject *key, *ele;
+
+ if (!bser_append(bser, &bser_object_hdr, sizeof(bser_object_hdr))) {
+ return 0;
+ }
+
+ if (!bser_long(bser, len)) {
+ return 0;
+ }
+
+ while (PyDict_Next(val, &pos, &key, &ele)) {
+ if (!bser_string(bser, key)) {
+ return 0;
+ }
+ if (!bser_recursive(bser, ele)) {
+ return 0;
+ }
+ }
+
+ return 1;
+ }
+
+ PyErr_SetString(PyExc_ValueError, "Unsupported value type");
+ return 0;
+}
+
+static PyObject *bser_dumps(PyObject *self, PyObject *args)
+{
+ PyObject *val = NULL, *res;
+ bser_t bser;
+ uint32_t len;
+
+ if (!PyArg_ParseTuple(args, "O", &val)) {
+ return NULL;
+ }
+
+ if (!bser_init(&bser)) {
+ return PyErr_NoMemory();
+ }
+
+ if (!bser_recursive(&bser, val)) {
+ bser_dtor(&bser);
+ if (errno == ENOMEM) {
+ return PyErr_NoMemory();
+ }
+ // otherwise, we've already set the error to something reasonable
+ return NULL;
+ }
+
+ // Now fill in the overall length
+ len = bser.wpos - (sizeof(EMPTY_HEADER) - 1);
+ memcpy(bser.buf + 3, &len, sizeof(len));
+
+ res = PyString_FromStringAndSize(bser.buf, bser.wpos);
+ bser_dtor(&bser);
+
+ return res;
+}
+
+int bunser_int(const char **ptr, const char *end, int64_t *val)
+{
+ int needed;
+ const char *buf = *ptr;
+ int8_t i8;
+ int16_t i16;
+ int32_t i32;
+ int64_t i64;
+
+ switch (buf[0]) {
+ case BSER_INT8:
+ needed = 2;
+ break;
+ case BSER_INT16:
+ needed = 3;
+ break;
+ case BSER_INT32:
+ needed = 5;
+ break;
+ case BSER_INT64:
+ needed = 9;
+ break;
+ default:
+ PyErr_Format(PyExc_ValueError,
+ "invalid bser int encoding 0x%02x", buf[0]);
+ return 0;
+ }
+ if (end - buf < needed) {
+ PyErr_SetString(PyExc_ValueError, "input buffer to small for int encoding");
+ return 0;
+ }
+ *ptr = buf + needed;
+ switch (buf[0]) {
+ case BSER_INT8:
+ memcpy(&i8, buf + 1, sizeof(i8));
+ *val = i8;
+ return 1;
+ case BSER_INT16:
+ memcpy(&i16, buf + 1, sizeof(i16));
+ *val = i16;
+ return 1;
+ case BSER_INT32:
+ memcpy(&i32, buf + 1, sizeof(i32));
+ *val = i32;
+ return 1;
+ case BSER_INT64:
+ memcpy(&i64, buf + 1, sizeof(i64));
+ *val = i64;
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+static int bunser_string(const char **ptr, const char *end,
+ const char **start, int64_t *len)
+{
+ const char *buf = *ptr;
+
+ // skip string marker
+ buf++;
+ if (!bunser_int(&buf, end, len)) {
+ return 0;
+ }
+
+ if (buf + *len > end) {
+ PyErr_Format(PyExc_ValueError, "invalid string length in bser data");
+ return 0;
+ }
+
+ *ptr = buf + *len;
+ *start = buf;
+ return 1;
+}
+
+static PyObject *bunser_array(const char **ptr, const char *end, int mutable)
+{
+ const char *buf = *ptr;
+ int64_t nitems, i;
+ PyObject *res;
+
+ // skip array header
+ buf++;
+ if (!bunser_int(&buf, end, &nitems)) {
+ return 0;
+ }
+ *ptr = buf;
+
+ if (nitems > LONG_MAX) {
+ PyErr_Format(PyExc_ValueError, "too many items for python array");
+ return NULL;
+ }
+
+ if (mutable) {
+ res = PyList_New((Py_ssize_t)nitems);
+ } else {
+ res = PyTuple_New((Py_ssize_t)nitems);
+ }
+
+ for (i = 0; i < nitems; i++) {
+ PyObject *ele = bser_loads_recursive(ptr, end, mutable);
+
+ if (!ele) {
+ Py_DECREF(res);
+ return NULL;
+ }
+
+ if (mutable) {
+ PyList_SET_ITEM(res, i, ele);
+ } else {
+ PyTuple_SET_ITEM(res, i, ele);
+ }
+ // DECREF(ele) not required as SET_ITEM steals the ref
+ }
+
+ return res;
+}
+
+static PyObject *bunser_object(const char **ptr, const char *end,
+ int mutable)
+{
+ const char *buf = *ptr;
+ int64_t nitems, i;
+ PyObject *res;
+ bserObject *obj;
+
+ // skip array header
+ buf++;
+ if (!bunser_int(&buf, end, &nitems)) {
+ return 0;
+ }
+ *ptr = buf;
+
+ if (mutable) {
+ res = PyDict_New();
+ } else {
+ obj = PyObject_New(bserObject, &bserObjectType);
+ obj->keys = PyTuple_New((Py_ssize_t)nitems);
+ obj->values = PyTuple_New((Py_ssize_t)nitems);
+ res = (PyObject*)obj;
+ }
+
+ for (i = 0; i < nitems; i++) {
+ const char *keystr;
+ int64_t keylen;
+ PyObject *key;
+ PyObject *ele;
+
+ if (!bunser_string(ptr, end, &keystr, &keylen)) {
+ Py_DECREF(res);
+ return NULL;
+ }
+
+ if (keylen > LONG_MAX) {
+ PyErr_Format(PyExc_ValueError, "string too big for python");
+ Py_DECREF(res);
+ return NULL;
+ }
+
+ key = PyString_FromStringAndSize(keystr, (Py_ssize_t)keylen);
+ if (!key) {
+ Py_DECREF(res);
+ return NULL;
+ }
+
+ ele = bser_loads_recursive(ptr, end, mutable);
+
+ if (!ele) {
+ Py_DECREF(key);
+ Py_DECREF(res);
+ return NULL;
+ }
+
+ if (mutable) {
+ PyDict_SetItem(res, key, ele);
+ Py_DECREF(key);
+ Py_DECREF(ele);
+ } else {
+ /* PyTuple_SET_ITEM steals ele, key */
+ PyTuple_SET_ITEM(obj->values, i, ele);
+ PyTuple_SET_ITEM(obj->keys, i, key);
+ }
+ }
+
+ return res;
+}
+
+static PyObject *bunser_template(const char **ptr, const char *end,
+ int mutable)
+{
+ const char *buf = *ptr;
+ int64_t nitems, i;
+ PyObject *arrval;
+ PyObject *keys;
+ Py_ssize_t numkeys, keyidx;
+
+ if (buf[1] != BSER_ARRAY) {
+ PyErr_Format(PyExc_ValueError, "Expect ARRAY to follow TEMPLATE");
+ return NULL;
+ }
+
+ // skip header
+ buf++;
+ *ptr = buf;
+
+ // Load template keys
+ keys = bunser_array(ptr, end, mutable);
+ if (!keys) {
+ return NULL;
+ }
+
+ numkeys = PySequence_Length(keys);
+
+ // Load number of array elements
+ if (!bunser_int(ptr, end, &nitems)) {
+ Py_DECREF(keys);
+ return 0;
+ }
+
+ if (nitems > LONG_MAX) {
+ PyErr_Format(PyExc_ValueError, "Too many items for python");
+ Py_DECREF(keys);
+ return NULL;
+ }
+
+ arrval = PyList_New((Py_ssize_t)nitems);
+ if (!arrval) {
+ Py_DECREF(keys);
+ return NULL;
+ }
+
+ for (i = 0; i < nitems; i++) {
+ PyObject *dict = NULL;
+ bserObject *obj = NULL;
+
+ if (mutable) {
+ dict = PyDict_New();
+ } else {
+ obj = PyObject_New(bserObject, &bserObjectType);
+ if (obj) {
+ obj->keys = keys;
+ Py_INCREF(obj->keys);
+ obj->values = PyTuple_New(numkeys);
+ }
+ dict = (PyObject*)obj;
+ }
+ if (!dict) {
+fail:
+ Py_DECREF(keys);
+ Py_DECREF(arrval);
+ return NULL;
+ }
+
+ for (keyidx = 0; keyidx < numkeys; keyidx++) {
+ PyObject *key;
+ PyObject *ele;
+
+ if (**ptr == BSER_SKIP) {
+ *ptr = *ptr + 1;
+ ele = Py_None;
+ Py_INCREF(ele);
+ } else {
+ ele = bser_loads_recursive(ptr, end, mutable);
+ }
+
+ if (!ele) {
+ goto fail;
+ }
+
+ if (mutable) {
+ key = PyList_GET_ITEM(keys, keyidx);
+ PyDict_SetItem(dict, key, ele);
+ Py_DECREF(ele);
+ } else {
+ PyTuple_SET_ITEM(obj->values, keyidx, ele);
+ // DECREF(ele) not required as SET_ITEM steals the ref
+ }
+ }
+
+ PyList_SET_ITEM(arrval, i, dict);
+ // DECREF(obj) not required as SET_ITEM steals the ref
+ }
+
+ Py_DECREF(keys);
+
+ return arrval;
+}
+
+static PyObject *bser_loads_recursive(const char **ptr, const char *end,
+ int mutable)
+{
+ const char *buf = *ptr;
+
+ switch (buf[0]) {
+ case BSER_INT8:
+ case BSER_INT16:
+ case BSER_INT32:
+ case BSER_INT64:
+ {
+ int64_t ival;
+ if (!bunser_int(ptr, end, &ival)) {
+ return NULL;
+ }
+ if (ival < LONG_MIN || ival > LONG_MAX) {
+ return PyLong_FromLongLong(ival);
+ }
+ return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t));
+ }
+
+ case BSER_REAL:
+ {
+ double dval;
+ memcpy(&dval, buf + 1, sizeof(dval));
+ *ptr = buf + 1 + sizeof(double);
+ return PyFloat_FromDouble(dval);
+ }
+
+ case BSER_TRUE:
+ *ptr = buf + 1;
+ Py_INCREF(Py_True);
+ return Py_True;
+
+ case BSER_FALSE:
+ *ptr = buf + 1;
+ Py_INCREF(Py_False);
+ return Py_False;
+
+ case BSER_NULL:
+ *ptr = buf + 1;
+ Py_INCREF(Py_None);
+ return Py_None;
+
+ case BSER_STRING:
+ {
+ const char *start;
+ int64_t len;
+
+ if (!bunser_string(ptr, end, &start, &len)) {
+ return NULL;
+ }
+
+ if (len > LONG_MAX) {
+ PyErr_Format(PyExc_ValueError, "string too long for python");
+ return NULL;
+ }
+
+ return PyString_FromStringAndSize(start, (long)len);
+ }
+
+ case BSER_ARRAY:
+ return bunser_array(ptr, end, mutable);
+
+ case BSER_OBJECT:
+ return bunser_object(ptr, end, mutable);
+
+ case BSER_TEMPLATE:
+ return bunser_template(ptr, end, mutable);
+
+ default:
+ PyErr_Format(PyExc_ValueError, "unhandled bser opcode 0x%02x", buf[0]);
+ }
+
+ return NULL;
+}
+
+// Expected use case is to read a packet from the socket and
+// then call bser.pdu_len on the packet. It returns the total
+// length of the entire response that the peer is sending,
+// including the bytes already received. This allows the client
+// to compute the data size it needs to read before it can
+// decode the data
+static PyObject *bser_pdu_len(PyObject *self, PyObject *args)
+{
+ const char *start = NULL;
+ const char *data = NULL;
+ int datalen = 0;
+ const char *end;
+ int64_t expected_len, total_len;
+
+ if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) {
+ return NULL;
+ }
+ data = start;
+ end = data + datalen;
+
+ // Validate the header and length
+ if (memcmp(data, EMPTY_HEADER, 2) != 0) {
+ PyErr_SetString(PyExc_ValueError, "invalid bser header");
+ return NULL;
+ }
+
+ data += 2;
+
+ // Expect an integer telling us how big the rest of the data
+ // should be
+ if (!bunser_int(&data, end, &expected_len)) {
+ return NULL;
+ }
+
+ total_len = expected_len + (data - start);
+ if (total_len > LONG_MAX) {
+ return PyLong_FromLongLong(total_len);
+ }
+ return PyInt_FromLong((long)total_len);
+}
+
+static PyObject *bser_loads(PyObject *self, PyObject *args)
+{
+ const char *data = NULL;
+ int datalen = 0;
+ const char *end;
+ int64_t expected_len;
+ int mutable = 1;
+ PyObject *mutable_obj = NULL;
+
+ if (!PyArg_ParseTuple(args, "s#|O:loads", &data, &datalen, &mutable_obj)) {
+ return NULL;
+ }
+ if (mutable_obj) {
+ mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0;
+ }
+
+ end = data + datalen;
+
+ // Validate the header and length
+ if (memcmp(data, EMPTY_HEADER, 2) != 0) {
+ PyErr_SetString(PyExc_ValueError, "invalid bser header");
+ return NULL;
+ }
+
+ data += 2;
+
+ // Expect an integer telling us how big the rest of the data
+ // should be
+ if (!bunser_int(&data, end, &expected_len)) {
+ return NULL;
+ }
+
+ // Verify
+ if (expected_len + data != end) {
+ PyErr_SetString(PyExc_ValueError, "bser data len != header len");
+ return NULL;
+ }
+
+ return bser_loads_recursive(&data, end, mutable);
+}
+
+static PyMethodDef bser_methods[] = {
+ {"loads", bser_loads, METH_VARARGS, "Deserialize string."},
+ {"pdu_len", bser_pdu_len, METH_VARARGS, "Extract PDU length."},
+ {"dumps", bser_dumps, METH_VARARGS, "Serialize string."},
+ {NULL, NULL, 0, NULL}
+};
+
+PyMODINIT_FUNC initbser(void)
+{
+ (void)Py_InitModule("bser", bser_methods);
+ PyType_Ready(&bserObjectType);
+}
+
+/* vim:ts=2:sw=2:et:
+ */
+
+// no-check-code -- this is a 3rd party library
new file mode 100644
@@ -0,0 +1,69 @@
+# Copyright 2015 Facebook, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# * Neither the name Facebook nor the names of its contributors may be used to
+# endorse or promote products derived from this software without specific
+# prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import re
+
+def parse_version(vstr):
+ res = 0
+ for n in vstr.split('.'):
+ res = res * 1000
+ res = res + int(n)
+ return res
+
+cap_versions = {
+ "cmd-watch-del-all": "3.1.1",
+ "cmd-watch-project": "3.1",
+ "relative_root": "3.3",
+ "term-dirname": "3.1",
+ "term-idirname": "3.1",
+ "wildmatch": "3.7",
+}
+
+def check(version, name):
+ if name in cap_versions:
+ return version >= parse_version(cap_versions[name])
+ return False
+
+def synthesize(vers, opts):
+ """ Synthesize a capability enabled version response
+ This is a very limited emulation for relatively recent feature sets
+ """
+ parsed_version = parse_version(vers['version'])
+ vers['capabilities'] = {}
+ for name in opts['optional']:
+ vers['capabilities'][name] = check(parsed_version, name)
+ failed = False
+ for name in opts['required']:
+ have = check(parsed_version, name)
+ vers['capabilities'][name] = have
+ if not have:
+ vers['error'] = 'client required capability `' + name + \
+ '` is not supported by this server'
+ return vers
+
+# no-check-code -- this is a 3rd party library
new file mode 100644
@@ -0,0 +1,359 @@
+# Copyright 2015 Facebook, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# * Neither the name Facebook nor the names of its contributors may be used to
+# endorse or promote products derived from this software without specific
+# prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import collections
+import ctypes
+import struct
+import sys
+
+BSER_ARRAY = '\x00'
+BSER_OBJECT = '\x01'
+BSER_STRING = '\x02'
+BSER_INT8 = '\x03'
+BSER_INT16 = '\x04'
+BSER_INT32 = '\x05'
+BSER_INT64 = '\x06'
+BSER_REAL = '\x07'
+BSER_TRUE = '\x08'
+BSER_FALSE = '\x09'
+BSER_NULL = '\x0a'
+BSER_TEMPLATE = '\x0b'
+BSER_SKIP = '\x0c'
+
+# Leave room for the serialization header, which includes
+# our overall length. To make things simpler, we'll use an
+# int32 for the header
+EMPTY_HEADER = "\x00\x01\x05\x00\x00\x00\x00"
+
+# Python 3 conditional for supporting Python 2's int/long types
+if sys.version_info > (3,):
+ long = int
+
+def _int_size(x):
+ """Return the smallest size int that can store the value"""
+ if -0x80 <= x <= 0x7F:
+ return 1
+ elif -0x8000 <= x <= 0x7FFF:
+ return 2
+ elif -0x80000000 <= x <= 0x7FFFFFFF:
+ return 4
+ elif long(-0x8000000000000000) <= x <= long(0x7FFFFFFFFFFFFFFF):
+ return 8
+ else:
+ raise RuntimeError('Cannot represent value: ' + str(x))
+
+
+class _bser_buffer(object):
+
+ def __init__(self):
+ self.buf = ctypes.create_string_buffer(8192)
+ struct.pack_into(str(len(EMPTY_HEADER)) + 's', self.buf, 0, EMPTY_HEADER)
+ self.wpos = len(EMPTY_HEADER)
+
+ def ensure_size(self, size):
+ while ctypes.sizeof(self.buf) - self.wpos < size:
+ ctypes.resize(self.buf, ctypes.sizeof(self.buf) * 2)
+
+ def append_long(self, val):
+ size = _int_size(val)
+ to_write = size + 1
+ self.ensure_size(to_write)
+ if size == 1:
+ struct.pack_into('=cb', self.buf, self.wpos, BSER_INT8, val)
+ elif size == 2:
+ struct.pack_into('=ch', self.buf, self.wpos, BSER_INT16, val)
+ elif size == 4:
+ struct.pack_into('=ci', self.buf, self.wpos, BSER_INT32, val)
+ elif size == 8:
+ struct.pack_into('=cq', self.buf, self.wpos, BSER_INT64, val)
+ else:
+ raise RuntimeError('Cannot represent this long value')
+ self.wpos += to_write
+
+
+ def append_string(self, s):
+ if isinstance(s, unicode):
+ s = s.encode('utf-8')
+ s_len = len(s)
+ size = _int_size(s_len)
+ to_write = 2 + size + s_len
+ self.ensure_size(to_write)
+ if size == 1:
+ struct.pack_into('=ccb' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT8, s_len, s)
+ elif size == 2:
+ struct.pack_into('=cch' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT16, s_len, s)
+ elif size == 4:
+ struct.pack_into('=cci' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT32, s_len, s)
+ elif size == 8:
+ struct.pack_into('=ccq' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT64, s_len, s)
+ else:
+ raise RuntimeError('Cannot represent this string value')
+ self.wpos += to_write
+
+
+ def append_recursive(self, val):
+ if isinstance(val, bool):
+ needed = 1
+ self.ensure_size(needed)
+ if val:
+ to_encode = BSER_TRUE
+ else:
+ to_encode = BSER_FALSE
+ struct.pack_into('=c', self.buf, self.wpos, to_encode)
+ self.wpos += needed
+ elif val is None:
+ needed = 1
+ self.ensure_size(needed)
+ struct.pack_into('=c', self.buf, self.wpos, BSER_NULL)
+ self.wpos += needed
+ elif isinstance(val, (int, long)):
+ self.append_long(val)
+ elif isinstance(val, (str, unicode)):
+ self.append_string(val)
+ elif isinstance(val, float):
+ needed = 9
+ self.ensure_size(needed)
+ struct.pack_into('=cd', self.buf, self.wpos, BSER_REAL, val)
+ self.wpos += needed
+ elif isinstance(val, collections.Mapping) and isinstance(val, collections.Sized):
+ val_len = len(val)
+ size = _int_size(val_len)
+ needed = 2 + size
+ self.ensure_size(needed)
+ if size == 1:
+ struct.pack_into('=ccb', self.buf, self.wpos, BSER_OBJECT, BSER_INT8, val_len)
+ elif size == 2:
+ struct.pack_into('=cch', self.buf, self.wpos, BSER_OBJECT, BSER_INT16, val_len)
+ elif size == 4:
+ struct.pack_into('=cci', self.buf, self.wpos, BSER_OBJECT, BSER_INT32, val_len)
+ elif size == 8:
+ struct.pack_into('=ccq', self.buf, self.wpos, BSER_OBJECT, BSER_INT64, val_len)
+ else:
+ raise RuntimeError('Cannot represent this mapping value')
+ self.wpos += needed
+ for k, v in val.iteritems():
+ self.append_string(k)
+ self.append_recursive(v)
+ elif isinstance(val, collections.Iterable) and isinstance(val, collections.Sized):
+ val_len = len(val)
+ size = _int_size(val_len)
+ needed = 2 + size
+ self.ensure_size(needed)
+ if size == 1:
+ struct.pack_into('=ccb', self.buf, self.wpos, BSER_ARRAY, BSER_INT8, val_len)
+ elif size == 2:
+ struct.pack_into('=cch', self.buf, self.wpos, BSER_ARRAY, BSER_INT16, val_len)
+ elif size == 4:
+ struct.pack_into('=cci', self.buf, self.wpos, BSER_ARRAY, BSER_INT32, val_len)
+ elif size == 8:
+ struct.pack_into('=ccq', self.buf, self.wpos, BSER_ARRAY, BSER_INT64, val_len)
+ else:
+ raise RuntimeError('Cannot represent this sequence value')
+ self.wpos += needed
+ for v in val:
+ self.append_recursive(v)
+ else:
+ raise RuntimeError('Cannot represent unknown value type')
+
+
+def dumps(obj):
+ bser_buf = _bser_buffer()
+ bser_buf.append_recursive(obj)
+ # Now fill in the overall length
+ obj_len = bser_buf.wpos - len(EMPTY_HEADER)
+ struct.pack_into('=i', bser_buf.buf, 3, obj_len)
+ return bser_buf.buf.raw[:bser_buf.wpos]
+
+
+def _bunser_int(buf, pos):
+ try:
+ int_type = buf[pos]
+ except IndexError:
+ raise ValueError('Invalid bser int encoding, pos out of range')
+ if int_type == BSER_INT8:
+ needed = 2
+ fmt = '=b'
+ elif int_type == BSER_INT16:
+ needed = 3
+ fmt = '=h'
+ elif int_type == BSER_INT32:
+ needed = 5
+ fmt = '=i'
+ elif int_type == BSER_INT64:
+ needed = 9
+ fmt = '=q'
+ else:
+ raise ValueError('Invalid bser int encoding 0x%02x' % int(int_type))
+ int_val = struct.unpack_from(fmt, buf, pos + 1)[0]
+ return (int_val, pos + needed)
+
+
+def _bunser_string(buf, pos):
+ str_len, pos = _bunser_int(buf, pos + 1)
+ str_val = struct.unpack_from(str(str_len) + 's', buf, pos)[0]
+ return (str_val, pos + str_len)
+
+
+def _bunser_array(buf, pos, mutable=True):
+ arr_len, pos = _bunser_int(buf, pos + 1)
+ arr = []
+ for i in range(arr_len):
+ arr_item, pos = _bser_loads_recursive(buf, pos, mutable)
+ arr.append(arr_item)
+
+ if not mutable:
+ arr = tuple(arr)
+
+ return arr, pos
+
+
+# This is a quack-alike with the bserObjectType in bser.c
+# It provides by getattr accessors and getitem for both index
+# and name.
+class _BunserDict(object):
+ __slots__ = ('_keys', '_values')
+
+ def __init__(self, keys, values):
+ self._keys = keys
+ self._values = values
+
+ def __getattr__(self, name):
+ return self.__getitem__(name)
+
+ def __getitem__(self, key):
+ if isinstance(key, (int, long)):
+ return self._values[key]
+ elif key.startswith('st_'):
+ # hack^Wfeature to allow mercurial to use "st_size" to
+ # reference "size"
+ key = key[3:]
+ try:
+ return self._values[self._keys.index(key)]
+ except ValueError as ex:
+ raise KeyError('_BunserDict has no key %s' % key)
+
+ def __len__(self):
+ return len(self._keys)
+
+def _bunser_object(buf, pos, mutable=True):
+ obj_len, pos = _bunser_int(buf, pos + 1)
+ if mutable:
+ obj = {}
+ else:
+ keys = []
+ vals = []
+
+ for i in range(obj_len):
+ key, pos = _bunser_string(buf, pos)
+ val, pos = _bser_loads_recursive(buf, pos, mutable)
+ if mutable:
+ obj[key] = val
+ else:
+ keys.append(key)
+ vals.append(val)
+
+ if not mutable:
+ obj = _BunserDict(keys, vals)
+
+ return obj, pos
+
+
+def _bunser_template(buf, pos, mutable=True):
+ if buf[pos + 1] != BSER_ARRAY:
+ raise RuntimeError('Expect ARRAY to follow TEMPLATE')
+ keys, pos = _bunser_array(buf, pos + 1)
+ nitems, pos = _bunser_int(buf, pos)
+ arr = []
+ for i in range(nitems):
+ if mutable:
+ obj = {}
+ else:
+ vals = []
+
+ for keyidx in range(len(keys)):
+ if buf[pos] == BSER_SKIP:
+ pos += 1
+ ele = None
+ else:
+ ele, pos = _bser_loads_recursive(buf, pos, mutable)
+
+ if mutable:
+ key = keys[keyidx]
+ obj[key] = ele
+ else:
+ vals.append(ele)
+
+ if not mutable:
+ obj = _BunserDict(keys, vals)
+
+ arr.append(obj)
+ return arr, pos
+
+
+def _bser_loads_recursive(buf, pos, mutable=True):
+ val_type = buf[pos]
+ if (val_type == BSER_INT8 or val_type == BSER_INT16 or
+ val_type == BSER_INT32 or val_type == BSER_INT64):
+ return _bunser_int(buf, pos)
+ elif val_type == BSER_REAL:
+ val = struct.unpack_from('=d', buf, pos + 1)[0]
+ return (val, pos + 9)
+ elif val_type == BSER_TRUE:
+ return (True, pos + 1)
+ elif val_type == BSER_FALSE:
+ return (False, pos + 1)
+ elif val_type == BSER_NULL:
+ return (None, pos + 1)
+ elif val_type == BSER_STRING:
+ return _bunser_string(buf, pos)
+ elif val_type == BSER_ARRAY:
+ return _bunser_array(buf, pos, mutable)
+ elif val_type == BSER_OBJECT:
+ return _bunser_object(buf, pos, mutable)
+ elif val_type == BSER_TEMPLATE:
+ return _bunser_template(buf, pos, mutable)
+ else:
+ raise RuntimeError('unhandled bser opcode 0x%02x' % (val_type,))
+
+
+def pdu_len(buf):
+ if buf[0:2] != EMPTY_HEADER[0:2]:
+ raise RuntimeError('Invalid BSER header')
+ expected_len, pos = _bunser_int(buf, 2)
+ return expected_len + pos
+
+
+def loads(buf, mutable=True):
+ if buf[0:2] != EMPTY_HEADER[0:2]:
+ raise RuntimeError('Invalid BSER header')
+ expected_len, pos = _bunser_int(buf, 2)
+ if len(buf) != expected_len + pos:
+ raise RuntimeError('bser data len != header len')
+ return _bser_loads_recursive(buf, pos, mutable)[0]
+
+# no-check-code -- this is a 3rd party library
@@ -564,6 +564,8 @@
Extension('mercurial.osutil', ['mercurial/osutil.c'],
extra_link_args=osutil_ldflags,
depends=common_depends),
+ Extension('hgext.fsmonitor.pywatchman.bser',
+ ['hgext/fsmonitor/pywatchman/bser.c']),
]
try: