| Submitter | Kostia Balytskyi |
|---|---|
| Date | Dec. 6, 2016, 2:01 p.m. |
| Message ID | <008bf4590cd2bed2329a.1481032909@dev1902.lla1.facebook.com> |
| Download | mbox | patch |
| Permalink | /patch/17828/ |
| State | Superseded |
| Headers | show |
Comments
Please disregard v4, see v5. On 12/6/16 2:01 PM, Kostia Balytskyi wrote: > # HG changeset patch > # User Kostia Balytskyi <ikostia@fb.com> > # Date 1481032446 28800 > # Tue Dec 06 05:54:06 2016 -0800 > # Node ID 008bf4590cd2bed2329ad5eb8f2ad3e2bd121f2d > # Parent 243ecbd4f5c9f452275d4435866359cf84dc03ff > scmutil: add a simple key-value file helper > > The purpose of the added class is to serve purposes like save files of shelve > or state files of shelve, rebase and histedit. Keys of these files can be > alphanumeric and start with letters, while values must not contain newlines. > Keys which start with an uppercase letter are required, while other keys > are optional. > > In light of Mercurial's reluctancy to use Python's json module, this tries > to provide a reasonable alternative for a non-nested named data. > Comparing to current approach of storing state in plain text files, where > semantic meaning of lines of text is only determined by their oreder, > simple key-value file allows for reordering lines and thus helps handle > optional values. > > Initial use-case I see for this is obs-shelve's shelve files. Later we > can possibly migrate state files to this approach. > > The test is in a new file beause I did not figure out where to put it > within existing test suite. If you give me a better idea, I will gladly > follow it. > > diff --git a/mercurial/error.py b/mercurial/error.py > --- a/mercurial/error.py > +++ b/mercurial/error.py > @@ -243,3 +243,19 @@ class UnsupportedBundleSpecification(Exc > > class CorruptedState(Exception): > """error raised when a command is not able to read its state from file""" > + > +class InvalidKeyValueFile(Exception): > + """error raised when the file can't be parsed as simple key-value file""" > + > +class InvalidKeyInFileException(Exception): > + """error raised when invalid key is attempted to be written > + > + This is used in simple key-value file implementation""" > + > +class InvalidValueInFileException(Exception): > + """error raisesd when invalid value is attempted to be written > + > + This is used in simple key-value file implementation""" > + > +class MissingRequiredKeyInFileException(Exception): > + """error raised when simple key-value file misses a required key""" > diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py > --- a/mercurial/scmutil.py > +++ b/mercurial/scmutil.py > @@ -1571,3 +1571,123 @@ class checkambigatclosing(closewrapbase) > def close(self): > self._origfh.close() > self._checkambig() > + > +class simplekeyvaluefile(object): > + """A simple file with key=value lines > + > + Keys must be alphanumerics and start with a letter, values might not > + contain '\n' characters > + > + >>> contents = {} > + >>> class fileobj(object): > + ... def __init__(self, name): > + ... self.name = name > + ... def __enter__(self): > + ... return self > + ... def __exit__(self, *args, **kwargs): > + ... pass > + ... def write(self, text): > + ... contents[self.name] = text > + ... def read(self): > + ... return contents[self.name] > + >>> class mockvfs(object): > + ... def read(self, path): > + ... return fileobj(path).read() > + ... def readlines(self, path): > + ... return fileobj(path).read().split('\\n') > + ... def __call__(self, path, mode, atomictemp): > + ... return fileobj(path) > + >>> vfs = mockvfs() > + > + Basic testing of whether simple key-value file works: > + >>> d = {'key1': 'value1', 'Key2': 'value2'} > + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) > + >>> print sorted(vfs.read('kvfile').split('\\n')) > + ['', 'Key2=value2', 'key1=value1'] > + > + Testing of whether invalid keys are detected: > + >>> d = {'0key1': 'value1', 'Key2': 'value2'} > + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) > + Traceback (most recent call last): > + ... > + InvalidKeyInFileException: keys must start with a letter ... > + >>> d = {'key1@': 'value1', 'Key2': 'value2'} > + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) > + Traceback (most recent call last): > + ... > + InvalidKeyInFileException: invalid key name in a simple key-value file > + > + Testing of whether invalid values are detected: > + >>> d = {'key1': 'value1', 'Key2': 'value2\\n'} > + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) > + Traceback (most recent call last): > + ... > + InvalidValueInFileException: invalid value in a simple key-value file > + > + Test cases when necessary keys are present > + >>> d = {'key1': 'value1', 'Key2': 'value2'} > + >>> simplekeyvaluefile(vfs, 'allkeyshere').write(d) > + >>> class kvf(simplekeyvaluefile): > + ... KEYS = [('key3', False), ('Key2', True)] > + >>> print sorted(kvf(vfs, 'allkeyshere').read().items()) > + [('Key2', 'value'), ('key1', 'value')] > + > + Test cases when necessary keys are absent > + >>> d = {'key1': 'value1', 'Key3': 'value2'} > + >>> simplekeyvaluefile(vfs, 'missingkeys').write(d) > + >>> class kvf(simplekeyvaluefile): > + ... KEYS = [('key3', False), ('Key2', True)] > + >>> print sorted(kvf(vfs, 'missingkeys').read().items()) > + Traceback (most recent call last): > + ... > + MissingRequiredKeyInFileException: missing a required key: 'Key2' > + > + Test cases when file is not really a simple key-value file > + >>> contents['badfile'] = 'ababagalamaga\\n' > + >>> simplekeyvaluefile(vfs, 'badfile').read() > + Traceback (most recent call last): > + ... > + InvalidKeyValueFile: dictionary ... element #0 has length 1; 2 is required > + """ > + > + # if KEYS is non-empty, read values are validated against it: > + # each key is a tuple (keyname, required) > + KEYS = [] > + > + def __init__(self, vfs, path): > + self.vfs = vfs > + self.path = path > + > + def validate(self, d): > + for key, req in self.KEYS: > + if req and key not in d: > + e = "missing a required key: '%s'" % key > + raise error.MissingRequiredKeyInFileException(e) > + > + def read(self): > + lines = self.vfs.readlines(self.path) > + try: > + d = dict(line[:-1].split('=', 1) for line in lines if line) > + except ValueError as e: > + raise error.InvalidKeyValueFile(str(e)) > + self.validate(d) > + return d > + > + def write(self, data): > + """Write key=>value mapping to a file > + data is a dict. Keys should be alphanumerical and start with a letter. > + Values should not contain newline characters.""" > + lines = [] > + for k, v in data.items(): > + if not k[0].isalpha(): > + e = "keys must start with a letter in a key-value file" > + raise error.InvalidKeyInFileException(e) > + if not k.isalnum(): > + e = "invalid key name in a simple key-value file" > + raise error.InvalidKeyInFileException(e) > + if '\n' in v: > + e = "invalid value in a simple key-value file" > + raise error.InvalidValueInFileException(e) > + lines.append("%s=%s\n" % (k, v)) > + with self.vfs(self.path, mode='wb', atomictemp=True) as fp: > + fp.write(''.join(lines)) > diff --git a/tests/test-doctest.py b/tests/test-doctest.py > --- a/tests/test-doctest.py > +++ b/tests/test-doctest.py > @@ -28,6 +28,7 @@ testmod('mercurial.patch') > testmod('mercurial.pathutil') > testmod('mercurial.parser') > testmod('mercurial.revset') > +testmod('mercurial.scmutil', optionflags=doctest.ELLIPSIS) > testmod('mercurial.store') > testmod('mercurial.subrepo') > testmod('mercurial.templatefilters') > _______________________________________________ > Mercurial-devel mailing list > Mercurial-devel@mercurial-scm.org > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Patch
diff --git a/mercurial/error.py b/mercurial/error.py --- a/mercurial/error.py +++ b/mercurial/error.py @@ -243,3 +243,19 @@ class UnsupportedBundleSpecification(Exc class CorruptedState(Exception): """error raised when a command is not able to read its state from file""" + +class InvalidKeyValueFile(Exception): + """error raised when the file can't be parsed as simple key-value file""" + +class InvalidKeyInFileException(Exception): + """error raised when invalid key is attempted to be written + + This is used in simple key-value file implementation""" + +class InvalidValueInFileException(Exception): + """error raisesd when invalid value is attempted to be written + + This is used in simple key-value file implementation""" + +class MissingRequiredKeyInFileException(Exception): + """error raised when simple key-value file misses a required key""" diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py --- a/mercurial/scmutil.py +++ b/mercurial/scmutil.py @@ -1571,3 +1571,123 @@ class checkambigatclosing(closewrapbase) def close(self): self._origfh.close() self._checkambig() + +class simplekeyvaluefile(object): + """A simple file with key=value lines + + Keys must be alphanumerics and start with a letter, values might not + contain '\n' characters + + >>> contents = {} + >>> class fileobj(object): + ... def __init__(self, name): + ... self.name = name + ... def __enter__(self): + ... return self + ... def __exit__(self, *args, **kwargs): + ... pass + ... def write(self, text): + ... contents[self.name] = text + ... def read(self): + ... return contents[self.name] + >>> class mockvfs(object): + ... def read(self, path): + ... return fileobj(path).read() + ... def readlines(self, path): + ... return fileobj(path).read().split('\\n') + ... def __call__(self, path, mode, atomictemp): + ... return fileobj(path) + >>> vfs = mockvfs() + + Basic testing of whether simple key-value file works: + >>> d = {'key1': 'value1', 'Key2': 'value2'} + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) + >>> print sorted(vfs.read('kvfile').split('\\n')) + ['', 'Key2=value2', 'key1=value1'] + + Testing of whether invalid keys are detected: + >>> d = {'0key1': 'value1', 'Key2': 'value2'} + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) + Traceback (most recent call last): + ... + InvalidKeyInFileException: keys must start with a letter ... + >>> d = {'key1@': 'value1', 'Key2': 'value2'} + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) + Traceback (most recent call last): + ... + InvalidKeyInFileException: invalid key name in a simple key-value file + + Testing of whether invalid values are detected: + >>> d = {'key1': 'value1', 'Key2': 'value2\\n'} + >>> simplekeyvaluefile(vfs, 'kvfile').write(d) + Traceback (most recent call last): + ... + InvalidValueInFileException: invalid value in a simple key-value file + + Test cases when necessary keys are present + >>> d = {'key1': 'value1', 'Key2': 'value2'} + >>> simplekeyvaluefile(vfs, 'allkeyshere').write(d) + >>> class kvf(simplekeyvaluefile): + ... KEYS = [('key3', False), ('Key2', True)] + >>> print sorted(kvf(vfs, 'allkeyshere').read().items()) + [('Key2', 'value'), ('key1', 'value')] + + Test cases when necessary keys are absent + >>> d = {'key1': 'value1', 'Key3': 'value2'} + >>> simplekeyvaluefile(vfs, 'missingkeys').write(d) + >>> class kvf(simplekeyvaluefile): + ... KEYS = [('key3', False), ('Key2', True)] + >>> print sorted(kvf(vfs, 'missingkeys').read().items()) + Traceback (most recent call last): + ... + MissingRequiredKeyInFileException: missing a required key: 'Key2' + + Test cases when file is not really a simple key-value file + >>> contents['badfile'] = 'ababagalamaga\\n' + >>> simplekeyvaluefile(vfs, 'badfile').read() + Traceback (most recent call last): + ... + InvalidKeyValueFile: dictionary ... element #0 has length 1; 2 is required + """ + + # if KEYS is non-empty, read values are validated against it: + # each key is a tuple (keyname, required) + KEYS = [] + + def __init__(self, vfs, path): + self.vfs = vfs + self.path = path + + def validate(self, d): + for key, req in self.KEYS: + if req and key not in d: + e = "missing a required key: '%s'" % key + raise error.MissingRequiredKeyInFileException(e) + + def read(self): + lines = self.vfs.readlines(self.path) + try: + d = dict(line[:-1].split('=', 1) for line in lines if line) + except ValueError as e: + raise error.InvalidKeyValueFile(str(e)) + self.validate(d) + return d + + def write(self, data): + """Write key=>value mapping to a file + data is a dict. Keys should be alphanumerical and start with a letter. + Values should not contain newline characters.""" + lines = [] + for k, v in data.items(): + if not k[0].isalpha(): + e = "keys must start with a letter in a key-value file" + raise error.InvalidKeyInFileException(e) + if not k.isalnum(): + e = "invalid key name in a simple key-value file" + raise error.InvalidKeyInFileException(e) + if '\n' in v: + e = "invalid value in a simple key-value file" + raise error.InvalidValueInFileException(e) + lines.append("%s=%s\n" % (k, v)) + with self.vfs(self.path, mode='wb', atomictemp=True) as fp: + fp.write(''.join(lines)) diff --git a/tests/test-doctest.py b/tests/test-doctest.py --- a/tests/test-doctest.py +++ b/tests/test-doctest.py @@ -28,6 +28,7 @@ testmod('mercurial.patch') testmod('mercurial.pathutil') testmod('mercurial.parser') testmod('mercurial.revset') +testmod('mercurial.scmutil', optionflags=doctest.ELLIPSIS) testmod('mercurial.store') testmod('mercurial.subrepo') testmod('mercurial.templatefilters')