Patchwork [4,of,4] py3: use bytestr wrapper in revsetlang.tokenize()

login
register
mail settings
Submitter Yuya Nishihara
Date March 16, 2017, 3:31 p.m.
Message ID <d21fd508342b1d43ebef.1489678273@mimosa>
Download mbox | patch
Permalink /patch/19395/
State Accepted
Headers show

Comments

Yuya Nishihara - March 16, 2017, 3:31 p.m.
# HG changeset patch
# User Yuya Nishihara <yuya@tcha.org>
# Date 1489667781 -32400
#      Thu Mar 16 21:36:21 2017 +0900
# Node ID d21fd508342b1d43ebefaac65f1d6e2e5ca60704
# Parent  649866574c86bae35ab8143c50af72bee7fef810
py3: use bytestr wrapper in revsetlang.tokenize()

This backs out 77270ec0cdd9 and wraps program by bytestr() instead.
Augie Fackler - March 16, 2017, 4:59 p.m.
On Fri, Mar 17, 2017 at 12:31:13AM +0900, Yuya Nishihara wrote:
> # HG changeset patch
> # User Yuya Nishihara <yuya@tcha.org>
> # Date 1489667781 -32400
> #      Thu Mar 16 21:36:21 2017 +0900
> # Node ID d21fd508342b1d43ebefaac65f1d6e2e5ca60704
> # Parent  649866574c86bae35ab8143c50af72bee7fef810
> py3: use bytestr wrapper in revsetlang.tokenize()

Sure, queued, thanks. Much cleaner IMO.

>
> This backs out 77270ec0cdd9 and wraps program by bytestr() instead.
>
> diff --git a/mercurial/revsetlang.py b/mercurial/revsetlang.py
> --- a/mercurial/revsetlang.py
> +++ b/mercurial/revsetlang.py
> @@ -78,6 +78,7 @@ def tokenize(program, lookup=None, symin
>      [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)]
>
>      '''
> +    program = pycompat.bytestr(program)
>      if syminitletters is None:
>          syminitletters = _syminitletters
>      if symletters is None:
> @@ -100,7 +101,7 @@ def tokenize(program, lookup=None, symin
>
>      pos, l = 0, len(program)
>      while pos < l:
> -        c = program[pos:pos + 1]
> +        c = program[pos]
>          if c.isspace(): # skip inter-token whitespace
>              pass
>          elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully
> @@ -118,14 +119,14 @@ def tokenize(program, lookup=None, symin
>                program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings
>              if c == 'r':
>                  pos += 1
> -                c = program[pos:pos + 1]
> +                c = program[pos]
>                  decode = lambda x: x
>              else:
>                  decode = parser.unescapestr
>              pos += 1
>              s = pos
>              while pos < l: # find closing quote
> -                d = program[pos:pos + 1]
> +                d = program[pos]
>                  if d == '\\': # skip over escaped characters
>                      pos += 2
>                      continue
> @@ -140,11 +141,10 @@ def tokenize(program, lookup=None, symin
>              s = pos
>              pos += 1
>              while pos < l: # find end of symbol
> -                d = program[pos:pos + 1]
> +                d = program[pos]
>                  if d not in symletters:
>                      break
> -                if (d == '.'
> -                    and program[pos - 1:pos] == '.'): # special case for ..
> +                if d == '.' and program[pos - 1] == '.': # special case for ..
>                      pos -= 1
>                      break
>                  pos += 1
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Patch

diff --git a/mercurial/revsetlang.py b/mercurial/revsetlang.py
--- a/mercurial/revsetlang.py
+++ b/mercurial/revsetlang.py
@@ -78,6 +78,7 @@  def tokenize(program, lookup=None, symin
     [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)]
 
     '''
+    program = pycompat.bytestr(program)
     if syminitletters is None:
         syminitletters = _syminitletters
     if symletters is None:
@@ -100,7 +101,7 @@  def tokenize(program, lookup=None, symin
 
     pos, l = 0, len(program)
     while pos < l:
-        c = program[pos:pos + 1]
+        c = program[pos]
         if c.isspace(): # skip inter-token whitespace
             pass
         elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully
@@ -118,14 +119,14 @@  def tokenize(program, lookup=None, symin
               program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings
             if c == 'r':
                 pos += 1
-                c = program[pos:pos + 1]
+                c = program[pos]
                 decode = lambda x: x
             else:
                 decode = parser.unescapestr
             pos += 1
             s = pos
             while pos < l: # find closing quote
-                d = program[pos:pos + 1]
+                d = program[pos]
                 if d == '\\': # skip over escaped characters
                     pos += 2
                     continue
@@ -140,11 +141,10 @@  def tokenize(program, lookup=None, symin
             s = pos
             pos += 1
             while pos < l: # find end of symbol
-                d = program[pos:pos + 1]
+                d = program[pos]
                 if d not in symletters:
                     break
-                if (d == '.'
-                    and program[pos - 1:pos] == '.'): # special case for ..
+                if d == '.' and program[pos - 1] == '.': # special case for ..
                     pos -= 1
                     break
                 pos += 1