Patchwork py3: have a bytes version of shlex.split()

login
register
mail settings
Submitter Pulkit Goyal
Date Dec. 25, 2016, 12:16 a.m.
Message ID <4b6b316bc3d87df1ce7d.1482624976@pulkit-goyal>
Download mbox | patch
Permalink /patch/18037/
State Accepted
Headers show

Comments

Pulkit Goyal - Dec. 25, 2016, 12:16 a.m.
# HG changeset patch
# User Pulkit Goyal <7895pulkit@gmail.com>
# Date 1482615415 -19800
#      Sun Dec 25 03:06:55 2016 +0530
# Node ID 4b6b316bc3d87df1ce7deffbda1cb813b955d658
# Parent  81bf1a686b999c3c2d743135f8ee316546ac77c9
py3: have a bytes version of shlex.split()

shlex.split() only accepts unicodes on Python 3. After this patch we will be
using pycompat.shlexsplit(). This patch also replaces existing occurences of
shlex.split with pycompat.shlexsplit.
Pulkit Goyal - Jan. 5, 2017, 10:15 p.m.
I don't know who did it, but this one is already
pushed(https://www.mercurial-scm.org/repo/hg-committed/rev/caf7e1c5efe4).
I think this will prevent any further confusion.

On Sun, Dec 25, 2016 at 5:46 AM, Pulkit Goyal <7895pulkit@gmail.com> wrote:
> # HG changeset patch
> # User Pulkit Goyal <7895pulkit@gmail.com>
> # Date 1482615415 -19800
> #      Sun Dec 25 03:06:55 2016 +0530
> # Node ID 4b6b316bc3d87df1ce7deffbda1cb813b955d658
> # Parent  81bf1a686b999c3c2d743135f8ee316546ac77c9
> py3: have a bytes version of shlex.split()
>
> shlex.split() only accepts unicodes on Python 3. After this patch we will be
> using pycompat.shlexsplit(). This patch also replaces existing occurences of
> shlex.split with pycompat.shlexsplit.
>
> diff -r 81bf1a686b99 -r 4b6b316bc3d8 hgext/extdiff.py
> --- a/hgext/extdiff.py  Sun Dec 25 02:42:46 2016 +0530
> +++ b/hgext/extdiff.py  Sun Dec 25 03:06:55 2016 +0530
> @@ -64,7 +64,6 @@
>
>  import os
>  import re
> -import shlex
>  import shutil
>  import tempfile
>  from mercurial.i18n import _
> @@ -78,6 +77,7 @@
>      commands,
>      error,
>      filemerge,
> +    pycompat,
>      scmutil,
>      util,
>  )
> @@ -371,7 +371,7 @@
>              if path:
>                  # case "cmd = path opts"
>                  cmdline = path
> -                diffopts = len(shlex.split(cmdline)) > 1
> +                diffopts = len(pycompat.shlexsplit(cmdline)) > 1
>              else:
>                  # case "cmd ="
>                  path = util.findexe(cmd)
> diff -r 81bf1a686b99 -r 4b6b316bc3d8 mercurial/commands.py
> --- a/mercurial/commands.py     Sun Dec 25 02:42:46 2016 +0530
> +++ b/mercurial/commands.py     Sun Dec 25 03:06:55 2016 +0530
> @@ -11,7 +11,6 @@
>  import errno
>  import os
>  import re
> -import shlex
>  import socket
>  import string
>  import sys
> @@ -1981,7 +1980,7 @@
>      editor = ui.geteditor()
>      editor = util.expandpath(editor)
>      fm.write('editor', _("checking commit editor... (%s)\n"), editor)
> -    cmdpath = util.findexe(shlex.split(editor)[0])
> +    cmdpath = util.findexe(pycompat.shlexsplit(editor)[0])
>      fm.condwrite(not cmdpath and editor == 'vi', 'vinotfound',
>                   _(" No commit editor set and can't find %s in PATH\n"
>                     " (specify a commit editor in your configuration"
> diff -r 81bf1a686b99 -r 4b6b316bc3d8 mercurial/dispatch.py
> --- a/mercurial/dispatch.py     Sun Dec 25 02:42:46 2016 +0530
> +++ b/mercurial/dispatch.py     Sun Dec 25 03:06:55 2016 +0530
> @@ -14,7 +14,6 @@
>  import os
>  import pdb
>  import re
> -import shlex
>  import signal
>  import sys
>  import time
> @@ -279,7 +278,7 @@
>          cmd = re.sub(r'\$(\d+|\$)', replacer, cmd)
>          givenargs = [x for i, x in enumerate(givenargs)
>                       if i not in nums]
> -        args = shlex.split(cmd)
> +        args = pycompat.shlexsplit(cmd)
>      return args + givenargs
>
>  def aliasinterpolate(name, args, cmd):
> @@ -351,7 +350,7 @@
>              return
>
>          try:
> -            args = shlex.split(self.definition)
> +            args = pycompat.shlexsplit(self.definition)
>          except ValueError as inst:
>              self.badalias = (_("error in definition for alias '%s': %s")
>                               % (self.name, inst))
> @@ -461,7 +460,7 @@
>          args = aliasargs(entry[0], args)
>          defaults = ui.config("defaults", cmd)
>          if defaults:
> -            args = map(util.expandpath, shlex.split(defaults)) + args
> +            args = map(util.expandpath, pycompat.shlexsplit(defaults)) + args
>          c = list(entry[1])
>      else:
>          cmd = None
> diff -r 81bf1a686b99 -r 4b6b316bc3d8 mercurial/pycompat.py
> --- a/mercurial/pycompat.py     Sun Dec 25 02:42:46 2016 +0530
> +++ b/mercurial/pycompat.py     Sun Dec 25 03:06:55 2016 +0530
> @@ -12,6 +12,7 @@
>
>  import getopt
>  import os
> +import shlex
>  import sys
>
>  ispy3 = (sys.version_info[0] >= 3)
> @@ -122,6 +123,14 @@
>          dic = dict((k.encode('latin-1'), v) for k, v in dic.iteritems())
>          return dic
>
> +    # shlex.split() accepts unicodes on Python 3. This function takes bytes
> +    # argument, convert it into unicodes, pass into shlex.split(), convert the
> +    # returned value to bytes and return that.
> +    # TODO: handle shlex.shlex().
> +    def shlexsplit(s):
> +        ret = shlex.split(s.decode('latin-1'))
> +        return [a.encode('latin-1') for a in ret]
> +
>  else:
>      def sysstr(s):
>          return s
> @@ -162,6 +171,7 @@
>      getcwd = os.getcwd
>      osgetenv = os.getenv
>      sysexecutable = sys.executable
> +    shlexsplit = shlex.split
>
>  stringio = io.StringIO
>  empty = _queue.Empty
Sean Farley - Jan. 5, 2017, 10:42 p.m.
Pulkit Goyal <7895pulkit@gmail.com> writes:

> I don't know who did it, but this one is already
> pushed(https://www.mercurial-scm.org/repo/hg-committed/rev/caf7e1c5efe4).
> I think this will prevent any further confusion.

Hmm, good question. I've gone ahead and marked it as accepted in
patchwork.

Patch

diff -r 81bf1a686b99 -r 4b6b316bc3d8 hgext/extdiff.py
--- a/hgext/extdiff.py	Sun Dec 25 02:42:46 2016 +0530
+++ b/hgext/extdiff.py	Sun Dec 25 03:06:55 2016 +0530
@@ -64,7 +64,6 @@ 
 
 import os
 import re
-import shlex
 import shutil
 import tempfile
 from mercurial.i18n import _
@@ -78,6 +77,7 @@ 
     commands,
     error,
     filemerge,
+    pycompat,
     scmutil,
     util,
 )
@@ -371,7 +371,7 @@ 
             if path:
                 # case "cmd = path opts"
                 cmdline = path
-                diffopts = len(shlex.split(cmdline)) > 1
+                diffopts = len(pycompat.shlexsplit(cmdline)) > 1
             else:
                 # case "cmd ="
                 path = util.findexe(cmd)
diff -r 81bf1a686b99 -r 4b6b316bc3d8 mercurial/commands.py
--- a/mercurial/commands.py	Sun Dec 25 02:42:46 2016 +0530
+++ b/mercurial/commands.py	Sun Dec 25 03:06:55 2016 +0530
@@ -11,7 +11,6 @@ 
 import errno
 import os
 import re
-import shlex
 import socket
 import string
 import sys
@@ -1981,7 +1980,7 @@ 
     editor = ui.geteditor()
     editor = util.expandpath(editor)
     fm.write('editor', _("checking commit editor... (%s)\n"), editor)
-    cmdpath = util.findexe(shlex.split(editor)[0])
+    cmdpath = util.findexe(pycompat.shlexsplit(editor)[0])
     fm.condwrite(not cmdpath and editor == 'vi', 'vinotfound',
                  _(" No commit editor set and can't find %s in PATH\n"
                    " (specify a commit editor in your configuration"
diff -r 81bf1a686b99 -r 4b6b316bc3d8 mercurial/dispatch.py
--- a/mercurial/dispatch.py	Sun Dec 25 02:42:46 2016 +0530
+++ b/mercurial/dispatch.py	Sun Dec 25 03:06:55 2016 +0530
@@ -14,7 +14,6 @@ 
 import os
 import pdb
 import re
-import shlex
 import signal
 import sys
 import time
@@ -279,7 +278,7 @@ 
         cmd = re.sub(r'\$(\d+|\$)', replacer, cmd)
         givenargs = [x for i, x in enumerate(givenargs)
                      if i not in nums]
-        args = shlex.split(cmd)
+        args = pycompat.shlexsplit(cmd)
     return args + givenargs
 
 def aliasinterpolate(name, args, cmd):
@@ -351,7 +350,7 @@ 
             return
 
         try:
-            args = shlex.split(self.definition)
+            args = pycompat.shlexsplit(self.definition)
         except ValueError as inst:
             self.badalias = (_("error in definition for alias '%s': %s")
                              % (self.name, inst))
@@ -461,7 +460,7 @@ 
         args = aliasargs(entry[0], args)
         defaults = ui.config("defaults", cmd)
         if defaults:
-            args = map(util.expandpath, shlex.split(defaults)) + args
+            args = map(util.expandpath, pycompat.shlexsplit(defaults)) + args
         c = list(entry[1])
     else:
         cmd = None
diff -r 81bf1a686b99 -r 4b6b316bc3d8 mercurial/pycompat.py
--- a/mercurial/pycompat.py	Sun Dec 25 02:42:46 2016 +0530
+++ b/mercurial/pycompat.py	Sun Dec 25 03:06:55 2016 +0530
@@ -12,6 +12,7 @@ 
 
 import getopt
 import os
+import shlex
 import sys
 
 ispy3 = (sys.version_info[0] >= 3)
@@ -122,6 +123,14 @@ 
         dic = dict((k.encode('latin-1'), v) for k, v in dic.iteritems())
         return dic
 
+    # shlex.split() accepts unicodes on Python 3. This function takes bytes
+    # argument, convert it into unicodes, pass into shlex.split(), convert the
+    # returned value to bytes and return that.
+    # TODO: handle shlex.shlex().
+    def shlexsplit(s):
+        ret = shlex.split(s.decode('latin-1'))
+        return [a.encode('latin-1') for a in ret]
+
 else:
     def sysstr(s):
         return s
@@ -162,6 +171,7 @@ 
     getcwd = os.getcwd
     osgetenv = os.getenv
     sysexecutable = sys.executable
+    shlexsplit = shlex.split
 
 stringio = io.StringIO
 empty = _queue.Empty