From patchwork Fri Oct 7 12:05:42 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: [2,of,2] py3: handle multiple arguments in .encode() and .decode() From: Pulkit Goyal <7895pulkit@gmail.com> X-Patchwork-Id: 16873 Message-Id: <82a0fcbb0066b96a51c6.1475841942@pulkit-goyal> To: mercurial-devel@mercurial-scm.org Date: Fri, 07 Oct 2016 14:05:42 +0200 # HG changeset patch # User Pulkit Goyal <7895pulkit@gmail.com> # Date 1475841889 -7200 # Fri Oct 07 14:04:49 2016 +0200 # Node ID 82a0fcbb0066b96a51c6b16eec9c0384e677dee3 # Parent 480cfc1623c2ad732300d942bbfab41343f7b8c5 py3: handle multiple arguments in .encode() and .decode() There is a case and more can be present where these functions have multiple arguments. Our transformer used to handle the first argument, so added a loop to handle more arguments if present. diff -r 480cfc1623c2 -r 82a0fcbb0066 mercurial/__init__.py --- a/mercurial/__init__.py Fri Oct 07 12:13:28 2016 +0200 +++ b/mercurial/__init__.py Fri Oct 07 14:04:49 2016 +0200 @@ -278,21 +278,30 @@ # .encode() and .decode() on str/bytes/unicode don't accept # byte strings on Python 3. Rewrite the token to include the # unicode literal prefix so the string transformer above doesn't - # add the byte prefix. + # add the byte prefix. The loop helps in handling multiple + # arguments. if (fn in ('encode', 'decode') and prevtoken.type == token.OP and prevtoken.string == '.'): # (OP, '.') # (NAME, 'encode') # (OP, '(') + # [(VARIABLE, encoding)] + # [(OP, '.')] + # [(VARIABLE, encoding)] + # [(OP, ',')] # (STRING, 'utf-8') # (OP, ')') + j = i try: - st = tokens[i + 2] - if (st.type == token.STRING and - st.string[0] in ("'", '"')): - rt = tokenize.TokenInfo(st.type, 'u%s' % st.string, + while (tokens[j + 1].string in ('(', ',', '.')): + st = tokens[j + 2] + if (st.type == token.STRING and + st.string[0] in ("'", '"')): + rt = tokenize.TokenInfo(st.type, + 'u%s' % st.string, st.start, st.end, st.line) - tokens[i + 2] = rt + tokens[j + 2] = rt + j = j + 2 except IndexError: pass @@ -303,7 +312,7 @@ # ``replacetoken`` or any mechanism that changes semantics of module # loading is changed. Otherwise cached bytecode may get loaded without # the new transformation mechanisms applied. - BYTECODEHEADER = b'HG\x00\x02' + BYTECODEHEADER = b'HG\x00\x03' class hgloader(importlib.machinery.SourceFileLoader): """Custom module loader that transforms source code. diff -r 480cfc1623c2 -r 82a0fcbb0066 tests/test-check-py3-compat.t --- a/tests/test-check-py3-compat.t Fri Oct 07 12:13:28 2016 +0200 +++ b/tests/test-check-py3-compat.t Fri Oct 07 14:04:49 2016 +0200 @@ -120,53 +120,52 @@ mercurial/httpconnection.py: error importing: Can't mix strings and bytes in path components (error at i18n.py:*) mercurial/httppeer.py: error importing: Can't mix strings and bytes in path components (error at i18n.py:*) mercurial/i18n.py: error importing module: bytes expected, not str (line *) - mercurial/keepalive.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/localrepo.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/lock.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/mail.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/manifest.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/match.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/mdiff.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/merge.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/minirst.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/namespaces.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/obsolete.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/patch.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/pathutil.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/peer.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/profiling.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/pushkey.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/pvec.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/registrar.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/repair.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/repoview.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/revlog.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/revset.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/scmutil.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/scmwindows.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/similar.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/simplemerge.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/sshpeer.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/sshserver.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/sslutil.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/statichttprepo.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/store.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/streamclone.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/subrepo.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/tagmerge.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/tags.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/templatefilters.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/templatekw.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/templater.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/transaction.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/ui.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/unionrepo.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/url.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/util.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) - mercurial/verify.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) + mercurial/keepalive.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/localrepo.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/lock.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/mail.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/manifest.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/match.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/mdiff.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/merge.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/minirst.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/namespaces.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/obsolete.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/patch.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/pathutil.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/peer.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/profiling.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/pushkey.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/pvec.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/registrar.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/repair.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/repoview.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/revlog.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/revset.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/scmutil.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/scmwindows.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/similar.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/simplemerge.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/sshpeer.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/sshserver.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/sslutil.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/statichttprepo.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/store.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/streamclone.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/subrepo.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/tagmerge.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/tags.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/templatefilters.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/templatekw.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/templater.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/transaction.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/ui.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/unionrepo.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/url.py: error importing: int() can't convert non-string with explicit base (error at util.py:*) + mercurial/verify.py: error importing module: unorderable types: str() >= tuple() (line *) mercurial/win32.py: error importing module: No module named 'msvcrt' (line *) mercurial/windows.py: error importing module: No module named 'msvcrt' (line *) - mercurial/wireproto.py: error importing: encode() argument 2 must be str, not bytes (error at i18n.py:*) + mercurial/wireproto.py: error importing module: unorderable types: str() >= tuple() (line *) #endif