From patchwork Sat Oct 5 15:47:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: D6973: mangler: stop rewriting string constants to be bytes literals From: phabricator X-Patchwork-Id: 41983 Message-Id: To: Phabricator Cc: mercurial-devel@mercurial-scm.org Date: Sat, 5 Oct 2019 15:47:17 +0000 durin42 updated this revision to Diff 16833. REPOSITORY rHG Mercurial CHANGES SINCE LAST UPDATE https://phab.mercurial-scm.org/D6973?vs=16820&id=16833 CHANGES SINCE LAST ACTION https://phab.mercurial-scm.org/D6973/new/ REVISION DETAIL https://phab.mercurial-scm.org/D6973 AFFECTED FILES mercurial/__init__.py CHANGE DETAILS To: durin42, #hg-reviewers, indygreg Cc: indygreg, mercurial-devel diff --git a/mercurial/__init__.py b/mercurial/__init__.py --- a/mercurial/__init__.py +++ b/mercurial/__init__.py @@ -153,37 +153,6 @@ tokens[j] = st._replace(string=b'u%s' % st.string) for i, t in enumerate(tokens): - # Convert most string literals to byte literals. String literals - # in Python 2 are bytes. String literals in Python 3 are unicode. - # Most strings in Mercurial are bytes and unicode strings are rare. - # Rather than rewrite all string literals to use ``b''`` to indicate - # byte strings, we apply this token transformer to insert the ``b`` - # prefix nearly everywhere. - if t.type == token.STRING: - s = t.string - - # Preserve docstrings as string literals. This is inconsistent - # with regular unprefixed strings. However, the - # "from __future__" parsing (which allows a module docstring to - # exist before it) doesn't properly handle the docstring if it - # is b''' prefixed, leading to a SyntaxError. We leave all - # docstrings as unprefixed to avoid this. This means Mercurial - # components touching docstrings need to handle unicode, - # unfortunately. - if s[0:3] in (b"'''", b'"""'): - yield t - continue - - # If the first character isn't a quote, it is likely a string - # prefixing character (such as 'b', 'u', or 'r'. Ignore. - if s[0] not in (b"'", b'"'): - yield t - continue - - # String literal. Prefix to make a b'' string. - yield t._replace(string=b'b%s' % t.string) - continue - # Insert compatibility imports at "from __future__ import" line. # No '\n' should be added to preserve line numbers. if (