Patchwork [6,of,6] run-tests: fix Python 3 incompatibilities

login
register
mail settings
Submitter Gregory Szorc
Date Feb. 28, 2016, 5:35 a.m.
Message ID <030fd4e3aa88b04749dc.1456637757@ubuntu-vm-main>
Download mbox | patch
Permalink /patch/13445/
State Accepted
Delegated to: Yuya Nishihara
Headers show

Comments

Gregory Szorc - Feb. 28, 2016, 5:35 a.m.
# HG changeset patch
# User Gregory Szorc <gregory.szorc@gmail.com>
# Date 1456637382 28800
#      Sat Feb 27 21:29:42 2016 -0800
# Node ID 030fd4e3aa88b04749dcd0f7b3d05f6e604c27a4
# Parent  c8a7c41a7d218dabc55c0f7e235d316112d81f32
run-tests: fix Python 3 incompatibilities

At one point run-tests.py and test-run-tests.t worked and passed
under Python 3.5. Various changes to run-tests.py over the past
several months appear to have broken Python 3.5 compatibility.

This patch implements various fixes (all related to str/bytes type
coercion) to make run-tests.py and test-run-tests.t mostly work
again. There are still a few failures in test-run-tests.t due to
issues importing mercurial.* modules. But at least run-tests.py
seems to work under 3.5 again.

Patch

diff --git a/tests/run-tests.py b/tests/run-tests.py
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -693,17 +693,18 @@  class Test(unittest.TestCase):
     def tearDown(self):
         """Tasks to perform after run()."""
         for entry in self._daemonpids:
             killdaemons(entry)
         self._daemonpids = []
 
         if self._keeptmpdir:
             log('\nKeeping testtmp dir: %s\nKeeping threadtmp dir: %s' %
-                (self._testtmp, self._threadtmp))
+                (self._testtmp.decode('utf-8'),
+                 self._threadtmp.decode('utf-8')))
         else:
             shutil.rmtree(self._testtmp, True)
             shutil.rmtree(self._threadtmp, True)
 
         if (self._ret != 0 or self._out != self._refout) and not self._skipped \
             and not self._debug and self._out:
             f = open(self.errpath, 'wb')
             for line in self._out:
@@ -716,17 +717,17 @@  class Test(unittest.TestCase):
         # This should be implemented in child classes to run tests.
         raise SkipTest('unknown test type')
 
     def abort(self):
         """Terminate execution of this test."""
         self._aborted = True
 
     def _portmap(self, i):
-        offset = '' if i == 0 else '%s' % i
+        offset = b'' if i == 0 else b'%d' % i
         return (br':%d\b' % (self._startport + i), b':$HGPORT%s' % offset)
 
     def _getreplacements(self):
         """Obtain a mapping of text replacements to apply to test output.
 
         Test output needs to be normalized so it can be compared to expected
         output. This function defines how some of that normalization will
         occur.
@@ -904,18 +905,18 @@  checkcodeglobpats = [
 
 bchr = chr
 if PYTHON3:
     bchr = lambda x: bytes([x])
 
 class TTest(Test):
     """A "t test" is a test backed by a .t file."""
 
-    SKIPPED_PREFIX = 'skipped: '
-    FAILED_PREFIX = 'hghave check failed: '
+    SKIPPED_PREFIX = b'skipped: '
+    FAILED_PREFIX = b'hghave check failed: '
     NEEDESCAPE = re.compile(br'[\x00-\x08\x0b-\x1f\x7f-\xff]').search
 
     ESCAPESUB = re.compile(br'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub
     ESCAPEMAP = dict((bchr(i), br'\x%02x' % i) for i in range(256))
     ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'})
 
     @property
     def refpath(self):
@@ -1208,17 +1209,17 @@  class TTest(Test):
     @staticmethod
     def linematch(el, l):
         retry = False
         if el == l: # perfect match (fast)
             return True
         if el:
             if el.endswith(b" (?)\n"):
                 retry = "retry"
-                el = el[:-5] + "\n"
+                el = el[:-5] + b"\n"
             if el.endswith(b" (esc)\n"):
                 if PYTHON3:
                     el = el[:-7].decode('unicode_escape') + '\n'
                     el = el.encode('utf-8')
                 else:
                     el = el[:-7].decode('string-escape') + '\n'
             if el == l or os.name == 'nt' and el[:-1] + b'\r\n' == l:
                 return True
@@ -1240,20 +1241,20 @@  class TTest(Test):
         Return tuple of lists (missing, failed):
           * the missing/unknown features
           * the features for which existence check failed'''
         missing = []
         failed = []
         for line in lines:
             if line.startswith(TTest.SKIPPED_PREFIX):
                 line = line.splitlines()[0]
-                missing.append(line[len(TTest.SKIPPED_PREFIX):])
+                missing.append(line[len(TTest.SKIPPED_PREFIX):].decode('utf-8'))
             elif line.startswith(TTest.FAILED_PREFIX):
                 line = line.splitlines()[0]
-                failed.append(line[len(TTest.FAILED_PREFIX):])
+                failed.append(line[len(TTest.FAILED_PREFIX):].decode('utf-8'))
 
         return missing, failed
 
     @staticmethod
     def _escapef(m):
         return TTest.ESCAPEMAP[m.group(0)]
 
     @staticmethod
@@ -1637,17 +1638,17 @@  class TestSuite(unittest.TestSuite):
 # Save the most recent 5 wall-clock runtimes of each test to a
 # human-readable text file named .testtimes. Tests are sorted
 # alphabetically, while times for each test are listed from oldest to
 # newest.
 
 def loadtimes(testdir):
     times = []
     try:
-        with open(os.path.join(testdir, '.testtimes-')) as fp:
+        with open(os.path.join(testdir, b'.testtimes-')) as fp:
             for line in fp:
                 ts = line.split()
                 times.append((ts[0], [float(t) for t in ts[1:]]))
     except IOError as err:
         if err.errno != errno.ENOENT:
             raise
     return times
 
@@ -1657,22 +1658,22 @@  def savetimes(testdir, result):
     skipped = set([str(t[0]) for t in result.skipped])
     for tdata in result.times:
         test, real = tdata[0], tdata[3]
         if test not in skipped:
             ts = saved.setdefault(test, [])
             ts.append(real)
             ts[:] = ts[-maxruns:]
 
-    fd, tmpname = tempfile.mkstemp(prefix='.testtimes',
+    fd, tmpname = tempfile.mkstemp(prefix=b'.testtimes',
                                    dir=testdir, text=True)
     with os.fdopen(fd, 'w') as fp:
-        for name, ts in sorted(saved.iteritems()):
+        for name, ts in sorted(saved.items()):
             fp.write('%s %s\n' % (name, ' '.join(['%.3f' % (t,) for t in ts])))
-    timepath = os.path.join(testdir, '.testtimes')
+    timepath = os.path.join(testdir, b'.testtimes')
     try:
         os.unlink(timepath)
     except OSError:
         pass
     try:
         os.rename(tmpname, timepath)
     except OSError:
         pass
@@ -1734,39 +1735,39 @@  class TextTestRunner(unittest.TextTestRu
                         # fail if string isn't ASCII.
                         err = cdatasafe(err).decode('utf-8', 'replace')
                         cd = doc.createCDATASection(err)
                         t.appendChild(cd)
                         s.appendChild(t)
                     xuf.write(doc.toprettyxml(indent='  ', encoding='utf-8'))
 
             if self._runner.options.json:
-                jsonpath = os.path.join(self._runner._testdir, 'report.json')
+                jsonpath = os.path.join(self._runner._testdir, b'report.json')
                 with open(jsonpath, 'w') as fp:
                     timesd = {}
                     for tdata in result.times:
                         test = tdata[0]
                         timesd[test] = tdata[1:]
 
                     outcome = {}
                     groups = [('success', ((tc, None)
                                for tc in result.successes)),
                               ('failure', result.failures),
                               ('skip', result.skipped)]
                     for res, testcases in groups:
                         for tc, __ in testcases:
                             if tc.name in timesd:
+                                diff = result.faildata.get(tc.name, b'')
                                 tres = {'result': res,
                                         'time': ('%0.3f' % timesd[tc.name][2]),
                                         'cuser': ('%0.3f' % timesd[tc.name][0]),
                                         'csys': ('%0.3f' % timesd[tc.name][1]),
                                         'start': ('%0.3f' % timesd[tc.name][3]),
                                         'end': ('%0.3f' % timesd[tc.name][4]),
-                                        'diff': result.faildata.get(tc.name,
-                                                                    ''),
+                                        'diff': diff.decode('unicode_escape'),
                                         }
                             else:
                                 # blacklisted test
                                 tres = {'result': res}
 
                             outcome[tc.name] = tres
                     jsonout = json.dumps(outcome, sort_keys=True, indent=4)
                     fp.writelines(("testreport =", jsonout))