From patchwork Sun Feb 28 05:35:57 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: [6,of,6] run-tests: fix Python 3 incompatibilities From: Gregory Szorc X-Patchwork-Id: 13445 Message-Id: <030fd4e3aa88b04749dc.1456637757@ubuntu-vm-main> To: mercurial-devel@mercurial-scm.org Date: Sat, 27 Feb 2016 21:35:57 -0800 # HG changeset patch # User Gregory Szorc # Date 1456637382 28800 # Sat Feb 27 21:29:42 2016 -0800 # Node ID 030fd4e3aa88b04749dcd0f7b3d05f6e604c27a4 # Parent c8a7c41a7d218dabc55c0f7e235d316112d81f32 run-tests: fix Python 3 incompatibilities At one point run-tests.py and test-run-tests.t worked and passed under Python 3.5. Various changes to run-tests.py over the past several months appear to have broken Python 3.5 compatibility. This patch implements various fixes (all related to str/bytes type coercion) to make run-tests.py and test-run-tests.t mostly work again. There are still a few failures in test-run-tests.t due to issues importing mercurial.* modules. But at least run-tests.py seems to work under 3.5 again. diff --git a/tests/run-tests.py b/tests/run-tests.py --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -693,17 +693,18 @@ class Test(unittest.TestCase): def tearDown(self): """Tasks to perform after run().""" for entry in self._daemonpids: killdaemons(entry) self._daemonpids = [] if self._keeptmpdir: log('\nKeeping testtmp dir: %s\nKeeping threadtmp dir: %s' % - (self._testtmp, self._threadtmp)) + (self._testtmp.decode('utf-8'), + self._threadtmp.decode('utf-8'))) else: shutil.rmtree(self._testtmp, True) shutil.rmtree(self._threadtmp, True) if (self._ret != 0 or self._out != self._refout) and not self._skipped \ and not self._debug and self._out: f = open(self.errpath, 'wb') for line in self._out: @@ -716,17 +717,17 @@ class Test(unittest.TestCase): # This should be implemented in child classes to run tests. raise SkipTest('unknown test type') def abort(self): """Terminate execution of this test.""" self._aborted = True def _portmap(self, i): - offset = '' if i == 0 else '%s' % i + offset = b'' if i == 0 else b'%d' % i return (br':%d\b' % (self._startport + i), b':$HGPORT%s' % offset) def _getreplacements(self): """Obtain a mapping of text replacements to apply to test output. Test output needs to be normalized so it can be compared to expected output. This function defines how some of that normalization will occur. @@ -904,18 +905,18 @@ checkcodeglobpats = [ bchr = chr if PYTHON3: bchr = lambda x: bytes([x]) class TTest(Test): """A "t test" is a test backed by a .t file.""" - SKIPPED_PREFIX = 'skipped: ' - FAILED_PREFIX = 'hghave check failed: ' + SKIPPED_PREFIX = b'skipped: ' + FAILED_PREFIX = b'hghave check failed: ' NEEDESCAPE = re.compile(br'[\x00-\x08\x0b-\x1f\x7f-\xff]').search ESCAPESUB = re.compile(br'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub ESCAPEMAP = dict((bchr(i), br'\x%02x' % i) for i in range(256)) ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'}) @property def refpath(self): @@ -1208,17 +1209,17 @@ class TTest(Test): @staticmethod def linematch(el, l): retry = False if el == l: # perfect match (fast) return True if el: if el.endswith(b" (?)\n"): retry = "retry" - el = el[:-5] + "\n" + el = el[:-5] + b"\n" if el.endswith(b" (esc)\n"): if PYTHON3: el = el[:-7].decode('unicode_escape') + '\n' el = el.encode('utf-8') else: el = el[:-7].decode('string-escape') + '\n' if el == l or os.name == 'nt' and el[:-1] + b'\r\n' == l: return True @@ -1240,20 +1241,20 @@ class TTest(Test): Return tuple of lists (missing, failed): * the missing/unknown features * the features for which existence check failed''' missing = [] failed = [] for line in lines: if line.startswith(TTest.SKIPPED_PREFIX): line = line.splitlines()[0] - missing.append(line[len(TTest.SKIPPED_PREFIX):]) + missing.append(line[len(TTest.SKIPPED_PREFIX):].decode('utf-8')) elif line.startswith(TTest.FAILED_PREFIX): line = line.splitlines()[0] - failed.append(line[len(TTest.FAILED_PREFIX):]) + failed.append(line[len(TTest.FAILED_PREFIX):].decode('utf-8')) return missing, failed @staticmethod def _escapef(m): return TTest.ESCAPEMAP[m.group(0)] @staticmethod @@ -1637,17 +1638,17 @@ class TestSuite(unittest.TestSuite): # Save the most recent 5 wall-clock runtimes of each test to a # human-readable text file named .testtimes. Tests are sorted # alphabetically, while times for each test are listed from oldest to # newest. def loadtimes(testdir): times = [] try: - with open(os.path.join(testdir, '.testtimes-')) as fp: + with open(os.path.join(testdir, b'.testtimes-')) as fp: for line in fp: ts = line.split() times.append((ts[0], [float(t) for t in ts[1:]])) except IOError as err: if err.errno != errno.ENOENT: raise return times @@ -1657,22 +1658,22 @@ def savetimes(testdir, result): skipped = set([str(t[0]) for t in result.skipped]) for tdata in result.times: test, real = tdata[0], tdata[3] if test not in skipped: ts = saved.setdefault(test, []) ts.append(real) ts[:] = ts[-maxruns:] - fd, tmpname = tempfile.mkstemp(prefix='.testtimes', + fd, tmpname = tempfile.mkstemp(prefix=b'.testtimes', dir=testdir, text=True) with os.fdopen(fd, 'w') as fp: - for name, ts in sorted(saved.iteritems()): + for name, ts in sorted(saved.items()): fp.write('%s %s\n' % (name, ' '.join(['%.3f' % (t,) for t in ts]))) - timepath = os.path.join(testdir, '.testtimes') + timepath = os.path.join(testdir, b'.testtimes') try: os.unlink(timepath) except OSError: pass try: os.rename(tmpname, timepath) except OSError: pass @@ -1734,39 +1735,39 @@ class TextTestRunner(unittest.TextTestRu # fail if string isn't ASCII. err = cdatasafe(err).decode('utf-8', 'replace') cd = doc.createCDATASection(err) t.appendChild(cd) s.appendChild(t) xuf.write(doc.toprettyxml(indent=' ', encoding='utf-8')) if self._runner.options.json: - jsonpath = os.path.join(self._runner._testdir, 'report.json') + jsonpath = os.path.join(self._runner._testdir, b'report.json') with open(jsonpath, 'w') as fp: timesd = {} for tdata in result.times: test = tdata[0] timesd[test] = tdata[1:] outcome = {} groups = [('success', ((tc, None) for tc in result.successes)), ('failure', result.failures), ('skip', result.skipped)] for res, testcases in groups: for tc, __ in testcases: if tc.name in timesd: + diff = result.faildata.get(tc.name, b'') tres = {'result': res, 'time': ('%0.3f' % timesd[tc.name][2]), 'cuser': ('%0.3f' % timesd[tc.name][0]), 'csys': ('%0.3f' % timesd[tc.name][1]), 'start': ('%0.3f' % timesd[tc.name][3]), 'end': ('%0.3f' % timesd[tc.name][4]), - 'diff': result.faildata.get(tc.name, - ''), + 'diff': diff.decode('unicode_escape'), } else: # blacklisted test tres = {'result': res} outcome[tc.name] = tres jsonout = json.dumps(outcome, sort_keys=True, indent=4) fp.writelines(("testreport =", jsonout))