Patchwork [3,of,5] run-tests: write JSON reports to output dir

login
register
mail settings
Submitter Siddharth Agarwal
Date June 8, 2017, 4:26 a.m.
Message ID <511ebc1769a9de229c5c.1496895996@devvm31800.prn1.facebook.com>
Download mbox | patch
Permalink /patch/21241/
State Accepted
Headers show

Comments

Siddharth Agarwal - June 8, 2017, 4:26 a.m.
# HG changeset patch
# User Siddharth Agarwal <sid0@fb.com>
# Date 1496893603 25200
#      Wed Jun 07 20:46:43 2017 -0700
# Node ID 511ebc1769a9de229c5cca69ef3893bea1f2a63a
# Parent  47a479f6ee522ad1f1d4e72a8d32096000360dc2
run-tests: write JSON reports to output dir
Pierre-Yves David - June 8, 2017, 1:26 p.m.
On 06/08/2017 05:26 AM, Siddharth Agarwal wrote:
> # HG changeset patch
> # User Siddharth Agarwal <sid0@fb.com>
> # Date 1496893603 25200
> #      Wed Jun 07 20:46:43 2017 -0700
> # Node ID 511ebc1769a9de229c5cca69ef3893bea1f2a63a
> # Parent  47a479f6ee522ad1f1d4e72a8d32096000360dc2
> run-tests: write JSON reports to output dir

Small glob missing in that patch, easy to fix in-flight.

> diff --git a/tests/test-run-tests.t b/tests/test-run-tests.t
> --- a/tests/test-run-tests.t
> +++ b/tests/test-run-tests.t
> […]
> @@ -836,6 +836,68 @@ test for --json
>            "time": "\s*[\d\.]{4,5}" (re)
>        }
>    } (no-eol)
> +--json with --outputdir
> +
> +  $ rm report.json
> +  $ rm -r output
> +  $ mkdir output
> +  $ rt --json --outputdir output
> +
> +  --- $TESTTMP/test-failure.t
> +  +++ $TESTTMP/output/test-failure.t.err
> +  @@ -1,5 +1,5 @@
> +     $ echo babar
> +  -  rataxes
> +  +  babar
> +   This is a noop statement so that
> +   this test is still more bytes than success.
> +   pad pad pad pad............................................................
> +
> +  ERROR: test-failure.t output changed
> +  !.s
> +  Skipped test-skip.t: missing feature: nail clipper
> +  Failed test-failure.t: output changed
> +  # Ran 2 tests, 1 skipped, 0 warned, 1 failed.
> +  python hash seed: 1138307315
> +  [1]


That one passed your vigilance, that can be fixed in-flight
(patch 5 also fixes it)

> +  $ f report.json
> +  report.json: file not found
> +  $ cat output/report.json
> +  testreport ={
> +      "test-failure.t": [\{] (re)
> +          "csys": "\s*[\d\.]{4,5}", ? (re)
> +          "cuser": "\s*[\d\.]{4,5}", ? (re)
> +          "diff": "---.+\+\+\+.+", ? (re)
> +          "end": "\s*[\d\.]{4,5}", ? (re)
> +          "result": "failure", ? (re)
> +          "start": "\s*[\d\.]{4,5}", ? (re)
> +          "time": "\s*[\d\.]{4,5}" (re)
> +      }, ? (re)
> +      "test-skip.t": {
> +          "csys": "\s*[\d\.]{4,5}", ? (re)
> +          "cuser": "\s*[\d\.]{4,5}", ? (re)
> +          "diff": "", ? (re)
> +          "end": "\s*[\d\.]{4,5}", ? (re)
> +          "result": "skip", ? (re)
> +          "start": "\s*[\d\.]{4,5}", ? (re)
> +          "time": "\s*[\d\.]{4,5}" (re)
> +      }, ? (re)
> +      "test-success.t": [\{] (re)
> +          "csys": "\s*[\d\.]{4,5}", ? (re)
> +          "cuser": "\s*[\d\.]{4,5}", ? (re)
> +          "diff": "", ? (re)
> +          "end": "\s*[\d\.]{4,5}", ? (re)
> +          "result": "success", ? (re)
> +          "start": "\s*[\d\.]{4,5}", ? (re)
> +          "time": "\s*[\d\.]{4,5}" (re)
> +      }
> +  } (no-eol)
> +  $ ls -a output
> +  .
> +  ..
> +  .testtimes
> +  report.json
> +  test-failure.t.err
>
>  Test that failed test accepted through interactive are properly reported:
>
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>

Patch

diff --git a/tests/run-tests.py b/tests/run-tests.py
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -1925,7 +1925,7 @@  class TextTestRunner(unittest.TextTestRu
                 self._writexunit(result, xuf)
 
         if self._runner.options.json:
-            jsonpath = os.path.join(self._runner._testdir, b'report.json')
+            jsonpath = os.path.join(self._runner._outputdir, b'report.json')
             with open(jsonpath, 'w') as fp:
                 self._writejson(result, fp)
 
@@ -1960,7 +1960,7 @@  class TextTestRunner(unittest.TextTestRu
                     self._writexunit(result, xuf)
 
             if self._runner.options.json:
-                jsonpath = os.path.join(self._runner._testdir, b'report.json')
+                jsonpath = os.path.join(self._runner._outputdir, b'report.json')
                 with open(jsonpath, 'w') as fp:
                     self._writejson(result, fp)
 
diff --git a/tests/test-run-tests.t b/tests/test-run-tests.t
--- a/tests/test-run-tests.t
+++ b/tests/test-run-tests.t
@@ -300,10 +300,10 @@  test --xunit support
     <testcase name="test-success.t"/>
   </testsuite>
 
-  $ rt --list-tests test-failure* --json --xunit=xunit.xml
+  $ rt --list-tests test-failure* --json --xunit=xunit.xml --outputdir output
   test-failure-unicode.t
   test-failure.t
-  $ cat report.json
+  $ cat output/report.json
   testreport ={
       "test-failure-unicode.t": {
           "result": "success"
@@ -836,6 +836,68 @@  test for --json
           "time": "\s*[\d\.]{4,5}" (re)
       }
   } (no-eol)
+--json with --outputdir
+
+  $ rm report.json
+  $ rm -r output
+  $ mkdir output
+  $ rt --json --outputdir output
+  
+  --- $TESTTMP/test-failure.t
+  +++ $TESTTMP/output/test-failure.t.err
+  @@ -1,5 +1,5 @@
+     $ echo babar
+  -  rataxes
+  +  babar
+   This is a noop statement so that
+   this test is still more bytes than success.
+   pad pad pad pad............................................................
+  
+  ERROR: test-failure.t output changed
+  !.s
+  Skipped test-skip.t: missing feature: nail clipper
+  Failed test-failure.t: output changed
+  # Ran 2 tests, 1 skipped, 0 warned, 1 failed.
+  python hash seed: 1138307315
+  [1]
+  $ f report.json
+  report.json: file not found
+  $ cat output/report.json
+  testreport ={
+      "test-failure.t": [\{] (re)
+          "csys": "\s*[\d\.]{4,5}", ? (re)
+          "cuser": "\s*[\d\.]{4,5}", ? (re)
+          "diff": "---.+\+\+\+.+", ? (re)
+          "end": "\s*[\d\.]{4,5}", ? (re)
+          "result": "failure", ? (re)
+          "start": "\s*[\d\.]{4,5}", ? (re)
+          "time": "\s*[\d\.]{4,5}" (re)
+      }, ? (re)
+      "test-skip.t": {
+          "csys": "\s*[\d\.]{4,5}", ? (re)
+          "cuser": "\s*[\d\.]{4,5}", ? (re)
+          "diff": "", ? (re)
+          "end": "\s*[\d\.]{4,5}", ? (re)
+          "result": "skip", ? (re)
+          "start": "\s*[\d\.]{4,5}", ? (re)
+          "time": "\s*[\d\.]{4,5}" (re)
+      }, ? (re)
+      "test-success.t": [\{] (re)
+          "csys": "\s*[\d\.]{4,5}", ? (re)
+          "cuser": "\s*[\d\.]{4,5}", ? (re)
+          "diff": "", ? (re)
+          "end": "\s*[\d\.]{4,5}", ? (re)
+          "result": "success", ? (re)
+          "start": "\s*[\d\.]{4,5}", ? (re)
+          "time": "\s*[\d\.]{4,5}" (re)
+      }
+  } (no-eol)
+  $ ls -a output
+  .
+  ..
+  .testtimes
+  report.json
+  test-failure.t.err
 
 Test that failed test accepted through interactive are properly reported: