Submitter | Anurag Goel |
---|---|
Date | June 24, 2014, 9:15 a.m. |
Message ID | <49d16dcd51a0e4be7079.1403601330@ubuntu.ubuntu-domain> |
Download | mbox | patch |
Permalink | /patch/5050/ |
State | Changes Requested |
Headers | show |
Comments
On 06/24/2014 10:15 AM, Anurag Goel wrote: > # HG changeset patch > # User anuraggoel <anurag.dsps@gmail.com> > # Date 1403601145 -19800 > # Tue Jun 24 14:42:25 2014 +0530 > # Node ID 49d16dcd51a0e4be7079e32486021ede9fd3b175 > # Parent 4144906e24e4ec620fa7d815b8c2eff78ec0be6c > run-tests: added '--json' functionality to store test result data in json file > > This patch added a new functionality '--json'. While testing, if '--json' > is enabled then test result data gets stored in newly created "report.json" > file in the following format. > > testreport ={ > "test-success.t": [ > ".", > 7.2604079246521 > ], 1. do not use a list, use json object {"result": "success" "time": 7.26.04 } it make no sense to mix unrelated item in a list. 2. use real word for test result: "." and "!" are obscure "success" "failure" "error" are better. > "test-failure.t": [ > "!", > 4.783581018447876 > ], > "test-skipped.t": [ > "s", > 3.2986509799957275 > ] > } > > Json format contains a key-value pair where key represent 'testname' and value > is a list array contains 'successinfo' and 'timing data'. > > This "report.json" file will further accessed by html/javascript file for > graph usage. > > diff -r 4144906e24e4 -r 49d16dcd51a0 tests/run-tests.py > --- a/tests/run-tests.py Mon Jun 23 15:02:10 2014 +0530 > +++ b/tests/run-tests.py Tue Jun 24 14:42:25 2014 +0530 > @@ -58,6 +58,7 @@ > import killdaemons as killmod > import Queue as queue > import unittest > +import json Still won't work on 2.4 > > processlock = threading.Lock() > > @@ -185,6 +186,8 @@ > " (default: $%s or %d)" % defaults['timeout']) > parser.add_option("--time", action="store_true", > help="time how long each test takes") > + parser.add_option("--json", action="store_true", > + help="store test result data in 'report.json' file") > parser.add_option("--tmpdir", type="string", > help="run tests in the given temporary directory" > " (implies --keep-tmpdir)") > @@ -453,6 +456,11 @@ > return > > success = False > + > + # This carries the test success info corresponding to the testcase > + # where '!' represent failure, 's' represent skipped, '.' represent > + # success. > + successinfo = '!' > try: > self.runTest() > except KeyboardInterrupt: > @@ -460,6 +468,7 @@ > raise > except SkipTest, e: > result.addSkip(self, str(e)) > + successinfo = 's' > except IgnoreTest, e: > result.addIgnore(self, str(e)) > except WarnTest, e: > @@ -486,9 +495,10 @@ > success = False > > if success: > + successinfo = '.' > result.addSuccess(self) > finally: > - result.stopTest(self, interrupted=self._aborted) > + result.stopTest(self, successinfo, interrupted=self._aborted) > > def runTest(self): > """Run this test instance. > @@ -1075,6 +1085,13 @@ > self.warned = [] > > self.times = [] > + > + # Stores testfile name in the list > + self.testfiles = [] > + > + # Stores success info and timing data corresponding to each testcase > + self.outcome = [] > + Why do you need two lists here ? > self._started = {} > > def addFailure(self, test, reason): > @@ -1167,10 +1184,14 @@ > > self._started[test.name] = time.time() > > - def stopTest(self, test, interrupted=False): > + def stopTest(self, test, successinfo, interrupted=False): > super(TestResult, self).stopTest(test) > > - self.times.append((test.name, time.time() - self._started[test.name])) > + testtime = time.time() - self._started[test.name] > + > + self.testfiles.append(test.name) > + self.outcome.append((successinfo, testtime)) > + self.times.append((test.name, testtime)) > del self._started[test.name] > > if interrupted: > @@ -1341,9 +1362,19 @@ > os.environ['PYTHONHASHSEED']) > if self._runner.options.time: > self.printtimes(result.times) > + if self._runner.options.json: > + self.getjsonfile(result.testfiles, result.outcome) > > return result > > + def getjsonfile(self, testfiles, outcome): > + """Store test result info in json format in report.json file.""" > + > + fp = open("report.json", "w") This will write the file in the current working directory. not the test directory. sounds like a bad idea. > + testdata = dict(zip(testfiles, outcome)) > + fp.writelines(("testreport =", (json.dumps(testdata, indent=4)))) > + fp.close() > + You need to make sure the file is close using a try: … finally: … clause. > def printtimes(self, times): > self.stream.writeln('# Producing time report') > times.sort(key=lambda t: (t[1], t[0]), reverse=True) > diff -r 4144906e24e4 -r 49d16dcd51a0 tests/test-run-tests.t > --- a/tests/test-run-tests.t Mon Jun 23 15:02:10 2014 +0530 > +++ b/tests/test-run-tests.t Tue Jun 24 14:42:25 2014 +0530 > @@ -201,3 +201,37 @@ > # Ran 2 tests, 0 skipped, 0 warned, 1 failed. > python hash seed: * (glob) > [1] > + > +test for --json > +================== > + > + $ $TESTDIR/run-tests.py --with-hg=`which hg` --json > + > + --- $TESTTMP/test-failure.t > + +++ $TESTTMP/test-failure.t.err > + @@ -1,2 +1,2 @@ > + $ echo babar > + - rataxes > + + babar > + > + ERROR: test-failure.t output changed > + !. > + Failed test-failure.t: output changed > + # Ran 2 tests, 0 skipped, 0 warned, 1 failed. > + python hash seed: * (glob) > + [1] > + > + $ cat report.json > + testreport ={ > + "test-[a-z]{7}[\.]t": [\[] (re) > + "[\.|!]", (re) > + [\d\.]* (re) > + ], > + "test-[a-z]{7}[\.]t": [\[] (re) > + "[\.|!]", (re) > + [\d\.]* (re) > + ] > + } (no-eol) I'm sure you reduce this to a less generic output.
On Thu, Jun 26, 2014 at 9:05 PM, Pierre-Yves David < pierre-yves.david@ens-lyon.org> wrote: > > > On 06/24/2014 10:15 AM, Anurag Goel wrote: > >> # HG changeset patch >> # User anuraggoel <anurag.dsps@gmail.com> >> # Date 1403601145 -19800 >> # Tue Jun 24 14:42:25 2014 +0530 >> # Node ID 49d16dcd51a0e4be7079e32486021ede9fd3b175 >> # Parent 4144906e24e4ec620fa7d815b8c2eff78ec0be6c >> run-tests: added '--json' functionality to store test result data in json >> file >> >> This patch added a new functionality '--json'. While testing, if '--json' >> is enabled then test result data gets stored in newly created >> "report.json" >> file in the following format. >> >> testreport ={ >> "test-success.t": [ >> ".", >> 7.2604079246521 >> ], >> > > 1. do not use a list, use json object > > {"result": "success" > "time": 7.26.04 > } > > it make no sense to mix unrelated item in a list. > > 2. use real word for test result: "." and "!" are obscure "success" > "failure" "error" are better. > > > "test-failure.t": [ >> "!", >> 4.783581018447876 >> ], >> "test-skipped.t": [ >> "s", >> 3.2986509799957275 >> ] >> } >> >> Json format contains a key-value pair where key represent 'testname' and >> value >> is a list array contains 'successinfo' and 'timing data'. >> >> This "report.json" file will further accessed by html/javascript file for >> graph usage. >> >> diff -r 4144906e24e4 -r 49d16dcd51a0 tests/run-tests.py >> --- a/tests/run-tests.py Mon Jun 23 15:02:10 2014 +0530 >> +++ b/tests/run-tests.py Tue Jun 24 14:42:25 2014 +0530 >> @@ -58,6 +58,7 @@ >> import killdaemons as killmod >> import Queue as queue >> import unittest >> +import json >> > > Still won't work on 2.4 > > Yes, it won't work on 2.4. As we discussed in V2 of this patch, we will gracefully disable the option on 2.4 > >> processlock = threading.Lock() >> >> @@ -185,6 +186,8 @@ >> " (default: $%s or %d)" % defaults['timeout']) >> parser.add_option("--time", action="store_true", >> help="time how long each test takes") >> + parser.add_option("--json", action="store_true", >> + help="store test result data in 'report.json' file") >> parser.add_option("--tmpdir", type="string", >> help="run tests in the given temporary directory" >> " (implies --keep-tmpdir)") >> @@ -453,6 +456,11 @@ >> return >> >> success = False >> + >> + # This carries the test success info corresponding to the >> testcase >> + # where '!' represent failure, 's' represent skipped, '.' >> represent >> + # success. >> + successinfo = '!' >> try: >> self.runTest() >> except KeyboardInterrupt: >> @@ -460,6 +468,7 @@ >> raise >> except SkipTest, e: >> result.addSkip(self, str(e)) >> + successinfo = 's' >> except IgnoreTest, e: >> result.addIgnore(self, str(e)) >> except WarnTest, e: >> @@ -486,9 +495,10 @@ >> success = False >> >> if success: >> + successinfo = '.' >> result.addSuccess(self) >> finally: >> - result.stopTest(self, interrupted=self._aborted) >> + result.stopTest(self, successinfo, interrupted=self._aborted) >> >> def runTest(self): >> """Run this test instance. >> @@ -1075,6 +1085,13 @@ >> self.warned = [] >> >> self.times = [] >> + >> + # Stores testfile name in the list >> + self.testfiles = [] >> + >> + # Stores success info and timing data corresponding to each >> testcase >> + self.outcome = [] >> + >> > > Why do you need two lists here ? > > > self._started = {} >> >> def addFailure(self, test, reason): >> @@ -1167,10 +1184,14 @@ >> >> self._started[test.name] = time.time() >> >> - def stopTest(self, test, interrupted=False): >> + def stopTest(self, test, successinfo, interrupted=False): >> super(TestResult, self).stopTest(test) >> >> - self.times.append((test.name, time.time() - self._started[ >> test.name])) >> + testtime = time.time() - self._started[test.name] >> + >> + self.testfiles.append(test.name) >> + self.outcome.append((successinfo, testtime)) >> + self.times.append((test.name, testtime)) >> del self._started[test.name] >> >> if interrupted: >> @@ -1341,9 +1362,19 @@ >> os.environ['PYTHONHASHSEED']) >> if self._runner.options.time: >> self.printtimes(result.times) >> + if self._runner.options.json: >> + self.getjsonfile(result.testfiles, result.outcome) >> >> return result >> >> + def getjsonfile(self, testfiles, outcome): >> + """Store test result info in json format in report.json file.""" >> + >> + fp = open("report.json", "w") >> > > This will write the file in the current working directory. not the test > directory. sounds like a bad idea. > > > > + testdata = dict(zip(testfiles, outcome)) >> + fp.writelines(("testreport =", (json.dumps(testdata, indent=4)))) >> + fp.close() >> + >> > > You need to make sure the file is close using a try: … finally: … clause. > > > def printtimes(self, times): >> self.stream.writeln('# Producing time report') >> times.sort(key=lambda t: (t[1], t[0]), reverse=True) >> diff -r 4144906e24e4 -r 49d16dcd51a0 tests/test-run-tests.t >> --- a/tests/test-run-tests.t Mon Jun 23 15:02:10 2014 +0530 >> +++ b/tests/test-run-tests.t Tue Jun 24 14:42:25 2014 +0530 >> @@ -201,3 +201,37 @@ >> # Ran 2 tests, 0 skipped, 0 warned, 1 failed. >> python hash seed: * (glob) >> [1] >> + >> +test for --json >> +================== >> + >> + $ $TESTDIR/run-tests.py --with-hg=`which hg` --json >> + >> + --- $TESTTMP/test-failure.t >> + +++ $TESTTMP/test-failure.t.err >> + @@ -1,2 +1,2 @@ >> + $ echo babar >> + - rataxes >> + + babar >> + >> + ERROR: test-failure.t output changed >> + !. >> + Failed test-failure.t: output changed >> + # Ran 2 tests, 0 skipped, 0 warned, 1 failed. >> + python hash seed: * (glob) >> + [1] >> + >> + $ cat report.json >> + testreport ={ >> + "test-[a-z]{7}[\.]t": [\[] (re) >> + "[\.|!]", (re) >> + [\d\.]* (re) >> + ], >> + "test-[a-z]{7}[\.]t": [\[] (re) >> + "[\.|!]", (re) >> + [\d\.]* (re) >> + ] >> + } (no-eol) >> > > I'm sure you reduce this to a less generic output. > > I have changed the json format like as you suggested above. But i am facing issues in writing the test for it. Firstly there is uncertainty in ordering of testfile runs like sometime "test-success.t" runs first and sometimes "test-failure.t" runs first. Secondly there is no particular order, i am getting the json output. Sometime i get this { "result": "success", 'time": 2.04 } and sometimes i get this { 'time": 2.04, "result": "success" } As you can see order is different. Could you please help me with these issues?
On 06/28/2014 02:31 PM, Anurag Goel wrote: > On Thu, Jun 26, 2014 at 9:05 PM, Pierre-Yves David > <pierre-yves.david@ens-lyon.org <mailto:pierre-yves.david@ens-lyon.org>> > wrote: > > > > On 06/24/2014 10:15 AM, Anurag Goel wrote: > > # HG changeset patch > # User anuraggoel <anurag.dsps@gmail.com > <mailto:anurag.dsps@gmail.com>> > # Date 1403601145 -19800 > # Tue Jun 24 14:42:25 2014 +0530 > # Node ID 49d16dcd51a0e4be7079e32486021e__de9fd3b175 > # Parent 4144906e24e4ec620fa7d815b8c2ef__f78ec0be6c > run-tests: added '--json' functionality to store test result > data in json file > > This patch added a new functionality '--json'. While testing, if > '--json' > is enabled then test result data gets stored in newly created > "report.json" > file in the following format. > > testreport ={ > "test-success.t": [ > ".", > 7.2604079246521 > ], > > > 1. do not use a list, use json object > > {"result": "success" > "time": 7.26.04 > } > > it make no sense to mix unrelated item in a list. > > 2. use real word for test result: "." and "!" are obscure "success" > "failure" "error" are better. > > > "test-failure.t": [ > "!", > 4.783581018447876 > ], > "test-skipped.t": [ > "s", > 3.2986509799957275 > ] > } > > Json format contains a key-value pair where key represent > 'testname' and value > is a list array contains 'successinfo' and 'timing data'. > > This "report.json" file will further accessed by html/javascript > file for > graph usage. > > diff -r 4144906e24e4 -r 49d16dcd51a0 tests/run-tests.py > --- a/tests/run-tests.py Mon Jun 23 15:02:10 2014 +0530 > +++ b/tests/run-tests.py Tue Jun 24 14:42:25 2014 +0530 > @@ -58,6 +58,7 @@ > import killdaemons as killmod > import Queue as queue > import unittest > +import json > > > Still won't work on 2.4 > > Yes, it won't work on 2.4. As we discussed in V2 of this patch, we will > gracefully disable the option on 2.4 You have to handle this 2.4 thing -now- (in this patch) otherwise applying your patch will result in a broken test suite on 2.4. > processlock = threading.Lock() > > @@ -185,6 +186,8 @@ > " (default: $%s or %d)" % defaults['timeout']) > parser.add_option("--time", action="store_true", > help="time how long each test takes") > + parser.add_option("--json", action="store_true", > + help="store test result data in 'report.json' file") > parser.add_option("--tmpdir", type="string", > help="run tests in the given temporary directory" > " (implies --keep-tmpdir)") > @@ -453,6 +456,11 @@ > return > > success = False > + > + # This carries the test success info corresponding > to the testcase > + # where '!' represent failure, 's' represent > skipped, '.' represent > + # success. > + successinfo = '!' > try: > self.runTest() > except KeyboardInterrupt: > @@ -460,6 +468,7 @@ > raise > except SkipTest, e: > result.addSkip(self, str(e)) > + successinfo = 's' > except IgnoreTest, e: > result.addIgnore(self, str(e)) > except WarnTest, e: > @@ -486,9 +495,10 @@ > success = False > > if success: > + successinfo = '.' > result.addSuccess(self) > finally: > - result.stopTest(self, interrupted=self._aborted) > + result.stopTest(self, successinfo, > interrupted=self._aborted) > > def runTest(self): > """Run this test instance. > @@ -1075,6 +1085,13 @@ > self.warned = [] > > self.times = [] > + > + # Stores testfile name in the list > + self.testfiles = [] > + > + # Stores success info and timing data corresponding to > each testcase > + self.outcome = [] > + > > > Why do you need two lists here ? > > > self._started = {} > > def addFailure(self, test, reason): > @@ -1167,10 +1184,14 @@ > > self._started[test.name <http://test.name>] = time.time() > > - def stopTest(self, test, interrupted=False): > + def stopTest(self, test, successinfo, interrupted=False): > super(TestResult, self).stopTest(test) > > - self.times.append((test.name <http://test.name>, > time.time() - self._started[test.name <http://test.name>])) > + testtime = time.time() - self._started[test.name > <http://test.name>] > + > + self.testfiles.append(test.__name <http://test.name>) > + self.outcome.append((__successinfo, testtime)) > + self.times.append((test.name <http://test.name>, testtime)) > del self._started[test.name <http://test.name>] > > if interrupted: > @@ -1341,9 +1362,19 @@ > os.environ['PYTHONHASHSEED']) > if self._runner.options.time: > self.printtimes(result.times) > + if self._runner.options.json: > + self.getjsonfile(result.__testfiles, result.outcome) > > return result > > + def getjsonfile(self, testfiles, outcome): > + """Store test result info in json format in report.json > file.""" > + > + fp = open("report.json", "w") > > > This will write the file in the current working directory. not the > test directory. sounds like a bad idea. > > > > + testdata = dict(zip(testfiles, outcome)) > + fp.writelines(("testreport =", (json.dumps(testdata, > indent=4)))) > + fp.close() > + > > > You need to make sure the file is close using a try: … finally: … > clause. > > > def printtimes(self, times): > self.stream.writeln('# Producing time report') > times.sort(key=lambda t: (t[1], t[0]), reverse=True) > diff -r 4144906e24e4 -r 49d16dcd51a0 tests/test-run-tests.t > --- a/tests/test-run-tests.t Mon Jun 23 15:02:10 2014 +0530 > +++ b/tests/test-run-tests.t Tue Jun 24 14:42:25 2014 +0530 > @@ -201,3 +201,37 @@ > # Ran 2 tests, 0 skipped, 0 warned, 1 failed. > python hash seed: * (glob) > [1] > + > +test for --json > +================== > + > + $ $TESTDIR/run-tests.py --with-hg=`which hg` --json > + > + --- $TESTTMP/test-failure.t > + +++ $TESTTMP/test-failure.t.err > + @@ -1,2 +1,2 @@ > + $ echo babar > + - rataxes > + + babar > + > + ERROR: test-failure.t output changed > + !. > + Failed test-failure.t: output changed > + # Ran 2 tests, 0 skipped, 0 warned, 1 failed. > + python hash seed: * (glob) > + [1] > + > + $ cat report.json > + testreport ={ > + "test-[a-z]{7}[\.]t": [\[] (re) > + "[\.|!]", (re) > + [\d\.]* (re) > + ], > + "test-[a-z]{7}[\.]t": [\[] (re) > + "[\.|!]", (re) > + [\d\.]* (re) > + ] > + } (no-eol) > > > I'm sure you reduce this to a less generic output. > > I have changed the json format like as you suggested above. But i am > facing issues in writing the test for it. > > Firstly there is uncertainty in ordering of testfile runs like sometime > "test-success.t" runs first and sometimes "test-failure.t" runs first. The run order is deterministic (otherwise other tests outputs would have trouble). Is this a Json order issue? > Secondly there is no particular order, i am getting the json output. > > Sometime i get this > { > "result": "success", > 'time": 2.04 > } > > and sometimes i get this > { > 'time": 2.04, > "result": "success" > } > > As you can see order is different. Could you please help me with these > issues? I'm pretty sure python json have a way to sort the output. you should have a look at the documentation of the Json module.
Patch
diff -r 4144906e24e4 -r 49d16dcd51a0 tests/run-tests.py --- a/tests/run-tests.py Mon Jun 23 15:02:10 2014 +0530 +++ b/tests/run-tests.py Tue Jun 24 14:42:25 2014 +0530 @@ -58,6 +58,7 @@ import killdaemons as killmod import Queue as queue import unittest +import json processlock = threading.Lock() @@ -185,6 +186,8 @@ " (default: $%s or %d)" % defaults['timeout']) parser.add_option("--time", action="store_true", help="time how long each test takes") + parser.add_option("--json", action="store_true", + help="store test result data in 'report.json' file") parser.add_option("--tmpdir", type="string", help="run tests in the given temporary directory" " (implies --keep-tmpdir)") @@ -453,6 +456,11 @@ return success = False + + # This carries the test success info corresponding to the testcase + # where '!' represent failure, 's' represent skipped, '.' represent + # success. + successinfo = '!' try: self.runTest() except KeyboardInterrupt: @@ -460,6 +468,7 @@ raise except SkipTest, e: result.addSkip(self, str(e)) + successinfo = 's' except IgnoreTest, e: result.addIgnore(self, str(e)) except WarnTest, e: @@ -486,9 +495,10 @@ success = False if success: + successinfo = '.' result.addSuccess(self) finally: - result.stopTest(self, interrupted=self._aborted) + result.stopTest(self, successinfo, interrupted=self._aborted) def runTest(self): """Run this test instance. @@ -1075,6 +1085,13 @@ self.warned = [] self.times = [] + + # Stores testfile name in the list + self.testfiles = [] + + # Stores success info and timing data corresponding to each testcase + self.outcome = [] + self._started = {} def addFailure(self, test, reason): @@ -1167,10 +1184,14 @@ self._started[test.name] = time.time() - def stopTest(self, test, interrupted=False): + def stopTest(self, test, successinfo, interrupted=False): super(TestResult, self).stopTest(test) - self.times.append((test.name, time.time() - self._started[test.name])) + testtime = time.time() - self._started[test.name] + + self.testfiles.append(test.name) + self.outcome.append((successinfo, testtime)) + self.times.append((test.name, testtime)) del self._started[test.name] if interrupted: @@ -1341,9 +1362,19 @@ os.environ['PYTHONHASHSEED']) if self._runner.options.time: self.printtimes(result.times) + if self._runner.options.json: + self.getjsonfile(result.testfiles, result.outcome) return result + def getjsonfile(self, testfiles, outcome): + """Store test result info in json format in report.json file.""" + + fp = open("report.json", "w") + testdata = dict(zip(testfiles, outcome)) + fp.writelines(("testreport =", (json.dumps(testdata, indent=4)))) + fp.close() + def printtimes(self, times): self.stream.writeln('# Producing time report') times.sort(key=lambda t: (t[1], t[0]), reverse=True) diff -r 4144906e24e4 -r 49d16dcd51a0 tests/test-run-tests.t --- a/tests/test-run-tests.t Mon Jun 23 15:02:10 2014 +0530 +++ b/tests/test-run-tests.t Tue Jun 24 14:42:25 2014 +0530 @@ -201,3 +201,37 @@ # Ran 2 tests, 0 skipped, 0 warned, 1 failed. python hash seed: * (glob) [1] + +test for --json +================== + + $ $TESTDIR/run-tests.py --with-hg=`which hg` --json + + --- $TESTTMP/test-failure.t + +++ $TESTTMP/test-failure.t.err + @@ -1,2 +1,2 @@ + $ echo babar + - rataxes + + babar + + ERROR: test-failure.t output changed + !. + Failed test-failure.t: output changed + # Ran 2 tests, 0 skipped, 0 warned, 1 failed. + python hash seed: * (glob) + [1] + + $ cat report.json + testreport ={ + "test-[a-z]{7}[\.]t": [\[] (re) + "[\.|!]", (re) + [\d\.]* (re) + ], + "test-[a-z]{7}[\.]t": [\[] (re) + "[\.|!]", (re) + [\d\.]* (re) + ] + } (no-eol) + +(removing json file) + $ rm report.json