diff options
| -rw-r--r-- | test/pathological_tests.py | 39 |
1 files changed, 29 insertions, 10 deletions
diff --git a/test/pathological_tests.py b/test/pathological_tests.py index 40552de..b86f4c0 100644 --- a/test/pathological_tests.py +++ b/test/pathological_tests.py @@ -17,6 +17,8 @@ if __name__ == "__main__": default=None, help='directory containing dynamic library') args = parser.parse_args(sys.argv[1:]) +allowed_failures = {"many references": True} + cmark = CMark(prog=args.program, library_dir=args.library_dir) # list of pairs consisting of input and a regex that must match the output. @@ -74,40 +76,57 @@ pathological = { whitespace_re = re.compile('/s+/') -results = {'passed': 0, 'errored': 0, 'failed': 0} +results = {'passed': [], 'errored': [], 'failed': [], 'ignored': []} def run_pathological_test(description, results): (inp, regex) = pathological[description] [rc, actual, err] = cmark.to_html(inp) + extra = "" if rc != 0: print(description, '[ERRORED (return code %d)]' %rc) print(err) - results['errored'] += 1 + if allowed_failures[description]: + results['ignored'].append(description) + else: + results['errored'].append(description) elif regex.search(actual): print(description, '[PASSED]') - results['passed'] += 1 + results['passed'].append(description) else: print(description, '[FAILED]') print(repr(actual)) - results['failed'] += 1 - + if allowed_failures[description]: + results['ignored'].append(description) + else: + results['failed'].append(description) print("Testing pathological cases:") for description in pathological: p = multiprocessing.Process(target=run_pathological_test, args=(description, results,)) p.start() - # wait 8 seconds or until it finishes - p.join(8) + # wait 4 seconds or until it finishes + p.join(4) # kill it if still active if p.is_alive(): print(description, '[TIMEOUT]') + if allowed_failures[description]: + results['ignored'].append(description) + else: + results['errored'].append(description) p.terminate() - results['errored'] += 1 p.join() -print("%d passed, %d failed, %d errored" % - (results['passed'], results['failed'], results['errored'])) +passed = len(results['passed']) +failed = len(results['failed']) +errored = len(results['errored']) +ignored = len(results['ignored']) + +print("%d passed, %d failed, %d errored" % (passed, failed, errored)) +if ignored > 0: + print("Ignoring these allowed failures:") + for x in results['ignored']: + print(x) if (results['failed'] == 0 and results['errored'] == 0): exit(0) else: |