def adder(a,b): # function to test. """ adds a to b""" return a+b options = [9, 9.0, [9], "9"] # options for test function test_template = """ def test_adder_{}(): _ = adder{} """ # -- template for test that fail -- def test_discovery(): from pathlib import Path from itertools import product test_number = 0 for c in product(*[options, options]): test_number += 1 try: output = adder(*c) except Exception: new_test = test_template.format(test_number, c) # write up test. with Path(__file__).open('a') as fo: # append it to the file. fo.write(new_test) test_discovery() # main callable used to populate tests. # test that fail will be written from here.