From 03755364ff17e5bd6cf6408d982df748fa13376c Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Sat, 25 May 2019 20:00:04 +0100 Subject: Add scripts to run verifuzz --- scripts/run | 15 +++++++++++++++ scripts/swarm | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 scripts/run create mode 100755 scripts/swarm (limited to 'scripts') diff --git a/scripts/run b/scripts/run new file mode 100755 index 0000000..f6683cb --- /dev/null +++ b/scripts/run @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +import subprocess + +def main(): + i = 0 + name = "output_yosys_all" + config = "experiments/yosys_all.toml" + iterations = 50 + while True: + subprocess.call(["verifuzz", "fuzz", "-o", name + str(i), "-c", config, "-n", str(iterations)]) + i += 1 + +if __name__ == '__main__': + main() diff --git a/scripts/swarm b/scripts/swarm new file mode 100755 index 0000000..526fb30 --- /dev/null +++ b/scripts/swarm @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import subprocess + +def main(): + i = 0 + name = "output_large_rand" + config = "experiments/config_large.toml" + iterations = 20 + while True: + subprocess.call(["verifuzz", "config", "-c", config, "-o", "config_random.toml", "--randomise"]) + subprocess.call(["verifuzz", "fuzz", "-o", name + str(i), "-c", "config_random.toml", "-n", str(iterations)]) + i += 1 + +if __name__ == '__main__': + main() -- cgit From 87b0f99853072ad1b17b380726b3807f6f4f91d4 Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Sun, 26 May 2019 16:25:40 +0100 Subject: Add changes to work with older Vivado --- scripts/run | 4 ++-- scripts/swarm | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/run b/scripts/run index f6683cb..311721a 100755 --- a/scripts/run +++ b/scripts/run @@ -4,8 +4,8 @@ import subprocess def main(): i = 0 - name = "output_yosys_all" - config = "experiments/yosys_all.toml" + name = "output_vivado_all" + config = "experiments/vivado_all.toml" iterations = 50 while True: subprocess.call(["verifuzz", "fuzz", "-o", name + str(i), "-c", config, "-n", str(iterations)]) diff --git a/scripts/swarm b/scripts/swarm index 526fb30..b489f00 100755 --- a/scripts/swarm +++ b/scripts/swarm @@ -4,8 +4,8 @@ import subprocess def main(): i = 0 - name = "output_large_rand" - config = "experiments/config_large.toml" + name = "output_swarm_medium2_rand" + config = "experiments/config_medium.toml" iterations = 20 while True: subprocess.call(["verifuzz", "config", "-c", config, "-o", "config_random.toml", "--randomise"]) -- cgit From 850877011c58199f40ba26c47a071d06f8816b89 Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Thu, 30 May 2019 16:23:36 +0100 Subject: Add changes to swarm --- scripts/run | 13 ++++++++++--- scripts/swarm | 16 +++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/run b/scripts/run index 311721a..63295af 100755 --- a/scripts/run +++ b/scripts/run @@ -1,14 +1,21 @@ #!/usr/bin/env python3 import subprocess +import os def main(): i = 0 - name = "output_vivado_all" - config = "experiments/vivado_all.toml" + name = "mediumB" + config = "experiments/config_yosys.toml" iterations = 50 + directory = "yosys_all" + if not os.path.exists(directory): + os.makedirs(directory) while True: - subprocess.call(["verifuzz", "fuzz", "-o", name + str(i), "-c", config, "-n", str(iterations)]) + subprocess.call(["verifuzz", "fuzz" + , "-o", directory + "/" + name + str(i) + , "-c", config + , "-n", str(iterations)]) i += 1 if __name__ == '__main__': diff --git a/scripts/swarm b/scripts/swarm index b489f00..99b0c54 100755 --- a/scripts/swarm +++ b/scripts/swarm @@ -1,15 +1,25 @@ #!/usr/bin/env python3 import subprocess +import os def main(): i = 0 - name = "output_swarm_medium2_rand" + name = "medium" config = "experiments/config_medium.toml" iterations = 20 + directory = "swarm" + if not os.path.exists(directory): + os.makedirs(directory) while True: - subprocess.call(["verifuzz", "config", "-c", config, "-o", "config_random.toml", "--randomise"]) - subprocess.call(["verifuzz", "fuzz", "-o", name + str(i), "-c", "config_random.toml", "-n", str(iterations)]) + subprocess.call(["verifuzz", "config" + , "-c", config + , "-o", directory + "/config_medium_random.toml" + , "--randomise"]) + subprocess.call([ "verifuzz", "fuzz" + , "-o", directory + "/" + name + str(i) + , "-c", directory + "/config_medium_random.toml" + , "-n", str(iterations)]) i += 1 if __name__ == '__main__': -- cgit From a52240fe4f9a160f3fcd65217b7f7307fa13e820 Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Thu, 30 May 2019 16:00:57 +0100 Subject: Add conversion script --- scripts/convert.py | 21 +++++++++++++++++++++ scripts/run | 22 ---------------------- scripts/run.py | 22 ++++++++++++++++++++++ scripts/size | 30 ------------------------------ scripts/size.py | 30 ++++++++++++++++++++++++++++++ scripts/swarm | 26 -------------------------- scripts/swarm.py | 26 ++++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 78 deletions(-) create mode 100644 scripts/convert.py delete mode 100755 scripts/run create mode 100755 scripts/run.py delete mode 100755 scripts/size create mode 100755 scripts/size.py delete mode 100755 scripts/swarm create mode 100755 scripts/swarm.py (limited to 'scripts') diff --git a/scripts/convert.py b/scripts/convert.py new file mode 100644 index 0000000..e179774 --- /dev/null +++ b/scripts/convert.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import sys +from bs4 import BeautifulSoup +import csv + +def main(file_): + with open(file_, "r") as f: + file_contents = f.read() + + soup = BeautifulSoup(file_contents) + table = soup.select_one("table") + headers = [th.text.encode("utf-8") for th in table.select("tr th")] + + with open("out.csv", "w") as f: + wr = csv.writer(f) + wr.writerow(headers) + wr.writerows([[td.text.encode("utf-8") for td in row.find_all("td")] for row in table.select("tr + tr")]) + +if __name__ == '__main__': + main(sys.argv[1]) diff --git a/scripts/run b/scripts/run deleted file mode 100755 index 63295af..0000000 --- a/scripts/run +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import os - -def main(): - i = 0 - name = "mediumB" - config = "experiments/config_yosys.toml" - iterations = 50 - directory = "yosys_all" - if not os.path.exists(directory): - os.makedirs(directory) - while True: - subprocess.call(["verifuzz", "fuzz" - , "-o", directory + "/" + name + str(i) - , "-c", config - , "-n", str(iterations)]) - i += 1 - -if __name__ == '__main__': - main() diff --git a/scripts/run.py b/scripts/run.py new file mode 100755 index 0000000..63295af --- /dev/null +++ b/scripts/run.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import subprocess +import os + +def main(): + i = 0 + name = "mediumB" + config = "experiments/config_yosys.toml" + iterations = 50 + directory = "yosys_all" + if not os.path.exists(directory): + os.makedirs(directory) + while True: + subprocess.call(["verifuzz", "fuzz" + , "-o", directory + "/" + name + str(i) + , "-c", config + , "-n", str(iterations)]) + i += 1 + +if __name__ == '__main__': + main() diff --git a/scripts/size b/scripts/size deleted file mode 100755 index d6d7466..0000000 --- a/scripts/size +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import statistics -import argparse -import os - -def file_len(fname): - with open(fname) as f: - for i, l in enumerate(f): - pass - return i + 1 - -def main(c, n): - l = [] - for x in range(0, n): - subprocess.call(["verifuzz", "generate", "-o", "main.v", "-c", c]) - l.append(file_len("main.v")) - os.remove("main.v") - print("mean: ", statistics.mean(l)) - print("median: ", statistics.median(l)) - print("stdev: ", statistics.stdev(l)) - print("variance: ", statistics.variance(l)) - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Check the average size of a generated program.') - parser.add_argument('config', metavar='CONFIG', help='The config file to test.') - parser.add_argument('-n', '--num', help='How many iterations to run.', default=10, type=int) - args = parser.parse_args() - main(args.config, args.num) diff --git a/scripts/size.py b/scripts/size.py new file mode 100755 index 0000000..d6d7466 --- /dev/null +++ b/scripts/size.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import subprocess +import statistics +import argparse +import os + +def file_len(fname): + with open(fname) as f: + for i, l in enumerate(f): + pass + return i + 1 + +def main(c, n): + l = [] + for x in range(0, n): + subprocess.call(["verifuzz", "generate", "-o", "main.v", "-c", c]) + l.append(file_len("main.v")) + os.remove("main.v") + print("mean: ", statistics.mean(l)) + print("median: ", statistics.median(l)) + print("stdev: ", statistics.stdev(l)) + print("variance: ", statistics.variance(l)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Check the average size of a generated program.') + parser.add_argument('config', metavar='CONFIG', help='The config file to test.') + parser.add_argument('-n', '--num', help='How many iterations to run.', default=10, type=int) + args = parser.parse_args() + main(args.config, args.num) diff --git a/scripts/swarm b/scripts/swarm deleted file mode 100755 index 99b0c54..0000000 --- a/scripts/swarm +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import os - -def main(): - i = 0 - name = "medium" - config = "experiments/config_medium.toml" - iterations = 20 - directory = "swarm" - if not os.path.exists(directory): - os.makedirs(directory) - while True: - subprocess.call(["verifuzz", "config" - , "-c", config - , "-o", directory + "/config_medium_random.toml" - , "--randomise"]) - subprocess.call([ "verifuzz", "fuzz" - , "-o", directory + "/" + name + str(i) - , "-c", directory + "/config_medium_random.toml" - , "-n", str(iterations)]) - i += 1 - -if __name__ == '__main__': - main() diff --git a/scripts/swarm.py b/scripts/swarm.py new file mode 100755 index 0000000..99b0c54 --- /dev/null +++ b/scripts/swarm.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import subprocess +import os + +def main(): + i = 0 + name = "medium" + config = "experiments/config_medium.toml" + iterations = 20 + directory = "swarm" + if not os.path.exists(directory): + os.makedirs(directory) + while True: + subprocess.call(["verifuzz", "config" + , "-c", config + , "-o", directory + "/config_medium_random.toml" + , "--randomise"]) + subprocess.call([ "verifuzz", "fuzz" + , "-o", directory + "/" + name + str(i) + , "-c", directory + "/config_medium_random.toml" + , "-n", str(iterations)]) + i += 1 + +if __name__ == '__main__': + main() -- cgit From 515cd6d8da6a806ee4d5d830affb5aa7b2aed973 Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Thu, 30 May 2019 16:16:31 +0100 Subject: Fix the conversion --- scripts/convert.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) mode change 100644 => 100755 scripts/convert.py (limited to 'scripts') diff --git a/scripts/convert.py b/scripts/convert.py old mode 100644 new mode 100755 index e179774..a1e68ca --- a/scripts/convert.py +++ b/scripts/convert.py @@ -3,19 +3,29 @@ import sys from bs4 import BeautifulSoup import csv +import re def main(file_): with open(file_, "r") as f: file_contents = f.read() - soup = BeautifulSoup(file_contents) + sec = re.compile(r"([0-9.]+)s") + + soup = BeautifulSoup(file_contents, "html.parser") table = soup.select_one("table") - headers = [th.text.encode("utf-8") for th in table.select("tr th")] + headers = [th.text for th in table.select("tr th")] + vals = [[td.text for td in row.find_all("td")] for row in table.select("tr + tr")][:-2] + + headers = map(lambda x: "Size" if x == "Size (loc)" else x, headers) + + vals = map(lambda l: map(lambda s: sec.sub(r"\1", s), l), vals) + vals = map(lambda l: map(lambda s: re.sub(r"Failed", r"1", s), l), vals) + vals = map(lambda l: map(lambda s: re.sub(r"Passed", r"0", s), l), vals) with open("out.csv", "w") as f: wr = csv.writer(f) wr.writerow(headers) - wr.writerows([[td.text.encode("utf-8") for td in row.find_all("td")] for row in table.select("tr + tr")]) + wr.writerows(vals) if __name__ == '__main__': main(sys.argv[1]) -- cgit From 93fcb1e428a1d21a4b19b3e611e03e47399a5557 Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Thu, 30 May 2019 16:24:41 +0100 Subject: Add output --- scripts/convert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/convert.py b/scripts/convert.py index a1e68ca..0552367 100755 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -5,7 +5,7 @@ from bs4 import BeautifulSoup import csv import re -def main(file_): +def main(file_, output): with open(file_, "r") as f: file_contents = f.read() @@ -22,10 +22,10 @@ def main(file_): vals = map(lambda l: map(lambda s: re.sub(r"Failed", r"1", s), l), vals) vals = map(lambda l: map(lambda s: re.sub(r"Passed", r"0", s), l), vals) - with open("out.csv", "w") as f: + with open(output, "w") as f: wr = csv.writer(f) wr.writerow(headers) wr.writerows(vals) if __name__ == '__main__': - main(sys.argv[1]) + main(sys.argv[1], sys.argv[2]) -- cgit From 74f410dc07afe2d0a07f61a1dfa85c15cc31e446 Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Sun, 2 Jun 2019 13:23:23 +0100 Subject: Add selection for soup --- scripts/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/convert.py b/scripts/convert.py index 0552367..d592fd6 100755 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -12,7 +12,7 @@ def main(file_, output): sec = re.compile(r"([0-9.]+)s") soup = BeautifulSoup(file_contents, "html.parser") - table = soup.select_one("table") + table = soup.select_one("table.table") headers = [th.text for th in table.select("tr th")] vals = [[td.text for td in row.find_all("td")] for row in table.select("tr + tr")][:-2] -- cgit From 54c2072ac4b92bb3db3fcd0208cbb877f1abbf8d Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Thu, 6 Jun 2019 12:18:50 +0100 Subject: Add exclude script --- scripts/exclude.sh | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 scripts/exclude.sh (limited to 'scripts') diff --git a/scripts/exclude.sh b/scripts/exclude.sh new file mode 100644 index 0000000..cfaf514 --- /dev/null +++ b/scripts/exclude.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# bug0: ./output_vivado_all2/fuzz_8/vivado_2018.2/hs_err_pid108202.log +bug0="HOptGenControl::updateConst\(UConst\*, int, URange const&, UConst::Type, UConst::Type\)\+0x8d" + +# bug1: ./output_vivado_all2/fuzz_18/vivado_2016.2/hs_err_pid128529.log +bug1="HOptDfg::reconnectLoadPinToSource\(DFPin\*, DFPin\*\)\+0x247" + +# bug2: ./output_vivado_all2/fuzz_1/vivado_2016.2/hs_err_pid99371.log +bug2="HOptDfg::reconnectLoadPinToSource\(DFPin\*, DFPin\*\)\+0x2c0" + +# bug3: ./output_vivado_all2/fuzz_1/vivado_2018.2/hs_err_pid99120.log +bug3="HOptDfg::reconnectLoadPinToSource\(DFPin\*, DFPin\*\)\+0x23b" + +# bug4: ./size_test_length_no_combine/1/output5/fuzz_20/reduce_vivado/hs_err_pid52393.log +bug4="HOptDfg::mergeReconvergentPartitions\(DFPin\*, DFGraph\*, UHashSet, UHashSetNode, DFPin\*, UEValue > > const&, UHashMap >&, UHashMap >&, DFGraph::DFGraphType, UHashMapList, UHashListNode, DFPin\*>, DFGraph\*, UEKey >&\)\+0x1c6" + +# bug5: ./swarm/medium95/fuzz_14/reduce_vivado/hs_err_pid126430.log +bug5="HOptDfg::mergeReconvergentPartitions\(DFPin\*, DFGraph\*, UHashSet, UHashSetNode, DFPin\*, UEValue > > const&, UHashMap >&, UHashMap >&, DFGraph::DFGraphType, UHashMapList, UHashListNode, DFPin\*>, DFGraph\*, UEKey >&\)\+0x2e8" + +# bug6: ./swarm/medium108/fuzz_3/reduce_vivado/hs_err_pid95577.log +bug6="DD::DD\(Cudd\*, DdNode\*\)\+0x2a" + +# bug7: ./output_vivado/medium13/fuzz_12/vivado_2016.2/hs_err_pid47970.log +bug7="HOptDfg::mergeReconvergentPartitions\(DFPin\*, DFGraph\*, UHashSet, UHashSetNode, DFPin\*, UEValue > > const&, UHashMap >&, UHashMap >&, DFGraph::DFGraphType, UHashMapList, UHashListNode, DFPin\*>, DFGraph\*, UEKey >&\)\+0x241" + +# bug8: ./output_vivado/medium10/fuzz_5/vivado_2016.2/hs_err_pid50009.log +bug8="HOptGenControl::extractSyncRSForWireOrMerge\(DFGraph\*, DFNode\*, URange const&, DFEdge\*, DFPin\*\*, UConst\*, int, UConst\*, int, bool, DFPin\*&, UHashSet, UHashSetNode, DFNode\*, UEValue > >&, bool\)\+0xc33" + +grep -E "$bug0|$bug1|$bug2|$bug3|$bug4|$bug5|$bug6|$bug7|$bug8" $1 >/dev/null 2>&1 +exitcode=$? + +if [[ $exitcode -ne 0 ]]; then + echo $1 + head $1 + echo +fi -- cgit