From 798f54c05376ec3b4ebbe8326d0a91eec807df3d Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Thu, 14 Nov 2019 16:27:19 +0000 Subject: Add proper reports to bugs --- experiments/reduction.org | 194 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 experiments/reduction.org (limited to 'experiments/reduction.org') diff --git a/experiments/reduction.org b/experiments/reduction.org new file mode 100644 index 0000000..f0b8f1e --- /dev/null +++ b/experiments/reduction.org @@ -0,0 +1,194 @@ +#+TITLE: Reduction results + +* Run 4 + +|-----+----------------+------+----------------+-----------+------+------+----+-------------| +| Run | Path | Type | VS T | CR T | Size | VS S | CR | CR (unadj.) | +|-----+----------------+------+----------------+-----------+------+------+----+-------------| +| 0 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_13/fuzz_70][0.8/5/4_13/70]] | MS | 24.228676396 | 546.138 | 515 | 8 | 23 | 16 | +| 1 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_0_21/fuzz_12][0.8/5/0_21/12]] | MS | 48.020800025 | 549.916 | 675 | 12 | 40 | 29 | +| 2 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_1_25/fuzz_8][0.8/5/1_25/8]] | C | 31.025951033 | 225.177 | 1063 | 462 | -- | 30 | +| 3 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_1_27/fuzz_49][0.8/5/1_27/47]] | MS | 239.19471845 | 2954.062 | 1193 | 9 | 63 | 53 | +| 4 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_1_15/fuzz_45][0.8/5/1_15/45]] | MS | 78.85903502 | 2565.887 | 1155 | 19 | 66 | 52 | +| 5 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_6_26/fuzz_78][0.8/5/6_26/78]] | MS | 73.373059156 | 4355.297 | 1179 | 11 | 65 | 51 | +| 6 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_1_12/fuzz_33][0.8/5/1_12/33]] | MS | 396.931663908 | 1695.4 | 723 | 265 | 59 | (47) | +| 7 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_6_26/fuzz_38][0.8/5/6_26/38]] | C | 32.882851417 | 289.044 | 1299 | 405 | 69 | 52 | +| 8 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_2_21/fuzz_40][0.8/5/2_21/40]] | C | 49.630976912 | 136.239 | 672 | 75 | 33 | 20 | +| 9 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_30/fuzz_18][0.8/5/4_30/18]] | C | 4.096207305 | 194.167 | 534 | 49 | 33 | 26 | +| 10 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_2_38/fuzz_100][0.8/5/2_38/100]] | C | 10.081659552 | 352.996 | 1132 | 234 | -- | 52 | +| 11 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_0/fuzz_67][0.8/5/4_0/67]] | MS | 28.596337771 | 2355.947 | 707 | 9 | 32 | 23 | +| 12 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_0_39/fuzz_27][0.8/5/0_39/27]] | MS | 121.520180855 | 5091.577 | 1153 | 9 | 60 | 47 | +| 13 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_0_0/fuzz_96][0.8/5/0_0/96]] | MS | 392.16407163 | 1611.466 | 498 | 56 | 64 | 44 | +| 14 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_2_1/fuzz_60][0.8/5/2_1/60]] | MS | 9.194543707 | 89.294 | 31 | 12 | 15 | 10 | +| 15 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_2_2/fuzz_6][0.8/5/2_2/6]] | MS | 1106.024588047 | 6825.607 | 977 | 443 | -- | 137 | +| 16 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_36/fuzz_44][0.8/5/4_36/44]] | C | 23.921349163 | 313.808 | 1261 | 191 | 60 | 43 | +| 17 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_23/fuzz_93][0.8/5/4_23/93]] | MS | 88.786871352 | 13896.513 | 1237 | 9 | 74 | 57 | +| 18 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_41/fuzz_39][0.8/5/4_41/39]] | MS | 166.899571358 | 9126.388 | 1193 | 9 | 64 | 49 | +| 19 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_1_0/fuzz_37][0.8/5/1_0/37]] | MS | 142.222511227 | 4550.126 | 1289 | 120 | 90 | 74 | +| 20 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_6_12/fuzz_61][0.8/5/6_12/61]] | MS | 73.07548465 | 5553.653 | 1016 | 9 | 43 | 32 | +| 21 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_6_39/fuzz_78][0.8/5/6_39/78]] | C | 10.894472949 | 285.255 | 996 | 99 | 54 | 42 | +| 22 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_6_19/fuzz_3][0.8/5/6_19/3]] | C | 16.908927877 | 221.02 | 883 | 115 | 63 | 44 | +| 23 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_5_2/fuzz_67][0.8/5/5_2/67]] | MS | 71.661578111 | 1096.045 | 469 | 9 | 27 | 20 | +| 24 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_18/fuzz_25][0.8/5/4_18/25]] | C | 18.313146602 | 137.885 | 411 | 32 | 60 | 40 | +| 25 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_6_43/fuzz_94][0.8/5/6_43/94]] | MS | 47.691798645 | 340.533 | 410 | 26 | 16 | 11 | +| 26 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_4_23/fuzz_34][0.8/5/4_23/34]] | C | 1.4856776 | 223.37 | 1004 | 12 | 34 | 28 | +| 27 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_0_8/fuzz_25][0.8/5/0_8/25]] | C | 30.366253517 | 213.942 | 946 | 230 | 54 | 40 | +| 28 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_1_45/fuzz_24][0.8/5/1_45/24]] | MS | 153.32273806 | 2814.821 | 730 | 87 | 68 | 58 | +| 29 | [[file:/mnt/data/projects/verismith/runs/yosys_0.8/run_5/medium_1_11/fuzz_98][0.8/5/1_11/98]] | MS | 89.500623124 | 10592.831 | 1301 | 9 | 85 | 72 | +|-----+----------------+------+----------------+-----------+------+------+----+-------------| + + VS T CR T Size VS S CR CR (unadj.) + 24.228676396 546.138 515 8 23 16 (0.0155339805825,24.228676396) (0.0446601941748,546.138) (axis cs:0.0155339805825,24.228676396)--(axis cs:0.0446601941748,546.138) + 48.020800025 549.916 675 12 40 29 (0.0177777777778,48.020800025) (0.0592592592593,549.916) (axis cs:0.0177777777778,48.020800025)--(axis cs:0.0592592592593,549.916) + 31.025951033 225.177 1063 462 -- 30 (0.434619002822,31.025951033) (0.0282220131703,225.177) (axis cs:0.434619002822,31.025951033)--(axis cs:0.0282220131703,225.177) + 239.19471845 2954.062 1193 9 63 53 (7.54400670578e-3,239.19471845) (0.0528080469405,2954.062) (7.axis cs:54400670578e-3,239.19471845)--(axis cs:0.0528080469405,2954.062) + 78.85903502 2565.887 1155 19 66 52 (0.0164502164502,78.85903502) (0.0571428571429,2565.887) (axis cs:0.0164502164502,78.85903502)--(axis cs:0.0571428571429,2565.887) + 73.373059156 4355.297 1179 11 65 51 (9.32994062765e-3,73.373059156) (0.0551314673452,4355.297) (9.axis cs:32994062765e-3,73.373059156)--(axis cs:0.0551314673452,4355.297) + 396.931663908 1695.4 723 265 59 (47) (0.36652835408,396.931663908) (0.0816044260028,1695.4) (axis cs:0.36652835408,396.931663908)--(axis cs:0.0816044260028,1695.4) + 32.882851417 289.044 1299 405 69 52 (0.311778290993,32.882851417) (0.0531177829099,289.044) (axis cs:0.311778290993,32.882851417)--(axis cs:0.0531177829099,289.044) + 49.630976912 136.239 672 75 33 20 (0.111607142857,49.630976912) (0.0491071428571,136.239) (axis cs:0.111607142857,49.630976912)--(axis cs:0.0491071428571,136.239) + 4.096207305 194.167 534 49 33 26 (0.0917602996255,4.096207305) (0.061797752809,194.167) (axis cs:0.0917602996255,4.096207305)--(axis cs:0.061797752809,194.167) + 10.081659552 352.996 1132 234 -- 52 + 28.596337771 2355.947 707 9 32 23 (0.012729844413,28.596337771) (0.045261669024,2355.947) (axis cs:0.012729844413,28.596337771)--(axis cs:0.045261669024,2355.947) + 121.520180855 5091.577 1153 9 60 47 (7.80572419775e-3,121.520180855)(0.0520381613183,5091.577) (7.axis cs:80572419775e-3,121.520180855--(axis cs:0.0520381613183,5091.577) + 392.16407163 1611.466 498 56 64 44 (0.112449799197,392.16407163) (0.128514056225,1611.466) (axis cs:0.112449799197,392.16407163)--(axis cs:0.128514056225,1611.466) + 9.194543707 89.294 31 12 15 10 (0.387096774194,9.194543707) (0.483870967742,89.294) (axis cs:0.387096774194,9.194543707)--(axis cs:0.483870967742,89.294) + 1106.024588047 6825.607 977 443 -- 137 + 23.921349163 313.808 1261 191 60 43 (0.151467089611,23.921349163) (0.0475812846947,313.808) (axis cs:0.151467089611,23.921349163)--(axis cs:0.0475812846947,313.808) + 88.786871352 13896.513 1237 9 74 57 (7.27566693614e-3,88.786871352) (0.0598221503638,13896.513) (7.axis cs:27566693614e-3,88.786871352)--(axis cs:0.0598221503638,13896.513) + 166.899571358 9126.388 1193 9 64 49 (7.54400670578e-3,166.899571358)(0.0536462699078,9126.388) (7.axis cs:54400670578e-3,166.899571358--(axis cs:0.0536462699078,9126.388) + 142.222511227 4550.126 1289 120 90 74 (0.0930954228084,142.222511227) (0.0698215671063,4550.126) (axis cs:0.0930954228084,142.222511227)--(axis cs:0.0698215671063,4550.126) + 73.07548465 5553.653 1016 9 43 32 (8.85826771654e-3,73.07548465) (0.0423228346457,5553.653) (8.axis cs:85826771654e-3,73.07548465)--(axis cs:0.0423228346457,5553.653) + 10.894472949 285.255 996 99 54 42 (0.0993975903614,10.894472949) (0.0542168674699,285.255) (axis cs:0.0993975903614,10.894472949)--(axis cs:0.0542168674699,285.255) + 16.908927877 221.02 883 115 63 44 (0.130237825595,16.908927877) (0.0713476783692,221.02) (axis cs:0.130237825595,16.908927877)--(axis cs:0.0713476783692,221.02) + 71.661578111 1096.045 469 9 27 20 (0.0191897654584,71.661578111) (0.0575692963753,1096.045) (axis cs:0.0191897654584,71.661578111)--(axis cs:0.0575692963753,1096.045) + 18.313146602 137.885 411 32 60 40 (0.0778588807786,18.313146602) (0.14598540146,137.885) (axis cs:0.0778588807786,18.313146602)--(axis cs:0.14598540146,137.885) + 47.691798645 340.533 410 26 16 11 (0.0634146341463,47.691798645) (0.0390243902439,340.533) (axis cs:0.0634146341463,47.691798645)--(axis cs:0.0390243902439,340.533) + 1.4856776 223.37 1004 12 34 28 (0.0119521912351,1.4856776) (0.0338645418327,223.37) (axis cs:0.0119521912351,1.4856776)--(axis cs:0.0338645418327,223.37) + 30.366253517 213.942 946 230 54 40 (0.243128964059,30.366253517) (0.0570824524313,213.942) (axis cs:0.243128964059,30.366253517)--(axis cs:0.0570824524313,213.942) + 153.32273806 2814.821 730 87 68 58 (0.119178082192,153.32273806) (0.0931506849315,2814.821) (axis cs:0.119178082192,153.32273806)--(axis cs:0.0931506849315,2814.821) + 89.500623124 10592.831 1301 9 85 72 (6.91775557264e-3,89.500623124) (0.065334358186,10592.831) (6.axis cs:91775557264e-3,89.500623124)--(axis cs:0.065334358186,10592.831) + +\begin{figure}[tb] + \centering + \begin{tikzpicture} + \begin{axis}[xlabel=Ratio of final size / initial size, + ylabel=Time taken for reduction (s), + xticklabel style={ + /pgf/number format/fixed, + /pgf/number format/precision=2 + }, + scaled x ticks=false, + grid=major, + major grid style={line width=.2pt,draw=black!20}, + xmin=0, + ymin=0, + ] + \addplot[only marks,bblue,draw=bblue,mark=*] + coordinates { + % (32,5.17) (53,21.13) (539,45.23) +(0.0155339805825,24.228676396) +(0.0177777777778,48.020800025) +(0.434619002822,31.025951033) +(7.54400670578e-3,239.19471845) +(0.0164502164502,78.85903502) +(9.32994062765e-3,73.373059156) +(0.36652835408,396.931663908) +(0.311778290993,32.882851417) +(0.111607142857,49.630976912) +(0.0917602996255,4.096207305) + +(0.012729844413,28.596337771) +(7.80572419775e-3,121.520180855) +(0.112449799197,392.16407163) +(0.387096774194,9.194543707) + +(0.151467089611,23.921349163) +(7.27566693614e-3,88.786871352) +(7.54400670578e-3,166.899571358) +(0.0930954228084,142.222511227) +(8.85826771654e-3,73.07548465) +(0.0993975903614,10.894472949) +(0.130237825595,16.908927877) +(0.0191897654584,71.661578111) +(0.0778588807786,18.313146602) +(0.0634146341463,47.691798645) +(0.0119521912351,1.4856776) +(0.243128964059,30.366253517) +(0.119178082192,153.32273806) +(6.91775557264e-3,89.500623124) + }; + \addplot[only marks,rred,draw=rred,mark=square*] + coordinates { + % (61,375) (56,335.5) (283,2040) +(0.0446601941748,546.138) +(0.0592592592593,549.916) +(0.0282220131703,225.177) +(0.0528080469405,2954.062) +(0.0571428571429,2565.887) +(0.0551314673452,4355.297) +(0.0816044260028,1695.4) +(0.0531177829099,289.044) +(0.0491071428571,136.239) +(0.061797752809,194.167) + +(0.045261669024,2355.947) +(0.0520381613183,5091.577) +(0.128514056225,1611.466) +(0.483870967742,89.294) + +(0.0475812846947,313.808) +(0.0598221503638,13896.513) +(0.0536462699078,9126.388) +(0.0698215671063,4550.126) +(0.0423228346457,5553.653) +(0.0542168674699,285.255) +(0.0713476783692,221.02) +(0.0575692963753,1096.045) +(0.14598540146,137.885) +(0.0390243902439,340.533) +(0.0338645418327,223.37) +(0.0570824524313,213.942) +(0.0931506849315,2814.821) +(0.065334358186,10592.831) + }; + \draw[-{Stealth},very thick] (axis cs:0.160949868074,375) -- (axis cs:0.0844327176781,5.17); + \draw[-{Stealth},very thick] (axis cs:0.0906148867314,335.5) -- (axis cs:0.0857605177994,21.13); + \draw[-{Stealth},very thick] (axis cs:0.0961930659415,2040) -- (axis cs:0.183208701564,45.23); + \draw[-{Stealth},very thick] (axis cs:0.0155339805825,24.228676396)--(axis cs:0.0446601941748,546.138) ; + \draw[-{Stealth},very thick] (axis cs:0.0177777777778,48.020800025)--(axis cs:0.0592592592593,549.916) ; + \draw[-{Stealth},very thick] (axis cs:0.434619002822,31.025951033)--(axis cs:0.0282220131703,225.177) ; + \draw[-{Stealth},very thick] (axis cs:7.54400670578e-3,239.19471845)--(axis cs:0.0528080469405,2954.062) ; + \draw[-{Stealth},very thick] (axis cs:0.0164502164502,78.85903502)--(axis cs:0.0571428571429,2565.887) ; + \draw[-{Stealth},very thick] (axis cs:9.32994062765e-3,73.373059156)--(axis cs:0.0551314673452,4355.297) ; + \draw[-{Stealth},very thick] (axis cs:0.36652835408,396.931663908)--(axis cs:0.0816044260028,1695.4) ; + \draw[-{Stealth},very thick] (axis cs:0.311778290993,32.882851417)--(axis cs:0.0531177829099,289.044) ; + \draw[-{Stealth},very thick] (axis cs:0.111607142857,49.630976912)--(axis cs:0.0491071428571,136.239) ; + \draw[-{Stealth},very thick] (axis cs:0.0917602996255,4.096207305)--(axis cs:0.061797752809,194.167) ; + \draw[-{Stealth},very thick] (axis cs:0.012729844413,28.596337771)--(axis cs:0.045261669024,2355.947) ; + \draw[-{Stealth},very thick] (axis cs:7.80572419775e-3,121.520180855--(axis cs:0.0520381613183,5091.577) ; + \draw[-{Stealth},very thick] (axis cs:0.112449799197,392.16407163)--(axis cs:0.128514056225,1611.466) ; + \draw[-{Stealth},very thick] (axis cs:0.387096774194,9.194543707)--(axis cs:0.483870967742,89.294) ; + \draw[-{Stealth},very thick] (axis cs:0.151467089611,23.921349163)--(axis cs:0.0475812846947,313.808) ; + \draw[-{Stealth},very thick] (axis cs:7.27566693614e-3,88.786871352)--(axis cs:0.0598221503638,13896.513) ; + \draw[-{Stealth},very thick] (axis cs:7.54400670578e-3,166.899571358--(axis cs:0.0536462699078,9126.388) ; + \draw[-{Stealth},very thick] (axis cs:0.0930954228084,142.222511227)--(axis cs:0.0698215671063,4550.126) ; + \draw[-{Stealth},very thick] (axis cs:8.85826771654e-3,73.07548465)--(axis cs:0.0423228346457,5553.653) ; + \draw[-{Stealth},very thick] (axis cs:0.0993975903614,10.894472949)--(axis cs:0.0542168674699,285.255) ; + \draw[-{Stealth},very thick] (axis cs:0.130237825595,16.908927877)--(axis cs:0.0713476783692,221.02) ; + \draw[-{Stealth},very thick] (axis cs:0.0191897654584,71.661578111)--(axis cs:0.0575692963753,1096.045) ; + \draw[-{Stealth},very thick] (axis cs:0.0778588807786,18.313146602)--(axis cs:0.14598540146,137.885) ; + \draw[-{Stealth},very thick] (axis cs:0.0634146341463,47.691798645)--(axis cs:0.0390243902439,340.533) ; + \draw[-{Stealth},very thick] (axis cs:0.0119521912351,1.4856776)--(axis cs:0.0338645418327,223.37) ; + \draw[-{Stealth},very thick] (axis cs:0.243128964059,30.366253517)--(axis cs:0.0570824524313,213.942) ; + \draw[-{Stealth},very thick] (axis cs:0.119178082192,153.32273806)--(axis cs:0.0931506849315,2814.821) ; + \draw[-{Stealth},very thick] (axis cs:6.91775557264e-3,89.500623124)--(axis cs:0.065334358186,10592.831) ; + \legend{\verismith{},Delta,Start} + \end{axis} + \end{tikzpicture} + \caption{Comparing the effectiveness of testcase reduction by Verismith and by + Delta. Each line represents one testcase, and each point describes the + factor by which the testcase was reduced and the time taken to do so. Points + towards the bottom left of the graph are + favoured.}\label{fig:reducer_comparison} +\end{figure} -- cgit