From eac948b89034474b601d45ded400083d6e55bb22 Mon Sep 17 00:00:00 2001 From: Yann Herklotz Date: Tue, 23 Feb 2021 16:09:32 +0000 Subject: Add performance graph with CSV data --- main.tex | 44 ++++++++++++++++++++++++++++++++++++++++++++ results/comparison.csv | 26 ++++++++++++++++++++++++++ results/comparison.numbers | Bin 0 -> 209098 bytes results/coqup.csv | 28 ++++++++++++++++++++++++++++ results/legup.csv | 28 ++++++++++++++++++++++++++++ results/poly.csv | 28 ++++++++++++++++++++++++++++ 6 files changed, 154 insertions(+) create mode 100644 results/comparison.csv create mode 100644 results/comparison.numbers create mode 100644 results/coqup.csv create mode 100644 results/legup.csv create mode 100644 results/poly.csv diff --git a/main.tex b/main.tex index 95a6b7b..2e05268 100644 --- a/main.tex +++ b/main.tex @@ -218,6 +218,50 @@ However, there are also assumptions that have to be made about the tool. In the Finally, it is interesting to compare the performance of Vericert compared to existing HLS tools, to see how correctness guarantees affect the size and speed of the final hardware. This was done on a common compiler benchmark called PolyBench/C~\cite{polybench}. +\begin{figure} +\definecolor{polycol}{HTML}{e6ab02} +\definecolor{polywocol}{HTML}{7570b3} +\begin{tikzpicture} +\begin{axis}[ + xmode=log, + ymode=log, + height=80mm, + width=80mm, + xlabel={\legup{} execution time (ms)}, + ylabel={\vericert{} execution time (ms)}, + xmin=10, + xmax=1000000, + ymax=1000000, + ymin=10, + legend pos=south east, + %log ticks with fixed point, + ] + + +\addplot[draw=none, mark=*, draw opacity=0, fill opacity=0.8, polycol] + table [x expr={\thisrow{legupcycles}/\thisrow{legupfreqMHz}}, y expr={\thisrow{vericertcycles}/\thisrow{vericertoldfreqMHz}}, col sep=comma] + {results/poly.csv}; + +\addlegendentry{PolyBench} + +\addplot[draw=none, mark=o, fill opacity=0, polywocol] + table [x expr={\thisrow{legupcycles}/\thisrow{legupfreqMHz}}, y expr={\thisrow{vericertcycles}/\thisrow{vericertfreqMHz}}, col sep=comma] + {results/poly.csv}; + +\addlegendentry{PolyBench w/o division} + +\addplot[dotted, domain=10:1000000]{x}; +%\addplot[dashed, domain=10:10000]{9.02*x + 442}; + +\end{axis} +\end{tikzpicture} +\caption{A comparison of the execution time of hardware designs generated by \vericert{} and by \legup{}. \YH{Need to update performance figures.}}\label{fig:comparison_time} +\end{figure} + +\section{Improvements to \vericert{}} + + + %% Acknowledgments %\begin{acks}%% acks environment is optional %% contents suppressed with 'anonymous' diff --git a/results/comparison.csv b/results/comparison.csv new file mode 100644 index 0000000..bb990a3 --- /dev/null +++ b/results/comparison.csv @@ -0,0 +1,26 @@ +benchmark,vericertcycles,vericertfreqMHz,vericertlogicutilisation,vericertregs,vericertrams,vericertdsps,vericertwallclockMS,vericertutilisation,vericertcomptime,legupcycles,legupfreqMHz,leguplogicutilisation,legupregs,leguprams,legupdsps,legupwallclockMS,leguputilisation,legupcomptime +durbin,21581,18.23,2941,4750,0,8,1.18,0.69,0.20,15106,188.61,2509,4008,0,8,0.08,0.59,4.77 +lu,2685403,19.21,54188,104005,0,6,139.79,12.68,0.20,482766,244.62,3116,4593,0,10,1.97,0.73,4.72 +ludcmp,2460807,18.78,58022,108157,0,10,131.03,13.58,0.20,470843,249.69,3605,5397,0,15,1.89,0.84,4.87 +trisolv,24501,19.1,28792,54716,0,4,1.28,6.74,0.20,35382,213.9,2412,3749,0,3,0.17,0.56,4.73 +2mm,381553,20.21,130339,75976,0,19,18.88,30.51,0.20,60088,226.5,1114,1936,0,7,0.27,0.26,4.80 +3mm,498115,19.74,48152,87308,0,18,25.23,11.27,0.20,204195,188.96,4210,4801,0,43,1.08,0.99,4.88 +atas,54037,21.75,29525,55787,0,6,2.48,6.91,0.20,126288,193.24,3026,3719,0,10,0.65,0.71,4.80 +bicg,47877,20.91,31721,57142,0,8,2.29,7.43,0.20,11907,303.4,308,537,0,6,0.04,0.07,4.78 +mvt,65779,20.71,33574,57552,0,6,3.18,7.86,0.20,16806,384.47,372,597,0,4,0.04,0.09,4.79 +doitgen,333951,20.14,20416,37147,0,4,16.58,4.78,0.20,57199,252.14,909,1402,0,2,0.23,0.21,5.05 +symm,218021,20.94,29518,52848,0,14,10.41,6.91,0.20,64903,284.41,2155,3170,0,10,0.23,0.50,4.63 +syrk,256315,20.81,71000,56199,0,8,12.32,16.62,0.20,57395,278.01,598,976,0,2,0.21,0.14,4.73 +syr2k,402419,20.82,116382,78402,0,12,19.33,27.24,0.20,125705,240.85,3149,3679,0,6,0.52,0.74,4.85 +trmm,134887,20.3,30757,33192,0,4,6.64,7.20,0.20,41432,281.61,610,990,0,4,0.15,0.14,4.71 +gemm,302137,18.64,33531,60552,0,16,16.21,7.85,0.20,83676,192.68,1029,1544,0,35,0.43,0.24,4.79 +gemver,106533,18.9,35603,63113,0,14,5.64,8.33,0.20,28087,303.49,1854,2380,0,8,0.09,0.43,4.68 +gesummv,34257,20.67,33930,61677,0,10,1.66,7.94,0.20,6634,310.46,298,504,0,4,0.02,0.07,4.77 +covariance,262265,18.73,30052,56122,0,4,14.00,7.03,0.20,109992,245.16,2098,3096,0,5,0.45,0.49,4.77 +fdtd-2d,826507,20.27,32133,60497,0,6,40.77,7.52,0.20,214153,262.61,2736,3801,0,2,0.82,0.64,4.73 +heat-3d,544497,20.38,34572,67057,0,13,26.72,8.09,0.20,41059,115.54,3132,2910,0,60,0.36,0.73,4.92 +jacobi-1d,17481,21.07,2478,2989,0,0,0.83,0.58,0.20,6914,386.25,1355,1885,0,0,0.02,0.32,4.72 +jacobi-2d,314377,20.75,31101,59410,0,4,15.15,7.28,0.20,84609,240.79,2347,3185,0,2,0.35,0.55,4.81 +seidel-2d,609259,18.8,27638,52840,0,2,32.41,6.47,0.20,345294,232.4,2128,3337,0,2,1.49,0.50,4.68 +floyd-warshall,4556081,17.2,60713,116401,0,2,264.89,14.21,0.20,1238764,235.52,1869,2367,0,2,5.26,0.44,4.71 +nussinov,881801,87.67,60649,119210,0,0,10.06,14.20,0.20,216467,273.07,1078,1431,0,2,0.79,0.25,4.79 diff --git a/results/comparison.numbers b/results/comparison.numbers new file mode 100644 index 0000000..b96e704 Binary files /dev/null and b/results/comparison.numbers differ diff --git a/results/coqup.csv b/results/coqup.csv new file mode 100644 index 0000000..9b5f075 --- /dev/null +++ b/results/coqup.csv @@ -0,0 +1,28 @@ +benchmark,cycles,freq,luts,regs,rams,dsps +durbin,22554,193.69,3068,6229,0,8 +lu,2742482,91.19,54479,106996,0,6 +ludcmp,2509280,86.14,57634,112172,0,10 +trisolv,38924,106.51,28047,55053,0,4 +2mm,452506,111.17,33699,64977,0,18 +3mm,574780,93.13,46566,91256,0,18 +atas,105254,124.29,28898,56986,0,6 +bicg,133052,121.37,30715,60040,0,8 +mvt,65778,21.03,33619,57551,0,6 +doitgen,374264,136.33,19818,39021,0,4 +symm,265730,111.45,28526,55697,0,14 +syrk,330284,127.24,26154,51229,0,8 +syr2k,402418,20.69,37644,68915,0,12 +trmm,134886,20.52,18563,33213,0,4 +gemm,302136,18.28,111149,73087,0,18 +gemver,190700,110.9,32883,64506,0,14 +gesummv,34256,20.85,33924,61723,0,10 +covariance,332990,100.73,28682,56738,0,4 +fdtd-2d,889660,101.61,32106,62471,0,6 +heat-3d,572174,103.75,34233,67998,0,21 +jacobi-1d,17980,262.12,1652,3261,0,0 +jacobi-2d,385212,115.1,30426,59844,0,4 +seidel-2d,979334,128.88,26988,53293,0,2 +floyd-warshall,5038446,96.62,60609,119811,0,2 +nussinov,882120,73.22,60238,119719,0,0 +cholesky,2347850,90.88,53992,106180,0,8 +adi,1713702,0.160,110.44,31207,60639,0,36 \ No newline at end of file diff --git a/results/legup.csv b/results/legup.csv new file mode 100644 index 0000000..39ed1e9 --- /dev/null +++ b/results/legup.csv @@ -0,0 +1,28 @@ +benchmark,cycles,freq,luts,regs,rams,dsps,comptime,noopt +durbin,15106,188.61,2509,4008,0,8,4.77 +lu,482766,244.62,3116,4593,0,10,4.72 +ludcmp,470843,249.69,3605,5397,0,15,4.87 +trisolv,35382,213.9,2412,3749,0,3,4.73 +2mm,60088,226.5,1114,1936,0,7,4.80 +3mm,204195,188.96,4210,4801,0,43,4.88 +atas,126288,193.24,3026,3719,0,10,4.80 +bicg,11907,303.4,308,537,0,6,4.78,44523 +mvt,16806,384.47,372,597,0,4,4.79 +doitgen,57199,252.14,909,1402,0,2,5.05 +symm,64903,284.41,2155,3170,0,10,4.63 +syrk,57395,278.01,598,976,0,2,4.73 +syr2k,125705,240.85,3149,3679,0,6,4.85 +trmm,41432,281.61,610,990,0,4,4.71 +gemm,83676,192.68,1029,1544,0,35,4.79 +gemver,28087,303.49,1854,2380,0,8,4.68 +gesummv,6634,310.46,298,504,0,4,4.77 +covariance,109992,245.16,2098,3096,0,5,4.77 +fdtd-2d,214153,262.61,2736,3801,0,2,4.73 +heat-3d,41059,115.54,3132,2910,0,60,4.92 +jacobi-1d,6914,386.25,1355,1885,0,0,4.72 +jacobi-2d,84609,240.79,2347,3185,0,2,4.81 +seidel-2d,345294,232.4,2128,3337,0,2,4.68 +floyd-warshall,1238764,235.52,1869,2367,0,2,4.71 +nussinov,216467,273.07,1078,1431,0,2,4.79 +adi,90644,256.28,1111,2391,0,5.32 +cholesky,148443,273.6,1528,3419,15,4.89 diff --git a/results/poly.csv b/results/poly.csv new file mode 100644 index 0000000..f02c58f --- /dev/null +++ b/results/poly.csv @@ -0,0 +1,28 @@ +benchmark,legupcycles,legupfreqMHz,legupluts,legupregs,leguprams,legupdsps,legupcomptime,vericertcycles,vericertfreqMHz,vericertluts,vericertregs,vericertrams,vericertdsps,vericertcomptime,vericertoldfreqMHz +durbin,15106,188.61,2509,4008,0,8,4.77,19852,199.2,3027,6168,0,8,0.077,18.23 +lu,482766,244.62,3116,4593,0,10,4.72,2634766,92.97,54806,106037,0,6,0.097,19.21 +ludcmp,470843,249.69,3605,5397,0,15,4.87,2401354,83.52,57145,111408,0,10,0.132,18.78 +trisolv,35382,213.9,2412,3749,0,3,4.73,33550,112.59,28101,55109,0,2,0.086,19.1 +2mm,60088,226.5,1114,1936,0,7,4.80,427098,116.9,32600,63212,0,18,0.136,20.21 +3mm,204195,188.96,4210,4801,0,43,4.88,539430,97.13,45073,89719,0,18,0.147,19.74 +atas,126288,193.24,3026,3719,0,10,4.80,92000,130.41,28603,56646,0,6,0.069,21.75 +bicg,11907,303.4,308,537,0,6,4.78,121134,122.74,30091,58799,0,8,0.097,20.91 +mvt,16806,384.47,372,597,0,4,4.79,139194,119.93,30753,60450,0,6,0.130,20.71 +doitgen,57199,252.14,909,1402,0,2,5.05,350302,126.18,19898,38461,0,4,0.105,20.14 +symm,64903,284.41,2155,3170,0,10,4.63,248930,113.92,27693,54514,0,14,0.115,20.94 +syrk,57395,278.01,598,976,0,2,4.73,309018,87.15,76889,57816,0,8,0.094,20.81 +syr2k,125705,240.85,3149,3679,0,6,4.85,478040,78.6,120116,82032,0,12,0.116,20.82 +trmm,41432,281.61,610,990,0,4,4.71,147528,105.61,64031,40502,0,4,0.089,20.3 +gemm,83676,192.68,1029,1544,0,35,4.79,360772,121.61,31853,62126,0,16,0.104,18.64 +gemver,28087,303.49,1854,2380,0,8,4.68,175326,107.27,32615,64118,0,14,0.099,18.9 +gesummv,6634,310.46,298,504,0,4,4.77,111094,79.97,113876,72908,0,10,0.101,20.67 +covariance,109992,245.16,2098,3096,0,5,4.77,297450,110.57,28729,56660,0,4,0.083,18.73 +fdtd-2d,214153,262.61,2736,3801,0,2,4.73,831912,108.23,31333,61421,0,6,0.116,20.27 +heat-3d,41059,115.54,3132,2910,0,60,4.92,555930,110.42,33915,67273,0,9,0.181,20.38 +jacobi-1d,6914,386.25,1355,1885,0,0,4.72,16606,277.93,1636,3305,0,0,0.071,21.07 +jacobi-2d,84609,240.79,2347,3185,0,2,4.81,357100,113.53,30393,59782,0,4,0.079,20.75 +seidel-2d,345294,232.4,2128,3337,0,2,4.68,875758,127.99,26948,53133,0,2,0.091,18.8 +floyd-warshall,1238764,235.52,1869,2367,0,2,4.71,4762766,109.4,59859,118101,0,2,0.094,17.2 +nussinov,216467,273.07,1078,1431,0,2,4.79,837958,90.73,60663,119303,0,0,0.080,20.38 +cholesky,148443,273.6,1528,3419,0,15,4.89,2347850,90.88,53992,106180,0,8,0.100,21.07 +adi,90644,256.28,1111,2391,0,0,5.32,1713702,110.44,31207,60639,0,36,0.160,20.75 \ No newline at end of file -- cgit