Update the artifact description

7 files changed, 204 insertions, 4 deletions

diff --git a/scripts/convert.sh b/scripts/convert.sh
new file mode 100755
index 0000000..c2ef311
--- /dev/null
+++ b/scripts/convert.sh
@@ -0,0 +1,14 @@
+file=$1
+benchmark=$(echo $1 | sed 's:.*/\([^/]\+\)/encode_report.xml:\1:')
+
+lut_flip_flop=$(sed -n "s/.*XILINX_LUT_FLIP_FLOP_PAIRS_USED.*\"\([0-9.]*\)\".*/\1/p" $file)
+slice=$(sed -n "s/.*XILINX_SLICE\".*\"\([0-9.]*\)\".*/\1/p" $file)
+regs=$(sed -n "s/.*XILINX_SLICE_REGISTERS.*\"\([0-9.]*\)\".*/\1/p" $file)
+luts=$(sed -n "s/.*XILINX_SLICE_LUTS.*\"\([0-9.]*\)\".*/\1/p" $file)
+ramfifo=$(sed -n "s/.*XILINX_BLOCK_RAMFIFO.*\"\([0-9.]*\)\".*/\1/p" $file)
+iopin=$(sed -n "s/.*XILINX_IOPIN.*\"\([0-9.]*\)\".*/\1/p" $file)
+dsps=$(sed -n "s/.*XILINX_DSPS.*\"\([0-9.]*\)\".*/\1/p" $file)
+power=$(sed -n "s/.*XILINX_POWER.*\"\([0-9.]*\)\".*/\1/p" $file)
+delay=$(sed -n "s/.*XILINX_DESIGN_DELAY.*\"\([0-9.]*\)\".*/\1/p" $file)
+
+echo $benchmark,$lut_flip_flop,$slice,$regs,$luts,$ramfifo,$iopin,$dsps,$power,$delay >>synth.csv
diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile
index 169971c..668219a 100644
--- a/scripts/docker/Dockerfile
+++ b/scripts/docker/Dockerfile
@@ -7,12 +7,13 @@ RUN nix-env -i yosys git tmux vim gcc iverilog
 
 ADD legup_polybench_syn.tar.gz /data/legup-polybench-syn
 ADD legup_polybench_syn_div.tar.gz /data/legup-polybench-syn-div
+ADD data.tar.gz /data
 
 RUN git clone --recursive https://github.com/ymherklotz/vericert
 
 WORKDIR /vericert
 RUN git checkout -b oopsla21
-RUN nix-shell --run "make -j7"
-RUN nix-shell --run "make install"
-
+#RUN nix-shell --run "make -j7"
+#RUN nix-shell --run "make install"
+#
 RUN echo "export PATH=/vericert/bin:$PATH" >>/root/.bashrc
diff --git a/scripts/docker/artifact.org b/scripts/docker/artifact.org
index 13d4b0a..c728320 100644
--- a/scripts/docker/artifact.org
+++ b/scripts/docker/artifact.org
@@ -133,11 +133,77 @@ Unfortunately, the benchmarks cannot be compiled from C to Verilog using LegUp,
 
 However, our compiled Verilog designs from LegUp have been included for all the optimisation options that were tested for in the paper in Section 5.
 
+To get the cycle counts, it suffices to go into an arbitrary directory, and run the following script, where the command line arguments select which set of cycle counts to generate:
+
+#+begin_src shell
+/vericert/scripts/run-legup.sh [syn|syn-div] \
+                               [opt|no_opt|no_chain|no_opt_no_chain]
+#+end_src
+
+For example, to run the LegUp benchmarks with no LLVM optimisations and no operation chaining, on the PolyBench/C benchmark with no dividers, one can run the following command:
+
+#+begin_src shell
+/vericert/scripts/run-legup.sh syn no_opt_no_chain
+#+end_src
+
+This will take some 30 minutes to run as well, and will generate an ~exec_legup.csv~ file, with the name of the benchmark and it's cycle count.
+
+** Comparing the results
+
+To compare the results to the results presented in the paper, the main comparison that is supported by this artifact is to compare the cycle counts to the ones used to generate the graphs in the evaluation section of the paper.
+
+The ~/data/data~ directory contains all the raw data which was used to generate the graphs in Section 5.  This data can therefore be used to examine the cycle counts used to draw the graphs.  This raw data can be examined better in ~/data/data/results.org~, which includes the tables in a nicer format.
+
+The ~legup-*~ csv files contain the raw size, timing and cycle count for the various LegUp configurations on the different benchmarks.  ~vericert-*~ is the equivalent but for Vericert.  Then, to draw the graphs, the actual csv files that are used are:
+
+- ~rel-size-*~ :: This contains the relative size of each run (denoted by slice in the csv files) compared to fully optimised LegUp.  This is obtained by taking the slice value of the tool being considered (LegUp with some optimisation turned off, or Vericert), and dividing that by the number of slices present in fully optimised LegUp.
+
+\[\frac{\text{slice}_t}{\text{slice}_{\text{legup\_opt}}}\]
+
+- ~rel-time-*~ :: This performs the same computation as for the size comparison, comparing to LegUp with all optimisations turned on, but instead compares the following values: cycles $\times$ delay:
+
+\[\frac{\text{cycles}_t \times \text{delay}_t}{\text{cycles}_{\text{legup\_opt}} \times \text{delay}_{\text{legup\_opt}}}\]
+
+Where $t$ is the tool being considered.
+
+*** Compiling the graph
+
+A tex file is included in the ~/data/data~ directory, which unfortunately can only be compiled outside of the docker file, but will recreate the graphs from the paper using the csv files in the directory.  This can be achieved using the following commands:
+
+#+begin_src shell
+docker create ymherklotz/vericert:v1.0 # returns container ID
+docker cp $container_id:/data/data .
+docker rm $container_id
+cd data
+pdflatex graphs
+#+end_src
+
+** Running with Vivado
+
+Finally, for the adventurous that downloaded Vivado, there are some short instructions for running it on single examples.  Running synthesis on a benchmark will normally take around 20 minutes to an hour depending on the benchmark, so it might take a long time to complete.
+
+First, create a new directory and copy the synthesis script into it, as well as the Verilog file that should be synthesisd.  For example, once ~make~ was run in the benchmarks folder, one of the benchmarks can be selected for Vericert, such as ~jacobi-1d~:
+
+#+begin_src shell
+mkdir synthesis
+cd synthesis
+cp /vericert/scripts/synth.tcl .
+cp /vericert/benchmarks/polybench-syn/stencils/jacobi-1d.v main.v
+#+end_src
+
+Then Vivado can be run in batch mode in that directory to generate the report:
+
+#+begin_src shell
+vivado -mode batch -source synth.tcl
+#+end_src
+
+Once this completes, the important results of the synthesis should be available in ~encode_report.xml~, where each field will also be present in the relevant CSV file, which is this case is ~/data/data/vericert-nodiv.csv~.
+
 ** Rebuilding the Docker image
 
 The docker image can be completely rebuilt from scratch as well, by using the Dockerfile that is located in the Vericert repository at ~/vericert/scripts/docker/Dockerfile~, which also contains this document.
 
-To rebuild the docker image, one first needs to download the legup results for the benchmarks without divider[fn:4] and with divider[fn:5], and the tar files should be placed in the same directory as the ~Dockerfile~.  Then, in the ~docker~ directory, the following will build the docker image, which might take around 20 minutes:
+To rebuild the docker image, one first needs to download the LegUp results for the benchmarks without divider[fn:4] and with divider[fn:5], as well as the csv folder with all the raw results[fn:6]. The tar files should be placed into the same directory as the ~Dockerfile~.  Then, in the ~docker~ directory, the following will build the docker image, which might take around 20 minutes:
 
 #+begin_src shell
 docker build .
@@ -151,6 +217,8 @@ docker run -it <hash> sh
 
 * Footnotes
 
+
+[fn:6] https://imperialcollegelondon.box.com/s/nqoaquk7j5mj70db16s6bdbhg44zjn52
 [fn:5] https://imperialcollegelondon.box.com/s/94clcbjowla3987opf3icjz087ozoi1o
 [fn:4] https://imperialcollegelondon.box.com/s/ril1utuk2n88fhoq3375oxiqcgw42b8a
 [fn:3] https://www.xilinx.com/support/download.html
diff --git a/scripts/docker/artifact.pdf b/scripts/docker/artifact.pdf
index a22e4cc..8118033 100644
--- a/scripts/docker/artifact.pdf
+++ b/scripts/docker/artifact.pdf
diff --git a/benchmarks/polybench-syn/run-legup.sh b/scripts/run-legup.sh
index 3b6f60f..3b6f60f 100644
--- a/benchmarks/polybench-syn/run-legup.sh
+++ b/scripts/run-legup.sh
diff --git a/scripts/run-vivado.sh b/scripts/run-vivado.sh
new file mode 100755
index 0000000..117054d
--- /dev/null
+++ b/scripts/run-vivado.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+benchmark=./$1/$2
+echo $benchmark
+
+    cp ./synth.tcl $benchmark/. 2>/dev/null
+    cd $benchmark || exit 1
+    vivado -mode batch -source synth.tcl >vivado.log 2>&1
diff --git a/scripts/synth.tcl b/scripts/synth.tcl
new file mode 100644
index 0000000..2b032be
--- /dev/null
+++ b/scripts/synth.tcl
@@ -0,0 +1,109 @@
+proc dump_statistics {  } {
+  set util_rpt [report_utilization -return_string]
+  set LUTFFPairs 0
+  set SliceRegisters 0
+  set Slice 0
+  set SliceLUTs 0
+  set SliceLUTs1 0
+  set BRAMFIFO36 0
+  set BRAMFIFO18 0
+  set BRAMFIFO36_star 0
+  set BRAMFIFO18_star 0
+  set BRAM18 0
+  set BRAMFIFO 0
+  set BIOB 0
+  set DSPs 0
+  set TotPower 0
+  set design_slack 0
+  set design_req 0
+  set design_delay 0
+  regexp --  {\s*LUT Flip Flop Pairs\s*\|\s*([^[:blank:]]+)} $util_rpt ignore LUTFFPairs
+  regexp --  {\s*Slice Registers\s*\|\s*([^[:blank:]]+)} $util_rpt ignore SliceRegisters
+  regexp --  {\s*Slice\s*\|\s*([^[:blank:]]+)} $util_rpt ignore Slice
+  regexp --  {\s*Slice LUTs\s*\|\s*([^[:blank:]]+)} $util_rpt ignore SliceLUTs
+  regexp --  {\s*Slice LUTs\*\s*\|\s*([^[:blank:]]+)} $util_rpt ignore SliceLUTs1
+  if { [expr {$LUTFFPairs == 0}] } {
+    set LUTFFPairs $SliceLUTs1
+    puts $SliceLUTs1
+  }
+  if { [expr {$SliceLUTs == 0}] } {
+    set SliceLUTs $SliceLUTs1
+  }
+  regexp --  {\s*RAMB36/FIFO36\s*\|\s*([^[:blank:]]+)} $util_rpt ignore BRAMFIFO36
+  regexp --  {\s*RAMB18/FIFO18\s*\|\s*([^[:blank:]]+)} $util_rpt ignore BRAMFIFO18
+  regexp --  {\s*RAMB36/FIFO\*\s*\|\s*([^[:blank:]]+)} $util_rpt ignore BRAMFIFO36_star
+  regexp --  {\s*RAMB18/FIFO\*\s*\|\s*([^[:blank:]]+)} $util_rpt ignore BRAMFIFO18_star
+  regexp --  {\s*RAMB18\s*\|\s*([^[:blank:]]+)} $util_rpt ignore BRAM18
+  set BRAMFIFO [expr {(2 *$BRAMFIFO36) + $BRAMFIFO18 + (2*$BRAMFIFO36_star) + $BRAMFIFO18_star + $BRAM18}]
+  regexp --  {\s*Bonded IOB\s*\|\s*([^[:blank:]]+)} $util_rpt ignore BIOB
+  regexp --  {\s*DSPs\s*\|\s*([^[:blank:]]+)} $util_rpt ignore DSPs
+  set power_rpt [report_power -return_string]
+  regexp --  {\s*Total On-Chip Power \(W\)\s*\|\s*([^[:blank:]]+)} $power_rpt ignore TotPower
+  set Timing_Paths [get_timing_paths -max_paths 1 -nworst 1 -setup]
+  if { [expr {$Timing_Paths == ""}] } {
+    set design_slack 0
+    set design_req 0
+  } else {
+    set design_slack [get_property SLACK $Timing_Paths]
+    set design_req [get_property REQUIREMENT  $Timing_Paths]
+  }
+  if { [expr {$design_slack == ""}] } {
+    set design_slack 0
+  }
+  if { [expr {$design_req == ""}] } {
+    set design_req 0
+  }
+  set design_delay [expr {$design_req - $design_slack}]
+  file delete -force encode_report.xml
+  set ofile_report [open encode_report.xml w]
+  puts $ofile_report "<?xml version=\"1.0\"?>"
+  puts $ofile_report "<document>"
+  puts $ofile_report "  <application>"
+  puts $ofile_report "    <section stringID=\"XILINX_SYNTHESIS_SUMMARY\">"
+  puts $ofile_report "      <item stringID=\"XILINX_LUT_FLIP_FLOP_PAIRS_USED\" value=\"$LUTFFPairs\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_SLICE\" value=\"$Slice\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_SLICE_REGISTERS\" value=\"$SliceRegisters\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_SLICE_LUTS\" value=\"$SliceLUTs\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_BLOCK_RAMFIFO\" value=\"$BRAMFIFO\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_IOPIN\" value=\"$BIOB\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_DSPS\" value=\"$DSPs\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_POWER\" value=\"$TotPower\"/>"
+  puts $ofile_report "      <item stringID=\"XILINX_DESIGN_DELAY\" value=\"$design_delay\"/>"
+  puts $ofile_report "    </section>"
+  puts $ofile_report "  </application>"
+  puts $ofile_report "</document>"
+  close $ofile_report
+}; #END PROC
+set outputDir .
+create_project -in_memory -part xc7z020clg484-1 -force
+read_verilog main.v
+synth_design -mode out_of_context -no_iobuf -top top -part xc7z020clg484-1
+write_checkpoint -force $outputDir/post_synth.dcp
+report_timing_summary -file $outputDir/post_synth_timing_summary.rpt
+report_utilization -file $outputDir/post_synth_util.rpt
+create_clock -name clk -period 10.000 [get_ports clk]
+dump_statistics
+opt_design
+dump_statistics
+report_utilization -file $outputDir/post_opt_design_util.rpt
+place_design -directive Explore
+report_clock_utilization -file $outputDir/clock_util.rpt
+# Optionally run optimization if there are timing violations after placement
+if {[get_property SLACK [get_timing_paths -max_paths 1 -nworst 1 -setup]] < 0.5} {
+  puts "Found setup timing violations => running physical optimization"
+  phys_opt_design
+}
+write_checkpoint -force $outputDir/post_place.dcp
+report_utilization -file $outputDir/post_place_util.rpt
+report_timing_summary -file $outputDir/post_place_timing_summary.rpt
+dump_statistics
+route_design -directive Explore
+write_checkpoint -force $outputDir/post_route.dcp
+report_route_status -file $outputDir/post_route_status.rpt
+report_timing_summary -file $outputDir/post_route_timing_summary.rpt
+report_power -file $outputDir/post_route_power.rpt
+report_drc -file $outputDir/post_imp_drc.rpt
+report_utilization -file $outputDir/post_route_util.rpt
+dump_statistics
+close_design
+close_project