diff options
Diffstat (limited to 'benchmarks')
-rw-r--r-- | benchmarks/.gitignore | 6 | ||||
-rwxr-xr-x | benchmarks/count-calls | 89 | ||||
-rw-r--r-- | benchmarks/polybench-syn/benchmark-list-master | 3 | ||||
-rwxr-xr-x | benchmarks/polybench-syn/run-vericert.sh | 43 | ||||
-rwxr-xr-x | benchmarks/polybench-syn/setup-syn-vivado.sh | 25 | ||||
-rwxr-xr-x | benchmarks/polybench-syn/syn-quartus.sh (renamed from benchmarks/polybench-syn/syn-remote.sh) | 0 | ||||
-rw-r--r-- | benchmarks/polybench-syn/syn-quartus.tcl (renamed from benchmarks/polybench-syn/quartus_synth.tcl) | 0 | ||||
-rwxr-xr-x | benchmarks/polybench-syn/syn-vivado.sh | 54 | ||||
-rw-r--r-- | benchmarks/polybench-syn/syn-vivado.tcl | 6 | ||||
-rwxr-xr-x | benchmarks/run-vericert.sh | 78 |
10 files changed, 258 insertions, 46 deletions
diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 0000000..fa9cc88 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,6 @@ +*.clog +*.iver +*.comp +*.tmp +*.v +*-exec.csv diff --git a/benchmarks/count-calls b/benchmarks/count-calls new file mode 100755 index 0000000..2a14b97 --- /dev/null +++ b/benchmarks/count-calls @@ -0,0 +1,89 @@ +#!/usr/bin/env nix-shell +#! nix-shell -p "haskellPackages.ghcWithPackages (pkgs: with pkgs; [ turtle parsec text ])" -i runghc + +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE BlockArguments #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE RecordWildCards #-} + +import Turtle +import Text.Parsec as P +import qualified Data.Text as T +import Data.List (group) +import Debug.Trace + +args = argPath "benchmark" "The benchmark directory to check" + +data ParserState = ParserState { + macroNames :: [String], + functionCalls :: [String] +} + +addMacroName :: String -> ParserState -> ParserState +addMacroName macroName ParserState{macroNames, ..} = ParserState {macroNames = macroName:macroNames, ..} + +addFunctionCall :: String -> ParserState -> ParserState +addFunctionCall functionCall ParserState{functionCalls, ..} = ParserState {functionCalls = functionCall:functionCalls, ..} + +cParser :: Parsec T.Text ParserState [T.Text] +cParser = do + P.manyTill + (betweenFuncs >> parserTrace "interesting" >> (try macroDef P.<|> funcBody)) + (try endOfFile) + map T.pack . functionCalls <$> getState + where + betweenFuncs = do + traceM "betweenFuncs" + manyTill P.anyChar $ lookAhead (P.oneOf "#{") + + funcBody :: Parsec T.Text ParserState () + funcBody = do + parserTrace "funcBody" + P.char '{' + void (P.char '}') P.<|> (call >> funcBody) P.<|> (P.anyChar >> funcBody) + + macroDef :: Parsec T.Text ParserState () + macroDef = do + parserTrace "macroDef" + P.char '#' + traceM "macroDef #" + P.many (P.char ' ') + traceM "macroDef spaces" + macroName <- P.many P.alphaNum + traceM ("macroDef " ++ macroName) + modifyState (addMacroName macroName) + + endOfFile = P.many P.space >> P.eof + + call = do + identifier <- many1 idChar + inParens <- P.between (P.char '(') (P.char ')') inParensP + guard (identifier `notElem` ["for", "while"]) + modifyState (addFunctionCall identifier) + pure identifier + + -- handle balanced parens inside calls + inParensP = P.option "" do + t1 <- P.many (P.noneOf "()") + t2 <- P.option "" $ P.between (P.char '(') (P.char ')') inParensP + t3 <- P.many (P.noneOf "()") + return (t1 ++ t2 ++ t3) + idChar = label (P.alphaNum P.<|> P.oneOf "_-") "identifier" + +main = sh do + benchmarkDir <- options "count-calls" args + + printf "benchmark,totalCalls,repeatedCalls\n" + benchmarkName <- fromText . lineToText <$> input (benchmarkDir </> "benchmark-list-master") + let filePath = benchmarkDir </> benchmarkName <.> "c" + + fileContents <- strict $ input filePath + printf (fp%"\n") benchmarkName + + case runParser cParser (ParserState [] []) "file" fileContents of + Left err -> error $ show err + Right calls -> + let + callCount = length calls + repeatedCalls = calls & group & map length & sum + in printf (fp%","%d%","%d%"\n") (basename filePath) callCount repeatedCalls diff --git a/benchmarks/polybench-syn/benchmark-list-master b/benchmarks/polybench-syn/benchmark-list-master index ef0d0d0..e549997 100644 --- a/benchmarks/polybench-syn/benchmark-list-master +++ b/benchmarks/polybench-syn/benchmark-list-master @@ -1,4 +1,3 @@ -stencils/adi stencils/heat-3d stencils/fdtd-2d stencils/jacobi-1d @@ -19,9 +18,7 @@ linear-algebra/blas/gesummv linear-algebra/blas/gemm linear-algebra/blas/trmm linear-algebra/blas/syr2k -linear-algebra/solvers/cholesky linear-algebra/solvers/trisolv -linear-algebra/solvers/lu linear-algebra/solvers/ludcmp linear-algebra/solvers/durbin data-mining/covariance diff --git a/benchmarks/polybench-syn/run-vericert.sh b/benchmarks/polybench-syn/run-vericert.sh deleted file mode 100755 index 2f8097f..0000000 --- a/benchmarks/polybench-syn/run-vericert.sh +++ /dev/null @@ -1,43 +0,0 @@ -#! /bin/bash - -top=$(pwd) - #set up -while read benchmark ; do - echo "Running "$benchmark - clang -Wall -Werror -fsanitize=undefined $benchmark.c -o $benchmark.o - ./$benchmark.o > $benchmark.clog - cresult=$(cat $benchmark.clog | cut -d' ' -f2) - echo "C output: "$cresult - { time ../../bin/vericert -DSYNTHESIS -finline -fschedule --debug-hls $benchmark.c -o $benchmark.v ; } 2> $benchmark.comp - iverilog -o $benchmark.iver -- $benchmark.v - ./$benchmark.iver > $benchmark.tmp - veriresult=$(tail -1 $benchmark.tmp | cut -d' ' -f2) - cycles=$(tail -2 $benchmark.tmp | head -1 | tr -s ' ' | cut -d' ' -f2) - ctime=$(cat $benchmark.comp | head -2 | tail -1 | xargs | cut -d' ' -f2 | cut -d'm' -f2 | sed 's/s//g') - echo "Veri output: "$veriresult - - #Undefined checks - if test -z $veriresult - then - echo "FAIL: Verilog returned nothing" - #exit 0 - fi - - # Don't care checks - if [ $veriresult == "x" ] - then - echo "FAIL: Verilog returned don't cares" - #exit 0 - fi - - # unequal result check - if [ $cresult -ne $veriresult ] - then - echo "FAIL: Verilog and C output do not match!" - #exit 0 - else - echo "PASS" - fi - name=$(echo $benchmark | awk -v FS="/" '{print $NF}') - echo $name","$cycles","$ctime >> exec.csv -done < benchmark-list-master diff --git a/benchmarks/polybench-syn/setup-syn-vivado.sh b/benchmarks/polybench-syn/setup-syn-vivado.sh new file mode 100755 index 0000000..3441c0f --- /dev/null +++ b/benchmarks/polybench-syn/setup-syn-vivado.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +#set up +sshhost="$1" +basedir=${2:-"poly-syn-vivado"} + +echo "Setting up in $sshhost:$basedir" + +echo "Creating directory" +ssh -q "$sshhost" "cd ~; rm -r $basedir; mkdir $basedir" + +echo "Copying scripts over" +scp -q syn-vivado.tcl "$sshhost:$basedir" +scp -q syn-vivado.sh "$sshhost:$basedir" +rm syn-list + +while read -r benchmark; do +echo "Copying $benchmark over" +name=$(echo "$benchmark" | awk -v FS="/" '{print $NF}') +scp -q "$benchmark.v" "$sshhost:~/$basedir" +echo "$name" >> syn-list +done < benchmark-list-master + +echo "Copying syn-list" +scp -q syn-list "$sshhost:$basedir" diff --git a/benchmarks/polybench-syn/syn-remote.sh b/benchmarks/polybench-syn/syn-quartus.sh index 879db2e..879db2e 100755 --- a/benchmarks/polybench-syn/syn-remote.sh +++ b/benchmarks/polybench-syn/syn-quartus.sh diff --git a/benchmarks/polybench-syn/quartus_synth.tcl b/benchmarks/polybench-syn/syn-quartus.tcl index 6edbf0c..6edbf0c 100644 --- a/benchmarks/polybench-syn/quartus_synth.tcl +++ b/benchmarks/polybench-syn/syn-quartus.tcl diff --git a/benchmarks/polybench-syn/syn-vivado.sh b/benchmarks/polybench-syn/syn-vivado.sh new file mode 100755 index 0000000..f4d5fa7 --- /dev/null +++ b/benchmarks/polybench-syn/syn-vivado.sh @@ -0,0 +1,54 @@ +#! /bin/bash + +#setup +while read -r benchmark; do + echo "Setting up $benchmark" + rm -r "$benchmark-vivado" + mkdir "$benchmark-vivado" + cp "$benchmark.v" "$benchmark-vivado/top.v" +done < syn-list + +#synthesis +count=0 +while read -r benchmark; do + echo "Synthesising $benchmark" + + cd "$benchmark-vivado" || { + echo "$benchmark dir does not exist" + continue + } + vivado -mode batch -source ../syn-vivado.tcl + cd .. + (( count=count+1 )) + + if [ "$count" -eq 4 ]; then + echo "I am here" + wait + count=0 + fi +done < syn-list + +if [ $count -lt 4 ]; then +wait +fi + +#extract +while read -r benchmark ; do + cd "$benchmark-vivado" || { + echo "$benchmark-vivado does not exist" + continue + } + + pwd + logfile="vivado.log" + timingfile="worst_timing.txt" + + luts=$(grep "|LUT" "$logfile" | cut -d'|' -f 4 | paste -sd + | bc) + brams=$(sed -n -e "s/BRAMs: \([0-9]\+\).*$/\1/p" "$logfile") + dsps=$(sed -n -e "s/DSPs: \([0-9]\+\).*$/\1/p" "$logfile") + cells=$(grep -A4 "Report Instance Areas:" "$logfile" | tail -1 | cut -d '|' -f5 | tr -d [:space:]) + slack=$(sed -n -e 's/\s\+slack\s\+\(-\?[0-9.]\+\)/\1ns/p' "$timingfile" | tr -d [:space:]) + + echo "$benchmark,$slack,$luts,$brams,$dsps" >> results +done < syn-list + diff --git a/benchmarks/polybench-syn/syn-vivado.tcl b/benchmarks/polybench-syn/syn-vivado.tcl new file mode 100644 index 0000000..733a94e --- /dev/null +++ b/benchmarks/polybench-syn/syn-vivado.tcl @@ -0,0 +1,6 @@ +create_project -in_memory -part xc7k70t +read_verilog top.v +synth_design -part xc7k70t -top main +create_clock -name clk -period 5.000 [get_ports clk] +report_timing -nworst 1 -path_type full -input_pins -file worst_timing.txt +write_verilog -force out.v diff --git a/benchmarks/run-vericert.sh b/benchmarks/run-vericert.sh new file mode 100755 index 0000000..eb23047 --- /dev/null +++ b/benchmarks/run-vericert.sh @@ -0,0 +1,78 @@ +#! /bin/bash + +# Kill children on Ctrl-c. I have no idea how or why this works +trap 'trap " " SIGTERM; kill 0; wait;' SIGINT SIGTERM + +function error() { echo "$1" >&2; } +function crash() { echo "$1" >&2; exit 1; } +function info() { echo "$1" >&2; } + +[ $# -ge 1 ] || crash "Usage: $0 <benchmark-suite>" +benchmark_dir="$1" + +[ -f "$benchmark_dir/benchmark-list-master" ] || crash "$benchmark_dir/benchmark-list-master does not exist" + +vericert=../bin/vericert +[ -x $vericert ] || crash "Vericert executable does not exist or is not marked as executable. Did you run make; make install?" + +function run_benchmark() { + benchmark_rel="$1" + benchmark="$benchmark_dir/$benchmark_rel" + [ -f "$benchmark.c" ] || { error "$benchmark.c does not exist"; return; } + + info "[$benchmark] Running" + clang -Wall -Werror -fsanitize=undefined "$benchmark".c -o "$benchmark".o + ./"$benchmark".o > "$benchmark".clog + cresult=$(cut -d' ' -f2 "$benchmark.clog") + info "[$benchmark] C output: $cresult" + { time ../bin/vericert -DSYNTHESIS --debug-hls "$benchmark.c" -o "$benchmark.v" ; vericert_result=$? ; } 2> "$benchmark".comp + + iverilog -o "$benchmark".iver -- "$benchmark".v + iverilog_result=$? + + timeout 10m ./"$benchmark".iver > "$benchmark".tmp + if [ $? -eq 124 ]; then + timeout=1 + else + veriresult="$(tail -1 "$benchmark".tmp | cut -d' ' -f2)" + fi + cycles="$(tail -4 "$benchmark".tmp | head -1 | tr -s ' ' | cut -d' ' -f3)" + ctime="$(head -2 "$benchmark".comp | tail -1 | xargs | cut -d' ' -f2 | cut -d'm' -f2 | sed 's/s//g')" + info "[$benchmark] Veri output: $veriresult" + + + if [ -n "$timeout" ]; then + info "[$benchmark] FAIL: Verilog timed out" + result="timeout" + elif [ "$vericert_result" -ne 0 ]; then + #Undefined + info "[$benchmark] FAIL: Vericert failed" + result="compile error" + elif [ "$iverilog_result" -ne 0 ]; then + #Undefined + info "[$benchmark] FAIL: iverilog failed" + result="elaboration error" + elif [ -z "$veriresult" ]; then + #Undefined + info "[$benchmark] FAIL: Verilog returned nothing" + result="timeout" + elif [ "$veriresult" == "x" ]; then + # Don't care + info "[$benchmark] FAIL: Verilog returned don't cares" + result="dontcare" + elif [ "$cresult" -ne "$veriresult" ]; then + # unequal result + info "[$benchmark] FAIL: Verilog and C output do not match!" + result="incorrect result" + else + info "[$benchmark] PASS" + result="pass" + fi + name=$(echo "$benchmark" | awk -v FS="/" '{print $NF}') + echo "$name,$cycles,$ctime,$result" +} + +while read -r benchmark_rel; do + run_benchmark "$benchmark_rel" >> "$benchmark_dir-exec.csv" & +done < "$benchmark_dir/benchmark-list-master" +wait |