1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
|
{-|
Module : VeriFuzz.Fuzz
Description : Environment to run the simulator and synthesisers in a matrix.
Copyright : (c) 2019, Yann Herklotz
License : GPL-3
Maintainer : ymherklotz [at] gmail [dot] com
Stability : experimental
Portability : POSIX
Environment to run the simulator and synthesisers in a matrix.
-}
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE TemplateHaskell #-}
module VeriFuzz.Fuzz
( Fuzz
, fuzz
, fuzzInDir
, fuzzMultiple
, runFuzz
, sampleSeed
-- * Helpers
, make
, pop
)
where
import Control.DeepSeq (force)
import Control.Exception.Lifted (finally)
import Control.Lens hiding ((<.>))
import Control.Monad (forM, replicateM)
import Control.Monad.IO.Class
import Control.Monad.Trans.Class (lift)
import Control.Monad.Trans.Control (MonadBaseControl)
import Control.Monad.Trans.Maybe (runMaybeT)
import Control.Monad.Trans.Reader hiding (local)
import Control.Monad.Trans.State.Strict
import Data.List (nubBy, sort)
import Data.Maybe (isNothing)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Time
import Data.Tuple (swap)
import Hedgehog (Gen)
import qualified Hedgehog.Internal.Gen as Hog
import Hedgehog.Internal.Seed (Seed)
import qualified Hedgehog.Internal.Seed as Hog
import qualified Hedgehog.Internal.Tree as Hog
import Prelude hiding (FilePath)
import Shelly hiding (get)
import Shelly.Lifted (MonadSh, liftSh)
import System.FilePath.Posix (takeBaseName)
import VeriFuzz.Config
import VeriFuzz.Internal
import VeriFuzz.Reduce
import VeriFuzz.Report
import VeriFuzz.Result
import VeriFuzz.Sim.Internal
import VeriFuzz.Sim.Yosys
import VeriFuzz.Verilog.AST
import VeriFuzz.Verilog.CodeGen
data FuzzEnv = FuzzEnv { getSynthesisers :: ![SynthTool]
, getSimulators :: ![SimTool]
, yosysInstance :: {-# UNPACK #-} !Yosys
}
deriving (Eq, Show)
data FuzzState = FuzzState { _fuzzSynthResults :: ![SynthResult]
, _fuzzSimResults :: ![SimResult]
, _fuzzSynthStatus :: ![SynthStatus]
}
deriving (Eq, Show)
$(makeLenses ''FuzzState)
type Frequency a = [(Seed, a)] -> [(Int, Gen (Seed, a))]
-- | The main type for the fuzzing, which contains an environment that can be
-- read from and the current state of all the results.
type Fuzz m = StateT FuzzState (ReaderT FuzzEnv m)
type MonadFuzz m = (MonadBaseControl IO m, MonadIO m, MonadSh m)
runFuzz :: MonadIO m => Config -> Yosys -> (Config -> Fuzz Sh a) -> m a
runFuzz conf yos m = shelly $ runFuzz' conf yos m
runFuzz' :: Monad m => Config -> Yosys -> (Config -> Fuzz m b) -> m b
runFuzz' conf yos m = runReaderT
(evalStateT (m conf) (FuzzState [] [] []))
(FuzzEnv
( force
$ defaultIdentitySynth
: (descriptionToSynth <$> conf ^. configSynthesisers)
)
(force $ descriptionToSim <$> conf ^. configSimulators)
yos
)
synthesisers :: Monad m => Fuzz m [SynthTool]
synthesisers = lift $ asks getSynthesisers
--simulators :: (Monad m) => Fuzz () m [SimTool]
--simulators = lift $ asks getSimulators
--combinations :: [a] -> [b] -> [(a, b)]
--combinations l1 l2 = [ (x, y) | x <- l1, y <- l2 ]
logT :: MonadSh m => Text -> m ()
logT = liftSh . logger
timeit :: (MonadIO m, MonadSh m) => m a -> m (NominalDiffTime, a)
timeit a = do
start <- liftIO getCurrentTime
result <- a
end <- liftIO getCurrentTime
return (diffUTCTime end start, result)
synthesis :: (MonadBaseControl IO m, MonadSh m) => SourceInfo -> Fuzz m ()
synthesis src = do
synth <- synthesisers
resTimes <- liftSh $ mapM exec synth
fuzzSynthStatus
.= applyList (uncurry . SynthStatus <$> synth) (fmap swap resTimes)
liftSh $ inspect resTimes
where
exec a = toolRun ("synthesis with " <> toText a) . runResultT $ do
liftSh . mkdir_p . fromText $ toText a
pop (fromText $ toText a) $ runSynth a src
passedSynthesis :: MonadSh m => Fuzz m [SynthTool]
passedSynthesis = fmap toSynth . filter passed . _fuzzSynthStatus <$> get
where
passed (SynthStatus _ (Pass _) _) = True
passed _ = False
toSynth (SynthStatus s _ _) = s
failedSynthesis :: MonadSh m => Fuzz m [SynthTool]
failedSynthesis = fmap toSynth . filter failed . _fuzzSynthStatus <$> get
where
failed (SynthStatus _ (Fail SynthFail) _) = True
failed _ = False
toSynth (SynthStatus s _ _) = s
make :: MonadSh m => FilePath -> m ()
make f = liftSh $ do
mkdir_p f
cp_r "data" $ f </> fromText "data"
pop :: (MonadBaseControl IO m, MonadSh m) => FilePath -> m a -> m a
pop f a = do
dir <- liftSh pwd
finally (liftSh (cd f) >> a) . liftSh $ cd dir
applyList :: [a -> b] -> [a] -> [b]
applyList a b = apply' <$> zip a b where apply' (a', b') = a' b'
applyLots :: (a -> b -> c -> d -> e) -> [(a, b)] -> [(c, d)] -> [e]
applyLots func a b = applyList (uncurry . uncurry func <$> a) b
toSynthResult
:: [(SynthTool, SynthTool)]
-> [(NominalDiffTime, Result Failed ())]
-> [SynthResult]
toSynthResult a b = applyLots SynthResult a $ fmap swap b
toolRun :: (MonadIO m, MonadSh m) => Text -> m a -> m (NominalDiffTime, a)
toolRun t m = do
logT $ "Running " <> t
(diff, res) <- timeit m
logT $ "Finished " <> t <> " (" <> showT diff <> ")"
return (diff, res)
equivalence :: (MonadBaseControl IO m, MonadSh m) => SourceInfo -> Fuzz m ()
equivalence src = do
synth <- passedSynthesis
-- let synthComb =
-- nubBy tupEq . filter (uncurry (/=)) $ combinations synth synth
let synthComb =
nubBy tupEq
. filter (uncurry (/=))
$ (,) defaultIdentitySynth
<$> synth
resTimes <- liftSh $ mapM (uncurry equiv) synthComb
fuzzSynthResults .= toSynthResult synthComb resTimes
liftSh $ inspect resTimes
where
tupEq (a, b) (a', b') = (a == a' && b == b') || (a == b' && b == a')
equiv a b =
toolRun ("equivalence check for " <> toText a <> " and " <> toText b)
. runResultT
$ do
make dir
pop dir $ do
liftSh $ do
cp
( fromText ".."
</> fromText (toText a)
</> synthOutput a
)
$ synthOutput a
cp
( fromText ".."
</> fromText (toText b)
</> synthOutput b
)
$ synthOutput b
writefile "rtl.v" $ genSource src
runEquiv a b src
where dir = fromText $ "equiv_" <> toText a <> "_" <> toText b
failEquivWithIdentity :: (MonadSh m) => Fuzz m [SynthResult]
failEquivWithIdentity = filter withIdentity . _fuzzSynthResults <$> get
where
withIdentity (SynthResult (IdentitySynth _) _ (Fail EquivFail) _) = True
withIdentity (SynthResult _ (IdentitySynth _) (Fail EquivFail) _) = True
withIdentity _ = False
-- | Always reduces with respect to 'Identity'.
reduction :: (MonadSh m) => SourceInfo -> Fuzz m ()
reduction src = do
fails <- failEquivWithIdentity
synthFails <- failedSynthesis
_ <- liftSh $ mapM red fails
_ <- liftSh $ mapM redSynth synthFails
return ()
where
red (SynthResult a b _ _) = do
make dir
pop dir $ do
s <- reduceSynth a b src
writefile (fromText ".." </> dir <.> "v") $ genSource s
return s
where dir = fromText $ "reduce_" <> toText a <> "_" <> toText b
redSynth a = do
make dir
pop dir $ do
s <- reduceSynthesis a src
writefile (fromText ".." </> dir <.> "v") $ genSource s
return s
where dir = fromText $ "reduce_" <> toText a
titleRun
:: (MonadIO m, MonadSh m) => Text -> Fuzz m a -> Fuzz m (NominalDiffTime, a)
titleRun t f = do
logT $ "### Starting " <> t <> " ###"
(diff, res) <- timeit f
logT $ "### Finished " <> t <> " (" <> showT diff <> ") ###"
return (diff, res)
whenMaybe :: Applicative m => Bool -> m a -> m (Maybe a)
whenMaybe b x = if b then Just <$> x else pure Nothing
getTime :: (Num n) => Maybe (n, a) -> n
getTime = maybe 0 fst
generateSample
:: (MonadIO m, MonadSh m)
=> Fuzz m (Seed, SourceInfo)
-> Fuzz m (Seed, SourceInfo)
generateSample f = do
logT "Sampling Verilog from generator"
(t, v@(s, _)) <- timeit f
logT $ "Chose " <> showT s
logT $ "Generated Verilog (" <> showT t <> ")"
return v
verilogSize :: (Source a) => a -> Int
verilogSize = length . lines . T.unpack . genSource
sampleVerilog
:: (MonadSh m, MonadIO m, Source a, Ord a) =>
Frequency a -> Int -> Maybe Seed -> Gen a -> m (Seed, a)
sampleVerilog _ _ seed@(Just _) gen = sampleSeed seed gen
sampleVerilog freq n Nothing gen = do
res <- replicateM n $ sampleSeed Nothing gen
let sizes = fmap getSize res
let samples = fmap snd . sort $ zip sizes res
liftIO $ Hog.sample . Hog.frequency $ freq samples
where
getSize (_, s) = verilogSize s
hatFreqs :: Frequency a
hatFreqs l = zip hat (return <$> l)
where
h = length l `div` 2
hat = (+ h) . negate . abs . (h - ) <$> [1..length l]
meanFreqs :: Source a => Frequency a
meanFreqs l = zip hat (return <$> l)
where
hat = calc <$> sizes
calc i = if abs (mean - i) == min_ then 1 else 0
mean = sum sizes `div` length l
min_ = minimum $ abs . (mean -) <$> sizes
sizes = verilogSize . snd <$> l
medianFreqs :: Frequency a
medianFreqs l = zip hat (return <$> l)
where
h = length l `div` 2
hat = set_ <$> [1..length l]
set_ n = if n == h then 1 else 0
fuzz :: MonadFuzz m => Gen SourceInfo -> Config -> Fuzz m FuzzReport
fuzz gen conf = do
(seed', src) <- generateSample genMethod
let size = length . lines . T.unpack $ genSource src
liftSh
. writefile "config.toml"
. encodeConfig
$ conf
& configProperty
. propSeed
?~ seed'
(tsynth, _) <- titleRun "Synthesis" $ synthesis src
(tequiv, _) <- titleRun "Equivalence Check" $ equivalence src
fails <- failEquivWithIdentity
synthFails <- failedSynthesis
redResult <- whenMaybe (not $ null fails && null synthFails) . titleRun "Reduction" $ reduction
src
state_ <- get
currdir <- liftSh pwd
let vi = flip view state_
let report = FuzzReport currdir
(vi fuzzSynthResults)
(vi fuzzSimResults)
(vi fuzzSynthStatus)
size
tsynth
tequiv
(getTime redResult)
liftSh . writefile "index.html" $ printResultReport (bname currdir) report
return report
where
seed = conf ^. configProperty . propSeed
bname = T.pack . takeBaseName . T.unpack . toTextIgnore
genMethod = case T.toLower $ conf ^. configProperty . propSampleMethod of
"hat" -> do
logT "Using the hat function"
sv hatFreqs
"mean" -> do
logT "Using the mean function"
sv meanFreqs
"median" -> do
logT "Using the median function"
sv medianFreqs
_ -> do
logT "Using first seed"
sampleSeed seed gen
sv a = sampleVerilog a (conf ^. configProperty . propSampleSize) seed gen
relativeFuzzReport :: (MonadSh m) => FuzzReport -> m FuzzReport
relativeFuzzReport fr@(FuzzReport dir _ _ _ _ _ _ _) = liftSh $ do
newPath <- relPath dir
return $ (fuzzDir .~ newPath) fr
fuzzInDir
:: MonadFuzz m => FilePath -> Gen SourceInfo -> Config -> Fuzz m FuzzReport
fuzzInDir fp src conf = do
make fp
res <- pop fp $ fuzz src conf
relativeFuzzReport res
fuzzMultiple
:: MonadFuzz m
=> Int
-> Maybe FilePath
-> Gen SourceInfo
-> Config
-> Fuzz m [FuzzReport]
fuzzMultiple n fp src conf = do
x <- case fp of
Nothing -> do
ct <- liftIO getZonedTime
return
. fromText
. T.pack
$ "output_"
<> formatTime defaultTimeLocale "%Y-%m-%d_%H-%M-%S" ct
Just f -> return f
make x
pop x $ do
results <- if isNothing seed
then forM [1 .. n] fuzzDir'
else (: []) <$> fuzzDir' (1 :: Int)
liftSh . writefile (fromText "index" <.> "html") $ printSummary
"Fuzz Summary"
results
return results
where
fuzzDir' n' = fuzzInDir (fromText $ "fuzz_" <> showT n') src conf
seed = conf ^. configProperty . propSeed
sampleSeed :: MonadSh m => Maybe Seed -> Gen a -> m (Seed, a)
sampleSeed s gen =
liftSh
$ let
loop n = if n <= 0
then
error
"Hedgehog.Gen.sample: too many discards, could not generate a sample"
else do
seed <- maybe Hog.random return s
case
runIdentity
. runMaybeT
. Hog.runTree
$ Hog.runGenT 30 seed gen
of
Nothing -> loop (n - 1)
Just x -> return (seed, Hog.nodeValue x)
in loop (100 :: Int)
|