1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
|
{-|
Module : Verismith.Fuzz
Description : Environment to run the simulator and synthesisers in a matrix.
Copyright : (c) 2019, Yann Herklotz
License : GPL-3
Maintainer : yann [at] yannherklotz [dot] com
Stability : experimental
Portability : POSIX
Environment to run the simulator and synthesisers in a matrix.
-}
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE TemplateHaskell #-}
module Verismith.Fuzz
( Fuzz (..)
, FuzzOpts (..)
, fuzz
, fuzzInDir
, fuzzMultiple
, runFuzz
, sampleSeed
-- * Helpers
, make
, pop
)
where
import Control.DeepSeq (force)
import Control.Exception.Lifted (finally)
import Control.Lens hiding ((<.>))
import Control.Monad (forM, replicateM)
import Control.Monad.IO.Class
import Control.Monad.Reader
import Control.Monad.State.Strict
import Control.Monad.Trans.Control (MonadBaseControl)
import qualified Crypto.Random.DRBG as C
import Data.ByteString (ByteString)
import Data.List (nubBy, sort)
import Data.Maybe (catMaybes, fromMaybe, isNothing)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Time
import Data.Tuple (swap)
import Hedgehog (Gen)
import qualified Hedgehog.Internal.Gen as Hog
import Hedgehog.Internal.Seed (Seed)
import qualified Hedgehog.Internal.Seed as Hog
import qualified Hedgehog.Internal.Tree as Hog
import Prelude hiding (FilePath)
import Shelly hiding (get)
import Shelly.Lifted (MonadSh, liftSh)
import System.FilePath.Posix (takeBaseName)
import Verismith.Config
import Verismith.CounterEg (CounterEg (..))
import Verismith.Internal
import Verismith.Reduce
import Verismith.Report
import Verismith.Result
import Verismith.Tool.Icarus
import Verismith.Tool.Internal
import Verismith.Tool.Yosys
import Verismith.Verilog.AST
import Verismith.Verilog.CodeGen
data FuzzOpts = FuzzOpts { _fuzzOptsOutput :: !(Maybe FilePath)
, _fuzzOptsForced :: !Bool
, _fuzzOptsKeepAll :: !Bool
, _fuzzOptsIterations :: {-# UNPACK #-} !Int
, _fuzzOptsNoSim :: !Bool
, _fuzzOptsNoEquiv :: !Bool
, _fuzzOptsNoReduction :: !Bool
, _fuzzOptsConfig :: {-# UNPACK #-} !Config
, _fuzzDataDir :: {-# UNPACK #-} !FilePath
}
deriving (Show, Eq)
$(makeLenses ''FuzzOpts)
defaultFuzzOpts :: FuzzOpts
defaultFuzzOpts = FuzzOpts { _fuzzOptsOutput = Nothing
, _fuzzOptsForced = False
, _fuzzOptsKeepAll = False
, _fuzzOptsIterations = 1
, _fuzzOptsNoSim = False
, _fuzzOptsNoEquiv = False
, _fuzzOptsNoReduction = False
, _fuzzOptsConfig = defaultConfig
, _fuzzDataDir = fromText "."
}
data FuzzEnv = FuzzEnv { _getSynthesisers :: ![SynthTool]
, _getSimulators :: ![SimTool]
, _yosysInstance :: {-# UNPACK #-} !Yosys
, _fuzzEnvOpts :: {-# UNPACK #-} !FuzzOpts
}
deriving (Eq, Show)
$(makeLenses ''FuzzEnv)
data FuzzState = FuzzState { _fuzzSynthResults :: ![SynthResult]
, _fuzzSimResults :: ![SimResult]
, _fuzzSynthStatus :: ![SynthStatus]
}
deriving (Eq, Show)
$(makeLenses ''FuzzState)
type Frequency a = [(Seed, a)] -> [(Int, Gen (Seed, a))]
-- | The main type for the fuzzing, which contains an environment that can be
-- read from and the current state of all the results.
type Fuzz m = StateT FuzzState (ReaderT FuzzEnv m)
type MonadFuzz m = (MonadBaseControl IO m, MonadIO m, MonadSh m)
runFuzz :: MonadIO m => FuzzOpts -> Yosys -> Fuzz Sh a -> m a
runFuzz fo yos m = shelly $ runFuzz' fo yos m
runFuzz' :: Monad m => FuzzOpts -> Yosys -> Fuzz m b -> m b
runFuzz' fo yos m = runReaderT
(evalStateT m (FuzzState [] [] []))
(FuzzEnv { _getSynthesisers = ( force
$ defaultIdentitySynth
: (descriptionToSynth <$> conf ^. configSynthesisers)
)
, _getSimulators = (force $ descriptionToSim <$> conf ^. configSimulators)
, _yosysInstance = yos
, _fuzzEnvOpts = fo
}
)
where
conf = _fuzzOptsConfig fo
askConfig :: Monad m => Fuzz m Config
askConfig = asks (_fuzzOptsConfig . _fuzzEnvOpts)
askOpts :: Monad m => Fuzz m FuzzOpts
askOpts = asks _fuzzEnvOpts
genMethod conf seed gen =
case T.toLower $ conf ^. configProperty . propSampleMethod of
"hat" -> do
logT "Using the hat function"
sv hatFreqs
"mean" -> do
logT "Using the mean function"
sv meanFreqs
"median" -> do
logT "Using the median function"
sv medianFreqs
_ -> do
logT "Using first seed"
sampleSeed seed gen
where
sv a = sampleVerilog a (conf ^. configProperty . propSampleSize) seed gen
relativeFuzzReport :: (MonadSh m) => FuzzReport -> m FuzzReport
relativeFuzzReport fr@(FuzzReport dir _ _ _ _ _ _ _) = liftSh $ do
newPath <- relPath dir
return $ (fuzzDir .~ newPath) fr
filterSynth :: SynthResult -> Bool
filterSynth (SynthResult _ _ (Pass _) _) = True
filterSynth _ = False
filterSim :: SimResult -> Bool
filterSim (SimResult _ _ _ (Pass _) _) = True
filterSim _ = False
filterSynthStat :: SynthStatus -> Bool
filterSynthStat (SynthStatus _ (Pass _) _) = True
filterSynthStat _ = False
passedFuzz :: FuzzReport -> Bool
passedFuzz (FuzzReport _ synth sim synthstat _ _ _ _) =
(passedSynth + passedSim + passedSynthStat) == 0
where
passedSynth = length $ filter (not . filterSynth) synth
passedSim = length $ filter (not . filterSim) sim
passedSynthStat = length $ filter (not . filterSynthStat) synthstat
synthesisers :: Monad m => Fuzz m [SynthTool]
synthesisers = lift $ asks _getSynthesisers
--simulators :: (Monad m) => Fuzz () m [SimTool]
--simulators = lift $ asks getSimulators
--combinations :: [a] -> [b] -> [(a, b)]
--combinations l1 l2 = [ (x, y) | x <- l1, y <- l2 ]
logT :: MonadSh m => Text -> m ()
logT = liftSh . logger
timeit :: (MonadIO m, MonadSh m) => m a -> m (NominalDiffTime, a)
timeit a = do
start <- liftIO getCurrentTime
result <- a
end <- liftIO getCurrentTime
return (diffUTCTime end start, result)
synthesis :: (MonadBaseControl IO m, MonadSh m) => SourceInfo -> Fuzz m ()
synthesis src = do
synth <- synthesisers
resTimes <- liftSh $ mapM exec synth
fuzzSynthStatus
.= applyList (uncurry . SynthStatus <$> synth) (fmap swap resTimes)
liftSh $ inspect resTimes
where
exec a = toolRun ("synthesis with " <> toText a) . runResultT $ do
liftSh . mkdir_p . fromText $ toText a
pop (fromText $ toText a) $ runSynth a src
passedSynthesis :: MonadSh m => Fuzz m [SynthTool]
passedSynthesis = fmap toSynth . filter passed . _fuzzSynthStatus <$> get
where
passed (SynthStatus _ (Pass _) _) = True
passed _ = False
toSynth (SynthStatus s _ _) = s
failedSynthesis :: MonadSh m => Fuzz m [SynthTool]
failedSynthesis = fmap toSynth . filter failed . _fuzzSynthStatus <$> get
where
failed (SynthStatus _ (Fail SynthFail) _) = True
failed _ = False
toSynth (SynthStatus s _ _) = s
make :: MonadSh m => FilePath -> m ()
make f = liftSh $ mkdir_p f
pop :: (MonadBaseControl IO m, MonadSh m) => FilePath -> m a -> m a
pop f a = do
dir <- liftSh pwd
finally (liftSh (cd f) >> a) . liftSh $ cd dir
applyList :: [a -> b] -> [a] -> [b]
applyList a b = apply' <$> zip a b where apply' (a', b') = a' b'
applyLots :: (a -> b -> c -> d -> e) -> [(a, b)] -> [(c, d)] -> [e]
applyLots func a b = applyList (uncurry . uncurry func <$> a) b
toSynthResult :: [(SynthTool, SynthTool)]
-> [(NominalDiffTime, Result Failed ())]
-> [SynthResult]
toSynthResult a b = applyLots SynthResult a $ fmap swap b
toSimResult :: SimTool
-> [ByteString]
-> [SynthTool]
-> [(NominalDiffTime, Result Failed ByteString)]
-> [SimResult]
toSimResult sima bs as b =
applyList (applyList (repeat uncurry) (applyList (applyList (SimResult <$> as) (repeat sima)) (repeat bs))) $ fmap swap b
toolRun :: (MonadIO m, MonadSh m) => Text -> m a -> m (NominalDiffTime, a)
toolRun t m = do
logT $ "Running " <> t
(diff, res) <- timeit m
logT $ "Finished " <> t <> " (" <> showT diff <> ")"
return (diff, res)
equivalence :: (MonadBaseControl IO m, MonadSh m) => SourceInfo -> Fuzz m ()
equivalence src = do
datadir <- fmap _fuzzDataDir askOpts
synth <- passedSynthesis
-- let synthComb =
-- nubBy tupEq . filter (uncurry (/=)) $ combinations synth synth
let synthComb =
nubBy tupEq
. filter (uncurry (/=))
$ (,) defaultIdentitySynth
<$> synth
resTimes <- liftSh $ mapM (uncurry (equiv datadir)) synthComb
fuzzSynthResults .= toSynthResult synthComb resTimes
liftSh $ inspect resTimes
where
tupEq (a, b) (a', b') = (a == a' && b == b') || (a == b' && b == a')
equiv datadir a b =
toolRun ("equivalence check for " <> toText a <> " and " <> toText b)
. runResultT
$ do make dir
pop dir $ do
liftSh $ do
cp ( fromText ".."
</> fromText (toText a)
</> synthOutput a
) $ synthOutput a
cp ( fromText ".."
</> fromText (toText b)
</> synthOutput b
) $ synthOutput b
writefile "rtl.v" $ genSource src
runEquiv datadir a b src
where dir = fromText $ "equiv_" <> toText a <> "_" <> toText b
simulation :: (MonadIO m, MonadSh m) => SourceInfo -> Fuzz m ()
simulation src = do
datadir <- fmap _fuzzDataDir askOpts
synth <- passedSynthesis
counterEgs <- failEquivWithIdentityCE
vals <- liftIO $ generateByteString 20
ident <- liftSh $ sim datadir vals Nothing defaultIdentitySynth
resTimes <- liftSh $ mapM (sim datadir vals (justPass $ snd ident)) synth
resTimes2 <- liftSh $ mapM (simCounterEg datadir) counterEgs
fuzzSimResults .= toSimResult defaultIcarusSim vals synth resTimes
liftSh
. inspect
$ (\(_, r) -> bimap show (T.unpack . T.take 10 . showBS) r)
<$> (ident : resTimes)
where
sim datadir b i a = toolRun ("simulation for " <> toText a) . runResultT $ do
make dir
pop dir $ do
liftSh $ do
cp (fromText ".." </> fromText (toText a) </> synthOutput a)
$ synthOutput a
writefile "rtl.v" $ genSource src
runSimIc datadir defaultIcarus a src b i
where dir = fromText $ "simulation_" <> toText a
simCounterEg datadir (a, b) = toolRun ("counter-example simulation for " <> toText a) . runResultT $ do
make dir
pop dir $ do
liftSh $ do
cp (fromText ".." </> fromText (toText a) </> synthOutput a) $ synthOutput a
writefile "rtl.v" $ genSource src
ident <- runSimIcEC datadir defaultIcarus defaultIdentitySynth src b Nothing
runSimIcEC datadir defaultIcarus a src b (Just ident)
where dir = fromText $ "countereg_sim_" <> toText a
-- | Generate a specific number of random bytestrings of size 256.
randomByteString :: C.CtrDRBG -> Int -> [ByteString] -> [ByteString]
randomByteString gen n bytes
| n == 0 = ranBytes : bytes
| otherwise = randomByteString newGen (n - 1) $ ranBytes : bytes
where Right (ranBytes, newGen) = C.genBytes 32 gen
-- | generates the specific number of bytestring with a random seed.
generateByteString :: Int -> IO [ByteString]
generateByteString n = do
gen <- C.newGenIO :: IO C.CtrDRBG
return $ randomByteString gen n []
failEquivWithIdentity :: (MonadSh m) => Fuzz m [SynthResult]
failEquivWithIdentity = filter withIdentity . _fuzzSynthResults <$> get
where
withIdentity (SynthResult (IdentitySynth _) _ (Fail (EquivFail _)) _) = True
withIdentity (SynthResult _ (IdentitySynth _) (Fail (EquivFail _)) _) = True
withIdentity _ = False
failEquivWithIdentityCE :: (MonadSh m) => Fuzz m [(SynthTool, CounterEg)]
failEquivWithIdentityCE = catMaybes . fmap withIdentity . _fuzzSynthResults <$> get
where
withIdentity (SynthResult (IdentitySynth _) s (Fail (EquivFail c)) _) = Just (s, c)
withIdentity (SynthResult s (IdentitySynth _) (Fail (EquivFail c)) _) = Just (s, c)
withIdentity _ = Nothing
failedSimulations :: (MonadSh m) => Fuzz m [SimResult]
failedSimulations = filter failedSim . _fuzzSimResults <$> get
where
failedSim (SimResult _ _ _ (Fail (SimFail _)) _) = True
failedSim _ = False
passEquiv :: (MonadSh m) => Fuzz m [SynthResult]
passEquiv = filter withIdentity . _fuzzSynthResults <$> get
where
withIdentity (SynthResult _ _ (Pass _) _) = True
withIdentity _ = False
-- | Always reduces with respect to 'Identity'.
reduction :: (MonadSh m) => SourceInfo -> Fuzz m ()
reduction src = do
datadir <- fmap _fuzzDataDir askOpts
fails <- failEquivWithIdentity
synthFails <- failedSynthesis
simFails <- failedSimulations
_ <- liftSh $ mapM (red datadir) fails
_ <- liftSh $ mapM redSynth synthFails
_ <- liftSh $ mapM (redSim datadir) simFails
return ()
where
red datadir (SynthResult a b _ _) = do
make dir
pop dir $ do
s <- reduceSynth datadir a b src
writefile (fromText ".." </> dir <.> "v") $ genSource s
return s
where dir = fromText $ "reduce_" <> toText a <> "_" <> toText b
redSynth a = do
make dir
pop dir $ do
s <- reduceSynthesis a src
writefile (fromText ".." </> dir <.> "v") $ genSource s
return s
where dir = fromText $ "reduce_" <> toText a
redSim datadir (SimResult t _ bs _ _) = do
make dir
pop dir $ do
s <- reduceSimIc datadir bs t src
writefile (fromText ".." </> dir <.> "v") $ genSource s
return s
where dir = fromText $ "reduce_sim_" <> toText t
titleRun
:: (MonadIO m, MonadSh m) => Text -> Fuzz m a -> Fuzz m (NominalDiffTime, a)
titleRun t f = do
logT $ "### Starting " <> t <> " ###"
(diff, res) <- timeit f
logT $ "### Finished " <> t <> " (" <> showT diff <> ") ###"
return (diff, res)
whenMaybe :: Applicative m => Bool -> m a -> m (Maybe a)
whenMaybe b x = if b then Just <$> x else pure Nothing
getTime :: (Num n) => Maybe (n, a) -> n
getTime = maybe 0 fst
generateSample
:: (MonadIO m, MonadSh m)
=> Fuzz m (Seed, SourceInfo)
-> Fuzz m (Seed, SourceInfo)
generateSample f = do
logT "Sampling Verilog from generator"
(t, v@(s, _)) <- timeit f
logT $ "Chose " <> showT s
logT $ "Generated Verilog (" <> showT t <> ")"
return v
verilogSize :: (Source a) => a -> Int
verilogSize = length . lines . T.unpack . genSource
sampleVerilog
:: (MonadSh m, MonadIO m, Source a, Ord a)
=> Frequency a
-> Int
-> Maybe Seed
-> Gen a
-> m (Seed, a)
sampleVerilog _ _ seed@(Just _) gen = sampleSeed seed gen
sampleVerilog freq n Nothing gen = do
res <- replicateM n $ sampleSeed Nothing gen
let sizes = fmap getSize res
let samples = fmap snd . sort $ zip sizes res
liftIO $ Hog.sample . Hog.frequency $ freq samples
where getSize (_, s) = verilogSize s
hatFreqs :: Frequency a
hatFreqs l = zip hat (return <$> l)
where
h = length l `div` 2
hat = (+ h) . negate . abs . (h -) <$> [1 .. length l]
meanFreqs :: Source a => Frequency a
meanFreqs l = zip hat (return <$> l)
where
hat = calc <$> sizes
calc i = if abs (mean - i) == min_ then 1 else 0
mean = sum sizes `div` length l
min_ = minimum $ abs . (mean -) <$> sizes
sizes = verilogSize . snd <$> l
medianFreqs :: Frequency a
medianFreqs l = zip hat (return <$> l)
where
h = length l `div` 2
hat = set_ <$> [1 .. length l]
set_ n = if n == h then 1 else 0
fuzz :: MonadFuzz m => Gen SourceInfo -> Fuzz m FuzzReport
fuzz gen = do
conf <- askConfig
opts <- askOpts
let seed = conf ^. configProperty . propSeed
(seed', src) <- generateSample $ genMethod conf seed gen
let size = length . lines . T.unpack $ genSource src
liftSh
. writefile "config.toml"
. encodeConfig
$ conf
& configProperty
. propSeed
?~ seed'
(tsynth, _) <- titleRun "Synthesis" $ synthesis src
(tequiv, _) <- if (_fuzzOptsNoEquiv opts)
then return (0, mempty)
else titleRun "Equivalence Check" $ equivalence src
(_ , _) <- if (_fuzzOptsNoSim opts)
then return (0, mempty)
else titleRun "Simulation" $ simulation src
fails <- failEquivWithIdentity
failedSim <- failedSimulations
synthFails <- failedSynthesis
redResult <-
whenMaybe (not (null failedSim && null fails && null synthFails)
&& not (_fuzzOptsNoReduction opts))
. titleRun "Reduction"
$ reduction src
state_ <- get
currdir <- liftSh pwd
let vi = flip view state_
let report = FuzzReport currdir
(vi fuzzSynthResults)
(vi fuzzSimResults)
(vi fuzzSynthStatus)
size
tsynth
tequiv
(getTime redResult)
return report
fuzzInDir :: MonadFuzz m => Gen SourceInfo -> Fuzz m FuzzReport
fuzzInDir src = do
fuzzOpts <- askOpts
let fp = fromMaybe "fuzz" $ _fuzzOptsOutput fuzzOpts
make fp
res <- pop fp $ fuzz src
liftSh $ do
writefile (fp <.> "html") $ printResultReport (bname fp) res
when (passedFuzz res && not (_fuzzOptsKeepAll fuzzOpts)) $ rm_rf fp
relativeFuzzReport res
where
bname = T.pack . takeBaseName . T.unpack . toTextIgnore
fuzzMultiple
:: MonadFuzz m
=> Gen SourceInfo
-> Fuzz m [FuzzReport]
fuzzMultiple src = do
fuzzOpts <- askOpts
let seed = (_fuzzOptsConfig fuzzOpts) ^. configProperty . propSeed
x <- case _fuzzOptsOutput fuzzOpts of
Nothing -> do
ct <- liftIO getZonedTime
return
. fromText
. T.pack
$ "output_"
<> formatTime defaultTimeLocale "%Y-%m-%d_%H-%M-%S" ct
Just f -> return f
make x
pop x $ do
results <- if isNothing seed
then forM [1 .. (_fuzzOptsIterations fuzzOpts)] fuzzDir'
else (: []) <$> fuzzDir' (1 :: Int)
liftSh . writefile (fromText "index" <.> "html") $ printSummary
"Fuzz Summary"
results
return results
where
fuzzDir' :: (Show a, MonadFuzz m) => a -> Fuzz m FuzzReport
fuzzDir' n' = local (fuzzEnvOpts . fuzzOptsOutput .~
(Just . fromText $ "fuzz_" <> showT n'))
$ fuzzInDir src
sampleSeed :: MonadSh m => Maybe Seed -> Gen a -> m (Seed, a)
sampleSeed s gen =
liftSh
$ let loop n = if n <= 0
then
error
"Hedgehog.Gen.sample: too many discards, could not generate a sample"
else do
seed <- maybe Hog.random return s
case Hog.evalGen 30 seed gen of
Nothing ->
loop (n - 1)
Just x ->
pure (seed, Hog.treeValue x)
in loop (100 :: Int)
|