1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
module Shared
open Types
open Logger
open System.Text.RegularExpressions
// Helpers
/// Inverse the values in a tuple
let invTuple (a, b) = b, a
/// Try to find a key in a map, return Some value when it is found, otherwise it
/// returns None
let mapTryFind k (map: Map<'a, 'b>) = map.TryFind(k)
/// Try find key in a list
let listTryFind s =
List.map invTuple >> Map.ofList >> mapTryFind s
/// The list of characters used in the parser
let charList = ["#", HASH; "|", PIPE; "=", EQUAL; "-", MINUS; "+", PLUS; "*", ASTERISK
".", DOT; "**", DASTERISK; "***", TASTERISK; "_", UNDERSCORE; "__", DUNDERSCORE
"___", TUNDERSCORE; "~", TILDE; "~~", DTILDE; "~~~", TTILDE; "[", LSBRA
"]", RSBRA; "(", LBRA; ")", RBRA; @"\", BSLASH; "/", SLASH; "<", LABRA
">", RABRA; "{", LCBRA; "}", RCBRA; "`", BACKTICK;
"!", EXCLAMATION; ":", COLON; "^", CARET; "%", PERCENT; ",", COMMA]
let charMap = charList |> List.map invTuple |> Map.ofList
let (|CharTok|_|) tok =
mapTryFind tok charMap
let mapTok = function
| CharTok s -> s
| CODEBLOCK _ -> "CODEBLOCK"
| FOOTNOTE _ -> sprintf "FOOTNOTE found"
| CITATION _ -> sprintf "CITATION found"
| HEADER n -> sprintf "HEADER %d" n
| NUMBER s -> s
| LITERAL s -> s
| WHITESPACE n -> String.replicate n " "
| _ -> "\n"
/// turn all tokens into string
let strAllToks toks =
let pacMan str tok =
str + mapTok tok
List.fold pacMan "" toks
let mapLang lang =
match lang with
| Python -> "python"
| FSharp -> "fsharp"
| CPP -> "cpp"
| C -> "c"
| Empty -> ""
let xOnwards x lst = if List.length lst > x then lst.[x..] else []
let sOnwards s str = if String.length str > s then str.[s..] else ""
let removeChars lst s =
let folder (s:string) x = s.Replace(x,"")
List.fold folder s lst
let replaceChars pat (rep:string) s =
Regex.Replace(s,pat,rep)
let removeWhitespace (s:string) =
s |> removeChars ["\n";"\t";"\r";" "]
let sharedLog = Logger(LogLevel.WARNING)
|
