diff options
Diffstat (limited to 'FMark/src/Common/Lexer/Lexer.fs')
| -rw-r--r-- | FMark/src/Common/Lexer/Lexer.fs | 25 |
1 files changed, 13 insertions, 12 deletions
diff --git a/FMark/src/Common/Lexer/Lexer.fs b/FMark/src/Common/Lexer/Lexer.fs index 5e2c7d6..a7bd111 100644 --- a/FMark/src/Common/Lexer/Lexer.fs +++ b/FMark/src/Common/Lexer/Lexer.fs @@ -48,23 +48,23 @@ let nextToken state s = match s, state with | EscapedCharTok n, _ -> n, state | HTMLSingleton (s, r), Normal -> - (HTMLLIT s, r), Normal + (LITERAL s, r), Normal | HTMLStartTag (s, [t], r), Normal -> - (HTMLLIT s, r), InHTMLTag (t, 1) + (LITERAL s, r), InHTMLTag (t, 1) | HTMLStartTag (s, [t], r), InHTMLTag (tag, d) -> if t = tag then - (HTMLLIT s, r), InHTMLTag (tag, d+1) + (LITERAL s, r), InHTMLTag (tag, d+1) else - (HTMLLIT s, r), InHTMLTag (tag, d) + (LITERAL s, r), InHTMLTag (tag, d) | HTMLEndTag (s, [t], r), InHTMLTag (tag, d) -> if t = tag then - if d = 1 then (HTMLLIT s, r), Normal - else (HTMLLIT s, r), InHTMLTag (tag, d-1) - else (HTMLLIT s, r), InHTMLTag (tag, d) + if d = 1 then (LITERAL s, r), Normal + else (LITERAL s, r), InHTMLTag (tag, d-1) + else (LITERAL s, r), InHTMLTag (tag, d) | RegexMatch "^.+?(?=<)" (s, _, r), InHTMLTag (t, d) -> - (HTMLLIT s, r), InHTMLTag (t, d) + (LITERAL s, r), InHTMLTag (t, d) | RegexMatch "^.*" (s, _, r), InHTMLTag (t, d) -> - (HTMLLIT s, r), InHTMLTag (t, d) + (LITERAL s, r), InHTMLTag (t, d) | CharacterTok n, _ -> n, state | RegexMatch @"^\s+" (m, _, s), _ -> (replaceChars "\t" " " m @@ -80,8 +80,9 @@ let nextToken state s = /// Lexes a whole string and returns the result as a Token list let lexS state source = let rec lexS' state s tokList = - match s with - | ""-> ENDLINE :: tokList + match s, state with + | "", InHTMLTag _ -> tokList + | "", _ -> ENDLINE :: tokList | _ -> let (nt, st'), nstate = nextToken state s nt :: tokList |> lexS' nstate st' @@ -102,7 +103,7 @@ let returnTokens = function | _, InCodeBlock (s, l) -> [CODEBLOCK (s, l); ENDLINE] | tok, InHTMLTag (str, _) -> - tok @ [HTMLLIT str; ENDLINE] + tok @ [LITERAL str; ENDLINE] | tok, _ -> tok |