diff options
author | Yann Herklotz <ymherklotz@gmail.com> | 2018-03-23 03:42:06 +0000 |
---|---|---|
committer | Yann Herklotz <ymherklotz@gmail.com> | 2018-03-23 03:42:06 +0000 |
commit | dbc9281e3e97b557d2a4d17c345cc1c38c935a5b (patch) | |
tree | ce71be86e5cb583e63e98b8a10815882e8090811 /FMark | |
parent | 1edac097c4809ce1efd92559f6880af115694c9c (diff) | |
download | FMark-dbc9281e3e97b557d2a4d17c345cc1c38c935a5b.tar.gz FMark-dbc9281e3e97b557d2a4d17c345cc1c38c935a5b.zip |
[html] Removing newlines from html blocks
Diffstat (limited to 'FMark')
-rw-r--r-- | FMark/src/Common/Lexer/Lexer.fs | 25 | ||||
-rw-r--r-- | FMark/src/Common/Lexer/LexerTest.fs | 28 | ||||
-rw-r--r-- | FMark/src/Common/Types.fs | 2 |
3 files changed, 27 insertions, 28 deletions
diff --git a/FMark/src/Common/Lexer/Lexer.fs b/FMark/src/Common/Lexer/Lexer.fs index 5e2c7d6..a7bd111 100644 --- a/FMark/src/Common/Lexer/Lexer.fs +++ b/FMark/src/Common/Lexer/Lexer.fs @@ -48,23 +48,23 @@ let nextToken state s = match s, state with | EscapedCharTok n, _ -> n, state | HTMLSingleton (s, r), Normal -> - (HTMLLIT s, r), Normal + (LITERAL s, r), Normal | HTMLStartTag (s, [t], r), Normal -> - (HTMLLIT s, r), InHTMLTag (t, 1) + (LITERAL s, r), InHTMLTag (t, 1) | HTMLStartTag (s, [t], r), InHTMLTag (tag, d) -> if t = tag then - (HTMLLIT s, r), InHTMLTag (tag, d+1) + (LITERAL s, r), InHTMLTag (tag, d+1) else - (HTMLLIT s, r), InHTMLTag (tag, d) + (LITERAL s, r), InHTMLTag (tag, d) | HTMLEndTag (s, [t], r), InHTMLTag (tag, d) -> if t = tag then - if d = 1 then (HTMLLIT s, r), Normal - else (HTMLLIT s, r), InHTMLTag (tag, d-1) - else (HTMLLIT s, r), InHTMLTag (tag, d) + if d = 1 then (LITERAL s, r), Normal + else (LITERAL s, r), InHTMLTag (tag, d-1) + else (LITERAL s, r), InHTMLTag (tag, d) | RegexMatch "^.+?(?=<)" (s, _, r), InHTMLTag (t, d) -> - (HTMLLIT s, r), InHTMLTag (t, d) + (LITERAL s, r), InHTMLTag (t, d) | RegexMatch "^.*" (s, _, r), InHTMLTag (t, d) -> - (HTMLLIT s, r), InHTMLTag (t, d) + (LITERAL s, r), InHTMLTag (t, d) | CharacterTok n, _ -> n, state | RegexMatch @"^\s+" (m, _, s), _ -> (replaceChars "\t" " " m @@ -80,8 +80,9 @@ let nextToken state s = /// Lexes a whole string and returns the result as a Token list let lexS state source = let rec lexS' state s tokList = - match s with - | ""-> ENDLINE :: tokList + match s, state with + | "", InHTMLTag _ -> tokList + | "", _ -> ENDLINE :: tokList | _ -> let (nt, st'), nstate = nextToken state s nt :: tokList |> lexS' nstate st' @@ -102,7 +103,7 @@ let returnTokens = function | _, InCodeBlock (s, l) -> [CODEBLOCK (s, l); ENDLINE] | tok, InHTMLTag (str, _) -> - tok @ [HTMLLIT str; ENDLINE] + tok @ [LITERAL str; ENDLINE] | tok, _ -> tok diff --git a/FMark/src/Common/Lexer/LexerTest.fs b/FMark/src/Common/Lexer/LexerTest.fs index 9e65a89..6ed1132 100644 --- a/FMark/src/Common/Lexer/LexerTest.fs +++ b/FMark/src/Common/Lexer/LexerTest.fs @@ -309,46 +309,46 @@ let lexTest = "One line html", "<span>This is a span element</span>", - [HTMLLIT "<span>"; HTMLLIT "This is a span element"; HTMLLIT "</span>"; ENDLINE] + [LITERAL "<span>"; LITERAL "This is a span element"; LITERAL "</span>"; ENDLINE] "Online closing html", "<img src=\"https://github.com/IMAGE.png\" />", - [HTMLLIT "<img src=\"https://github.com/IMAGE.png\" />"; ENDLINE] + [LITERAL "<img src=\"https://github.com/IMAGE.png\" />"; ENDLINE] "HTML with non-HTML start", "This is an image: <span>Hello World</span>", [LITERAL "This"; WHITESPACE 1; LITERAL "is"; WHITESPACE 1; LITERAL "an" - WHITESPACE 1; LITERAL "image"; COLON; WHITESPACE 1; HTMLLIT "<span>" - HTMLLIT "Hello World"; HTMLLIT "</span>"; ENDLINE] + WHITESPACE 1; LITERAL "image"; COLON; WHITESPACE 1; LITERAL "<span>" + LITERAL "Hello World"; LITERAL "</span>"; ENDLINE] "Singleton HTML passthrough", "Singleton <br> passthrough and more text", - [LITERAL "Singleton"; WHITESPACE 1; HTMLLIT "<br>"; WHITESPACE 1; LITERAL "passthrough" + [LITERAL "Singleton"; WHITESPACE 1; LITERAL "<br>"; WHITESPACE 1; LITERAL "passthrough" WHITESPACE 1; LITERAL "and"; WHITESPACE 1; LITERAL "more"; WHITESPACE 1 LITERAL "text"; ENDLINE] "HTML image tag", "Embedding an <img src=\"https://github.com/IMAGE\"> in text", - [LITERAL "Embedding"; WHITESPACE 1; LITERAL "an"; WHITESPACE 1; HTMLLIT "<img src=\"https://github.com/IMAGE\">" + [LITERAL "Embedding"; WHITESPACE 1; LITERAL "an"; WHITESPACE 1; LITERAL "<img src=\"https://github.com/IMAGE\">" WHITESPACE 1; LITERAL "in"; WHITESPACE 1; LITERAL "text"; ENDLINE] "A lot of nested tags", "<p><p><p><p><p><p><p> </p></p></p></p></p></p></p>", - [HTMLLIT "<p>"; HTMLLIT "<p>"; HTMLLIT "<p>"; HTMLLIT "<p>"; HTMLLIT "<p>"; HTMLLIT "<p>" - HTMLLIT "<p>"; HTMLLIT " "; HTMLLIT "</p>"; HTMLLIT "</p>"; HTMLLIT "</p>"; HTMLLIT "</p>" - HTMLLIT "</p>"; HTMLLIT "</p>"; HTMLLIT "</p>"; ENDLINE] + [LITERAL "<p>"; LITERAL "<p>"; LITERAL "<p>"; LITERAL "<p>"; LITERAL "<p>"; LITERAL "<p>" + LITERAL "<p>"; LITERAL " "; LITERAL "</p>"; LITERAL "</p>"; LITERAL "</p>"; LITERAL "</p>" + LITERAL "</p>"; LITERAL "</p>"; LITERAL "</p>"; ENDLINE] "Half opened tag should just be outputted", "<a><", - [HTMLLIT "<a>"; HTMLLIT "<"; ENDLINE] + [LITERAL "<a>"; LITERAL "<"; ENDLINE] "Half opened with text after should be as expected", "<a><This text should appear as normal", - [HTMLLIT "<a>"; HTMLLIT "<This text should appear as normal"; ENDLINE] + [LITERAL "<a>"; LITERAL "<This text should appear as normal"; ENDLINE] "Wrong html close tag should be passed through", "<p></>s", - [HTMLLIT "<p>"; HTMLLIT "</>s"; ENDLINE] + [LITERAL "<p>"; LITERAL "</>s"; ENDLINE] ] /// Tests for the complete lexers with a string list as input @@ -384,9 +384,9 @@ let lexListTest = "This should not, <span>This should not be tokenized []</span>"], [LITERAL "This"; WHITESPACE 1; LITERAL "should"; WHITESPACE 1; LITERAL "not" WHITESPACE 1; LITERAL "be"; WHITESPACE 1; LITERAL "passed"; WHITESPACE 1 - LITERAL "through"; ENDLINE; HTMLLIT "<div>"; HTMLLIT "This should just all be passed through, "; HTMLLIT "</div>" + LITERAL "through"; ENDLINE; LITERAL "<div>"; LITERAL "This should just all be passed through, "; LITERAL "</div>" ENDLINE; LITERAL "This"; WHITESPACE 1; LITERAL "should"; WHITESPACE 1; LITERAL "not"; COMMA; WHITESPACE 1 - HTMLLIT "<span>"; HTMLLIT "This should not be tokenized []"; HTMLLIT "</span>"; ENDLINE] + LITERAL "<span>"; LITERAL "This should not be tokenized []"; LITERAL "</span>"; ENDLINE] ] // -------------------------------------------------- diff --git a/FMark/src/Common/Types.fs b/FMark/src/Common/Types.fs index 46667c9..65ed79b 100644 --- a/FMark/src/Common/Types.fs +++ b/FMark/src/Common/Types.fs @@ -10,7 +10,6 @@ type Language = type Token = | CODEBLOCK of string * Language | LITERAL of string - | HTMLLIT of string | WHITESPACE of size: int | NUMBER of string | HASH | PIPE | EQUAL | MINUS | PLUS | ASTERISK | DOT | COMMA @@ -25,7 +24,6 @@ type TFrmtedString = | Line of InlineElement list | Strike of InlineElement list | Literal of string - | Html of string | Code of string and InlineElement = |