From fdf4cac2439a7168bd005efbde4a1f76a00ada66 Mon Sep 17 00:00:00 2001 From: Bernhard Schommer Date: Mon, 21 Mar 2016 09:28:09 +0100 Subject: Also ignore windows line endings. Windows style line endings can end up in the Tokenize pass. This can lead to some problems for example in pragma processing. Bug 18316 --- lib/Tokenize.mll | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/Tokenize.mll') diff --git a/lib/Tokenize.mll b/lib/Tokenize.mll index 422068b1..70e21d55 100644 --- a/lib/Tokenize.mll +++ b/lib/Tokenize.mll @@ -20,7 +20,7 @@ let identcont = [ '0'-'9' 'A'-'Z' 'a'-'z' '$' '_' '-' '.' ] rule tokenize acc = parse | eof { List.rev acc } - | [' ' '\t' '\n'] + { tokenize acc lexbuf } + | [' ' '\t' '\n' '\r'] + { tokenize acc lexbuf } | "\"" { tok_dquote acc (Buffer.create 16) lexbuf } | "'" { tok_squote acc (Buffer.create 16) lexbuf } | (identstart identcont*) as s @@ -31,6 +31,7 @@ and tok_dquote acc buf = parse | "\"" | eof { tokenize (Buffer.contents buf :: acc) lexbuf } | "\\t" { Buffer.add_char buf '\t'; tok_dquote acc buf lexbuf } | "\\n" { Buffer.add_char buf '\n'; tok_dquote acc buf lexbuf } + | "\\r" { Buffer.add_char buf '\r'; tok_dquote acc buf lexbuf } | "\\" ([ '\\' '\"' ] as c) { Buffer.add_char buf c; tok_dquote acc buf lexbuf } | _ as c { Buffer.add_char buf c; tok_dquote acc buf lexbuf } -- cgit