Also ignore windows line endings.

Windows style line endings can end up in the Tokenize pass. This
can lead to some problems for example in pragma processing.
Bug 18316

1 files changed, 2 insertions, 1 deletions

diff --git a/lib/Tokenize.mll b/lib/Tokenize.mll
index 422068b1..70e21d55 100644
--- a/lib/Tokenize.mll
+++ b/lib/Tokenize.mll
@@ -20,7 +20,7 @@ let identcont  = [ '0'-'9' 'A'-'Z' 'a'-'z' '$' '_' '-' '.' ]
 
 rule tokenize acc = parse
   | eof                 { List.rev acc }
-  | [' ' '\t' '\n'] +   { tokenize acc lexbuf }
+  | [' ' '\t' '\n' '\r'] +   { tokenize acc lexbuf }
   | "\""                { tok_dquote acc (Buffer.create 16) lexbuf }
   | "'"                 { tok_squote acc (Buffer.create 16) lexbuf }
   | (identstart identcont*) as s
@@ -31,6 +31,7 @@ and tok_dquote acc buf = parse
   | "\"" | eof          { tokenize (Buffer.contents buf :: acc) lexbuf }
   | "\\t"               { Buffer.add_char buf '\t'; tok_dquote acc buf lexbuf }
   | "\\n"               { Buffer.add_char buf '\n'; tok_dquote acc buf lexbuf }
+  | "\\r"               { Buffer.add_char buf '\r'; tok_dquote acc buf lexbuf }
   | "\\" ([ '\\' '\"' ] as c)
                         { Buffer.add_char buf c; tok_dquote acc buf lexbuf }
   | _ as c              { Buffer.add_char buf c; tok_dquote acc buf lexbuf }