[F#] Fix parsing of long lines

Fixes VSTS #963356

Make Lexer.parseLine tail recursive and remove mutable state

4 files changed, 101 insertions, 75 deletions

diff --git a/main/external/fsharpbinding/MonoDevelop.FSharp.Shared/Lexer.fs b/main/external/fsharpbinding/MonoDevelop.FSharp.Shared/Lexer.fs
index d25d4f643f..5caeafaa6b 100644
--- a/main/external/fsharpbinding/MonoDevelop.FSharp.Shared/Lexer.fs
+++ b/main/external/fsharpbinding/MonoDevelop.FSharp.Shared/Lexer.fs
@@ -150,22 +150,28 @@ module Lexer =
                     draftToken :: acc, Some draftToken
             ) ([], None)
         |> fst
-    
-    
+
+    let rec parseLine (tokenizer:FSharpLineTokenizer) tokens state =
+      match tokenizer.ScanToken(state) with
+      | Some tok, state ->
+          parseLine tokenizer (tok::tokens) state
+      | None, state -> tokens, state
+
+    let rec parseLines (sourceTok:FSharpSourceTokenizer) tokens state lines filename defines =
+        [ match lines with
+          | line::lines ->
+              // Create tokenizer & tokenize single line
+              let tokenizer = sourceTok.CreateLineTokenizer(line)
+              let tokens, state = parseLine tokenizer [] state
+              yield tokens, line
+              // Tokenize the rest of the lines using the new state
+              yield! parseLines sourceTok tokens state lines filename defines
+          | [] -> () ]
+
     let getTokensWithInitialState state lines filename defines =
-        [ let mutable state = state
-          let sourceTok = FSharpSourceTokenizer(defines, filename)
-          for lineText in lines do
-              let tokenizer = sourceTok.CreateLineTokenizer(lineText)
-              let rec parseLine() =
-                  [ match tokenizer.ScanToken(state) with
-                    | Some(tok), nstate ->
-                        state <- nstate
-                        yield tok
-                        yield! parseLine()
-                    | None, nstate -> state <- nstate ]
-              yield parseLine(), lineText ]
-    
+        let sourceTok = FSharpSourceTokenizer(defines, filename)
+        parseLines sourceTok [] state lines filename defines
+
     let findTokenAt col (tokens:FSharpTokenInfo list) =
         let isTokenAtOffset col (t:FSharpTokenInfo) = col-1 >= t.LeftColumn && col-1 <= t.RightColumn
         tokens |> List.tryFindBack (isTokenAtOffset col)
diff --git a/main/external/fsharpbinding/MonoDevelop.FSharp.Tests/LexerTests.fs b/main/external/fsharpbinding/MonoDevelop.FSharp.Tests/LexerTests.fs
new file mode 100644
index 0000000000..6123c021ae
--- /dev/null
+++ b/main/external/fsharpbinding/MonoDevelop.FSharp.Tests/LexerTests.fs
@@ -0,0 +1,16 @@
+namespace MonoDevelopTests
+
+open System
+open NUnit.Framework
+open MonoDevelop.FSharp.Shared
+open FSharp.Compiler.SourceCodeServices
+
+[<TestFixture>]
+module LexerTests =
+    [<Test>]
+    let ``can parse long line``() =
+        let line = sprintf "let x = \"%s\"" (String('*', 10000000))
+        let sourceTok = FSharpSourceTokenizer([], None)
+        let tokenizer = sourceTok.CreateLineTokenizer line
+        let tokens, state = Lexer.parseLine tokenizer [] FSharpTokenizerLexState.Initial
+        Assert.AreNotEqual(state, FSharpTokenizerLexState.Initial)
\ No newline at end of file
diff --git a/main/external/fsharpbinding/MonoDevelop.FSharp.Tests/MonoDevelop.FSharp.Tests.fsproj b/main/external/fsharpbinding/MonoDevelop.FSharp.Tests/MonoDevelop.FSharp.Tests.fsproj
index f9d5af1870..11dcb89113 100644
--- a/main/external/fsharpbinding/MonoDevelop.FSharp.Tests/MonoDevelop.FSharp.Tests.fsproj
+++ b/main/external/fsharpbinding/MonoDevelop.FSharp.Tests/MonoDevelop.FSharp.Tests.fsproj
@@ -73,6 +73,21 @@
   </ItemGroup>
   <Import Project="$(CustomBeforeMicrosoftCommonTargets)" />
   <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets" />
+  <Import Project="..\.paket\paket.targets" />
+  <ProjectExtensions>
+    <MonoDevelop>
+      <Properties>
+        <Policies>
+          <TextStylePolicy TabWidth="4" IndentWidth="4" RemoveTrailingWhitespace="True" NoTabsAfterNonTabs="False" EolMarker="Native" FileWidth="80" TabsToSpaces="True" scope="text/x-fsharp" />
+          <FSharpFormattingPolicy scope="text/x-fsharp">
+            <DefaultFormat IndentOnTryWith="False" ReorderOpenDeclaration="False" SpaceAfterComma="True" SpaceAfterSemicolon="True" SpaceAroundDelimiter="True" SpaceBeforeArgument="True" SpaceBeforeColon="True" __added="0" />
+          </FSharpFormattingPolicy>
+          <TextStylePolicy inheritsSet="null" scope="application/fsproject+xml" />
+          <XmlFormattingPolicy inheritsSet="null" scope="application/fsproject+xml" />
+        </Policies>
+      </Properties>
+    </MonoDevelop>
+  </ProjectExtensions>
   <ItemGroup>
     <Compile Include="TestBase.fs" />
     <Compile Include="TestDocument.fs" />
@@ -105,86 +120,70 @@
       <Link>MonoDevelop.FSharpInteractive.Service.exe</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
-    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.Debugger\MonoDevelop.Debugger.csproj">
-      <Project>{2357AABD-08C7-4808-A495-8FF2D3CDFDB0}</Project>
-      <Name>MonoDevelop.Debugger</Name>
+    <IncludeCopyLocal Include="System.Reactive.dll" />
+    <IncludeCopyLocal Include="Newtonsoft.Json.dll" />
+    <IncludeCopyLocal Include="ExtCore.dll" />
+    <IncludeCopyLocal Include="Fantomas.dll" />
+    <IncludeCopyLocal Include="FSharp.Compiler.CodeDom.dll" />
+    <IncludeCopyLocal Include="FSharp.Compiler.Service.dll" />
+    <IncludeCopyLocal Include="FSharp.Core.dll" />
+    <IncludeCopyLocal Include="MonoDevelop.FSharp.Shared.dll" />
+    <IncludeCopyLocal Include="FSharp.Compiler.Interactive.Settings.dll" />
+    <Compile Include="LexerTests.fs" />
+    <ProjectReference Include="..\..\..\tests\UnitTests\UnitTests.csproj">
+      <Project>{1497D0A8-AFF1-4938-BC22-BE79B358BA5B}</Project>
+      <Name>UnitTests</Name>
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\..\src\core\MonoDevelop.Ide\MonoDevelop.Ide.csproj">
-      <Project>{27096E7F-C91C-4AC6-B289-6897A701DF21}</Project>
-      <Name>MonoDevelop.Ide</Name>
+    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.Refactoring\MonoDevelop.Refactoring.csproj">
+      <Project>{100568FC-F4E8-439B-94AD-41D11724E45B}</Project>
+      <Name>MonoDevelop.Refactoring</Name>
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\MonoDevelop.FSharpBinding\MonoDevelop.FSharp.fsproj">
-      <Project>{4C10F8F9-3816-4647-BA6E-85F5DE39883A}</Project>
-      <Name>MonoDevelop.FSharp</Name>
+    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.csproj">
+      <Project>{F218643D-2E74-4309-820E-206A54B7133F}</Project>
+      <Name>MonoDevelop.PackageManagement</Name>
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.UnitTesting\MonoDevelop.UnitTesting.csproj">
-      <Project>{A7A4246D-CEC4-42DF-A3C1-C31B9F51C4EC}</Project>
-      <Name>MonoDevelop.UnitTesting</Name>
-      <Private>False</Private>
+    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.Tests\MonoDevelop.PackageManagement.Tests.csproj">
+      <Project>{2645C9F3-9ED5-4806-AB09-DAD9BE90C67B}</Project>
+      <Name>MonoDevelop.PackageManagement.Tests</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\MonoDevelop.FSharp.Shared\MonoDevelop.FSharp.Shared.fsproj">
+      <Project>{AF5FEAD5-B50E-4F07-A274-32F23D5C504D}</Project>
+      <Name>MonoDevelop.FSharp.Shared</Name>
     </ProjectReference>
     <ProjectReference Include="..\..\..\src\addins\MonoDevelop.SourceEditor2\MonoDevelop.SourceEditor.csproj">
       <Project>{F8F92AA4-A376-4679-A9D4-60E7B7FBF477}</Project>
       <Name>MonoDevelop.SourceEditor</Name>
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\MonoDevelop.FSharp.Shared\MonoDevelop.FSharp.Shared.fsproj">
-      <Project>{AF5FEAD5-B50E-4F07-A274-32F23D5C504D}</Project>
-      <Name>MonoDevelop.FSharp.Shared</Name>
-    </ProjectReference>
-    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.csproj">
-      <Project>{F218643D-2E74-4309-820E-206A54B7133F}</Project>
-      <Name>MonoDevelop.PackageManagement</Name>
+    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.UnitTesting\MonoDevelop.UnitTesting.csproj">
+      <Project>{A7A4246D-CEC4-42DF-A3C1-C31B9F51C4EC}</Project>
+      <Name>MonoDevelop.UnitTesting</Name>
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.Tests\MonoDevelop.PackageManagement.Tests.csproj">
-      <Project>{2645C9F3-9ED5-4806-AB09-DAD9BE90C67B}</Project>
-      <Name>MonoDevelop.PackageManagement.Tests</Name>
+    <ProjectReference Include="..\MonoDevelop.FSharpBinding\MonoDevelop.FSharp.fsproj">
+      <Project>{4C10F8F9-3816-4647-BA6E-85F5DE39883A}</Project>
+      <Name>MonoDevelop.FSharp</Name>
+      <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\..\src\core\MonoDevelop.Core\MonoDevelop.Core.csproj">
-      <Project>{7525BB88-6142-4A26-93B9-A30C6983390A}</Project>
-      <Name>MonoDevelop.Core</Name>
+    <ProjectReference Include="..\..\..\src\core\MonoDevelop.Ide\MonoDevelop.Ide.csproj">
+      <Project>{27096E7F-C91C-4AC6-B289-6897A701DF21}</Project>
+      <Name>MonoDevelop.Ide</Name>
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\..\tests\UnitTests\UnitTests.csproj">
-      <Project>{1497D0A8-AFF1-4938-BC22-BE79B358BA5B}</Project>
-      <Name>UnitTests</Name>
+    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.Debugger\MonoDevelop.Debugger.csproj">
+      <Project>{2357AABD-08C7-4808-A495-8FF2D3CDFDB0}</Project>
+      <Name>MonoDevelop.Debugger</Name>
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\..\src\addins\MonoDevelop.Refactoring\MonoDevelop.Refactoring.csproj">
-      <Project>{100568FC-F4E8-439B-94AD-41D11724E45B}</Project>
-      <Name>MonoDevelop.Refactoring</Name>
+    <ProjectReference Include="..\..\..\src\core\MonoDevelop.Core\MonoDevelop.Core.csproj">
+      <Project>{7525BB88-6142-4A26-93B9-A30C6983390A}</Project>
+      <Name>MonoDevelop.Core</Name>
       <Private>False</Private>
     </ProjectReference>
   </ItemGroup>
-  <Import Project="..\.paket\paket.targets" />
-  <ProjectExtensions>
-    <MonoDevelop>
-      <Properties>
-        <Policies>
-          <TextStylePolicy TabWidth="4" IndentWidth="4" RemoveTrailingWhitespace="True" NoTabsAfterNonTabs="False" EolMarker="Native" FileWidth="80" TabsToSpaces="True" scope="text/x-fsharp" />
-          <FSharpFormattingPolicy scope="text/x-fsharp">
-            <DefaultFormat IndentOnTryWith="False" ReorderOpenDeclaration="False" SpaceAfterComma="True" SpaceAfterSemicolon="True" SpaceAroundDelimiter="True" SpaceBeforeArgument="True" SpaceBeforeColon="True" __added="0" />
-          </FSharpFormattingPolicy>
-          <TextStylePolicy inheritsSet="null" scope="application/fsproject+xml" />
-          <XmlFormattingPolicy inheritsSet="null" scope="application/fsproject+xml" />
-        </Policies>
-      </Properties>
-    </MonoDevelop>
-  </ProjectExtensions>
-  <ItemGroup>
-    <IncludeCopyLocal Include="System.Reactive.dll" />
-    <IncludeCopyLocal Include="Newtonsoft.Json.dll" />
-    <IncludeCopyLocal Include="ExtCore.dll" />
-    <IncludeCopyLocal Include="Fantomas.dll" />
-    <IncludeCopyLocal Include="FSharp.Compiler.CodeDom.dll" />
-    <IncludeCopyLocal Include="FSharp.Compiler.Service.dll" />
-    <IncludeCopyLocal Include="FSharp.Core.dll" />
-    <IncludeCopyLocal Include="MonoDevelop.FSharp.Shared.dll" />
-    <IncludeCopyLocal Include="FSharp.Compiler.Interactive.Settings.dll" />
-  </ItemGroup>
   <Choose>
     <When Condition="$(TargetFrameworkIdentifier) == '.NETFramework' And $(TargetFrameworkVersion) == 'v4.7.2'">
       <ItemGroup>
diff --git a/main/external/fsharpbinding/MonoDevelop.FSharpBinding/FSharpTokens.fs b/main/external/fsharpbinding/MonoDevelop.FSharpBinding/FSharpTokens.fs
index ae30dd34e9..dfd67ed9d3 100644
--- a/main/external/fsharpbinding/MonoDevelop.FSharpBinding/FSharpTokens.fs
+++ b/main/external/fsharpbinding/MonoDevelop.FSharpBinding/FSharpTokens.fs
@@ -22,12 +22,17 @@ module Tokens =
         match token with 
         | Some token -> MonoDevelop.FSharp.Shared.Lexer.isNonTipToken token
         | None -> false
-                
+
+
+    //(FSharpTokenInfo list * string) list option         
     let tryGetTokens source defines fileName =
         try
             LoggingService.logDebug "FSharpParser: Processing tokens for %s" (Path.GetFileName fileName)
             let readOnlyDoc = TextEditorFactory.CreateNewReadonlyDocument (source, fileName)
-            let lines = readOnlyDoc.GetLines() |> Seq.map readOnlyDoc.GetLineText
+            let lines = readOnlyDoc.GetLines() 
+                        |> Seq.map readOnlyDoc.GetLineText
+                        |> List.ofSeq
+
             let tokens = MonoDevelop.FSharp.Shared.Lexer.getTokensWithInitialState FSharpTokenizerLexState.Initial lines (Some fileName) defines
             Some(tokens)
         with ex ->