Rewrote LaTeX reader with proper tokenization.

This rewrite is primarily motivated by the need to
get macros working properly.  A side benefit is that the
reader is significantly faster (27s -> 19s in one
benchmark, and there is a lot of room for further
optimization).

We now tokenize the input text, then parse the token stream.

Macros modify the token stream, so they should now be effective
in any context, including math. Thus, we no longer need the clunky
macro processing capacities of texmath.

A custom state LaTeXState is used instead of ParserState.
This, plus the tokenization, will require some rewriting
of the exported functions rawLaTeXInline, inlineCommand,
rawLaTeXBlock.

* Added Text.Pandoc.Readers.LaTeX.Types (new exported module).
  Exports Macro, Tok, TokType, Line, Column.  [API change]
* Text.Pandoc.Parsing: adjusted type of `insertIncludedFile`
  so it can be used with token parser.
* Removed old texmath macro stuff from Parsing.
  Use Macro from Text.Pandoc.Readers.LaTeX.Types instead.
* Removed texmath macro material from Markdown reader.
* Changed types for Text.Pandoc.Readers.LaTeX's
  rawLaTeXInline and rawLaTeXBlock.  (Both now return a String,
  and they are polymorphic in state.)
* Added orgMacros field to OrgState.  [API change]
* Removed readerApplyMacros from ReaderOptions.
  Now we just check the `latex_macros` reader extension.
* Allow `\newcommand\foo{blah}` without braces.

Fixes #1390.
Fixes #2118.
Fixes #3236.
Fixes #3779.
Fixes #934.
Fixes #982.

2 files changed, 13 insertions, 2 deletions

diff --git a/src/Text/Pandoc/Readers/Org/Inlines.hs b/src/Text/Pandoc/Readers/Org/Inlines.hs
index 66273e05d..42fdfd4dd 100644
--- a/src/Text/Pandoc/Readers/Org/Inlines.hs
+++ b/src/Text/Pandoc/Readers/Org/Inlines.hs
@@ -826,9 +826,10 @@ maybeRight = either (const Nothing) Just
 inlineLaTeXCommand :: PandocMonad m => OrgParser m String
 inlineLaTeXCommand = try $ do
   rest <- getInput
-  parsed <- (lift . lift) $ runParserT rawLaTeXInline def "source" rest
+  st <- getState
+  parsed <- (lift . lift) $ runParserT rawLaTeXInline st "source" rest
   case parsed of
-    Right (RawInline _ cs) -> do
+    Right cs -> do
       -- drop any trailing whitespace, those are not be part of the command as
       -- far as org mode is concerned.
       let cmdNoSpc = dropWhileEnd isSpace cs
diff --git a/src/Text/Pandoc/Readers/Org/ParserState.hs b/src/Text/Pandoc/Readers/Org/ParserState.hs
index 92f868516..fc98213fb 100644
--- a/src/Text/Pandoc/Readers/Org/ParserState.hs
+++ b/src/Text/Pandoc/Readers/Org/ParserState.hs
@@ -33,6 +33,7 @@ module Text.Pandoc.Readers.Org.ParserState
   , OrgNoteRecord
   , HasReaderOptions (..)
   , HasQuoteContext (..)
+  , HasMacros (..)
   , TodoMarker (..)
   , TodoSequence
   , TodoState (..)
@@ -57,14 +58,17 @@ import Control.Monad.Reader (ReaderT, asks, local)
 import Data.Default (Default (..))
 import qualified Data.Map as M
 import qualified Data.Set as Set
+import Data.Text (Text)
 
 import Text.Pandoc.Builder (Blocks, Inlines)
 import Text.Pandoc.Definition (Meta (..), nullMeta)
 import Text.Pandoc.Logging
 import Text.Pandoc.Options (ReaderOptions (..))
+import Text.Pandoc.Readers.LaTeX.Types (Macro)
 import Text.Pandoc.Parsing (Future, HasHeaderMap (..), HasIdentifierList (..),
                             HasIncludeFiles (..), HasLastStrPosition (..),
                             HasLogMessages (..), HasQuoteContext (..),
+                            HasMacros (..),
                             HasReaderOptions (..), ParserContext (..),
                             QuoteContext (..), SourcePos, askF, asksF, returnF,
                             runF, trimInlinesF)
@@ -118,6 +122,7 @@ data OrgParserState = OrgParserState
   , orgStateParserContext        :: ParserContext
   , orgStateTodoSequences        :: [TodoSequence]
   , orgLogMessages               :: [LogMessage]
+  , orgMacros                    :: M.Map Text Macro
   }
 
 data OrgParserLocal = OrgParserLocal { orgLocalQuoteContext :: QuoteContext }
@@ -148,6 +153,10 @@ instance HasLogMessages OrgParserState where
   addLogMessage msg st = st{ orgLogMessages = msg : orgLogMessages st }
   getLogMessages st = reverse $ orgLogMessages st
 
+instance HasMacros OrgParserState where
+  extractMacros st = orgMacros st
+  updateMacros f st = st{ orgMacros = f (orgMacros st) }
+
 instance HasIncludeFiles OrgParserState where
   getIncludeFiles = orgStateIncludeFiles
   addIncludeFile f st = st { orgStateIncludeFiles = f : orgStateIncludeFiles st }
@@ -178,6 +187,7 @@ defaultOrgParserState = OrgParserState
   , orgStateParserContext = NullState
   , orgStateTodoSequences = []
   , orgLogMessages = []
+  , orgMacros = M.empty
   }
 
 optionsToParserState :: ReaderOptions -> OrgParserState