Add `empty_paragraphs` extension.

* Deprecate `--strip-empty-paragraphs` option.  Instead we now
  use an `empty_paragraphs` extension that can be enabled on
  the reader or writer.  By default, disabled.

* Add `Ext_empty_paragraphs` constructor to `Extension`.

* Revert "Docx reader: don't strip out empty paragraphs."
  This reverts commit d6c58eb836f033a48955796de4d9ffb3b30e297b.

* Implement `empty_paragraphs` extension in docx reader and writer,
  opendocument writer, html reader and writer.

* Add tests for `empty_paragraphs` extension.

7 files changed, 115 insertions, 46 deletions

diff --git a/test/Tests/Readers/Docx.hs b/test/Tests/Readers/Docx.hs
index b5d7aa430..421acaa8b 100644
--- a/test/Tests/Readers/Docx.hs
+++ b/test/Tests/Readers/Docx.hs
@@ -10,7 +10,6 @@ import Test.Tasty
 import Test.Tasty.HUnit
 import Tests.Helpers
 import Text.Pandoc
-import Text.Pandoc.Shared (stripEmptyParagraphs)
 import qualified Text.Pandoc.Class as P
 import Text.Pandoc.MediaBag (MediaBag, lookupMedia, mediaDirectory)
 import Text.Pandoc.UTF8 as UTF8
@@ -38,23 +37,20 @@ instance ToString NoNormPandoc where
 instance ToPandoc NoNormPandoc where
   toPandoc = unNoNorm
 
-compareOutput :: Bool
-              -> ReaderOptions
-              -> FilePath
-              -> FilePath
-              -> IO (NoNormPandoc, NoNormPandoc)
-compareOutput strip opts docxFile nativeFile = do
+compareOutput :: ReaderOptions
+                 -> FilePath
+                 -> FilePath
+                 -> IO (NoNormPandoc, NoNormPandoc)
+compareOutput opts docxFile nativeFile = do
   df <- B.readFile docxFile
   nf <- UTF8.toText <$> BS.readFile nativeFile
   p <- runIOorExplode $ readDocx opts df
   df' <- runIOorExplode $ readNative def nf
-  return $ (noNorm (if strip
-                       then stripEmptyParagraphs p
-                       else p), noNorm df')
+  return $ (noNorm p, noNorm df')
 
 testCompareWithOptsIO :: ReaderOptions -> String -> FilePath -> FilePath -> IO TestTree
 testCompareWithOptsIO opts name docxFile nativeFile = do
-  (dp, np) <- compareOutput True opts docxFile nativeFile
+  (dp, np) <- compareOutput opts docxFile nativeFile
   return $ test id name (dp, np)
 
 testCompareWithOpts :: ReaderOptions -> String -> FilePath -> FilePath -> TestTree
@@ -75,11 +71,6 @@ testForWarningsWithOpts :: ReaderOptions -> String -> FilePath -> [String] -> Te
 testForWarningsWithOpts opts name docxFile expected =
   unsafePerformIO $ testForWarningsWithOptsIO opts name docxFile expected
 
-testCompareNoStrip :: String -> FilePath -> FilePath -> TestTree
-testCompareNoStrip name docxFile nativeFile = unsafePerformIO $ do
-  (dp, np) <- compareOutput False defopts docxFile nativeFile
-  return $ test id name (dp, np)
-
 -- testForWarnings :: String -> FilePath -> [String] -> TestTree
 -- testForWarnings = testForWarningsWithOpts defopts
 
@@ -266,10 +257,6 @@ tests = [ testGroup "inlines"
             "dropcap paragraphs"
             "docx/drop_cap.docx"
             "docx/drop_cap.native"
-          , testCompareNoStrip
-            "empty paragraphs without stripping"
-            "docx/drop_cap.docx"
-            "docx/drop_cap_nostrip.native"
           ]
         , testGroup "track changes"
           [ testCompare
diff --git a/test/command/2649.md b/test/command/2649.md
index af84693c4..8f594cfe1 100644
--- a/test/command/2649.md
+++ b/test/command/2649.md
@@ -88,20 +88,17 @@
 </tr>
 <tr class="even">
 <td><p>1</p></td>
-<td>
-<p><a href="Sébastien_Loeb" title="wikilink">Sébastien Loeb</a></p></td>
+<td><p><a href="Sébastien_Loeb" title="wikilink">Sébastien Loeb</a></p></td>
 <td><p>78</p></td>
 </tr>
 <tr class="odd">
 <td><p>2</p></td>
-<td>
-<p><strong><a href="Sébastien_Ogier" title="wikilink">Sébastien Ogier</a></strong></p></td>
+<td><p><strong><a href="Sébastien_Ogier" title="wikilink">Sébastien Ogier</a></strong></p></td>
 <td><p>38</p></td>
 </tr>
 <tr class="even">
 <td><p>10</p></td>
-<td>
-<p><a href="Hannu_Mikkola" title="wikilink">Hannu Mikkola</a></p></td>
+<td><p><a href="Hannu_Mikkola" title="wikilink">Hannu Mikkola</a></p></td>
 <td><p>18</p></td>
 </tr>
 </tbody>
diff --git a/test/command/3494.md b/test/command/3494.md
index 534041246..249973fb3 100644
--- a/test/command/3494.md
+++ b/test/command/3494.md
@@ -25,8 +25,7 @@
 <td style="text-align: left;">thank you</td>
 </tr>
 <tr class="odd">
-<td style="text-align: right;">
-<p><em>blah</em></p></td>
+<td style="text-align: right;"><p><em>blah</em></p></td>
 <td style="text-align: left;"><em>blah</em></td>
 <td style="text-align: left;"><em>blah</em></td>
 </tr>
diff --git a/test/command/empty_paragraphs.md b/test/command/empty_paragraphs.md
new file mode 100644
index 000000000..3064d3f7d
--- /dev/null
+++ b/test/command/empty_paragraphs.md
@@ -0,0 +1,95 @@
+```
+% pandoc -f native -t docx | pandoc -f docx -t native
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+[Para [Str "hi"]
+,Para [Str "lo"]]
+```
+
+```
+% pandoc -f native -t docx+empty_paragraphs | pandoc -f docx -t native
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+[Para [Str "hi"]
+,Para [Str "lo"]]
+```
+
+```
+% pandoc -f native -t docx | pandoc -f docx+empty_paragraphs -t native
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+[Para [Str "hi"]
+,Para [Str "lo"]]
+```
+
+```
+% pandoc -f native -t docx+empty_paragraphs | pandoc -f docx+empty_paragraphs -t native
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+[Para [Str "hi"]
+,Para []
+,Para []
+,Para [Str "lo"]]
+```
+
+```
+% pandoc -f native -t html5
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+<p>hi</p>
+
+
+<p>lo</p>
+```
+
+```
+% pandoc -f native -t html5+empty_paragraphs
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+<p>hi</p>
+<p></p>
+<p></p>
+<p>lo</p>
+```
+
+```
+% pandoc -f html+empty_paragraphs -t native
+<p>hi</p>
+<p></p>
+<p></p>
+<p>lo</p>
+^D
+[Para [Str "hi"]
+,Para []
+,Para []
+,Para [Str "lo"]]
+```
+
+```
+% pandoc -f html -t native
+<p>hi</p>
+<p></p>
+<p></p>
+<p>lo</p>
+^D
+[Para [Str "hi"]
+,Para [Str "lo"]]
+```
+
+```
+% pandoc -f native -t opendocument+empty_paragraphs
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+<text:p text:style-name="Text_20_body">hi</text:p>
+<text:p text:style-name="Text_20_body"></text:p>
+<text:p text:style-name="Text_20_body"></text:p>
+<text:p text:style-name="Text_20_body">lo</text:p>
+```
+
+```
+% pandoc -f native -t opendocument
+[Para [Str "hi"], Para [], Para [], Para [Str "lo"]]
+^D
+<text:p text:style-name="Text_20_body">hi</text:p>
+<text:p text:style-name="Text_20_body">lo</text:p>
+```
diff --git a/test/docx/0_level_headers.native b/test/docx/0_level_headers.native
index 973f4cfe5..804ad8732 100644
--- a/test/docx/0_level_headers.native
+++ b/test/docx/0_level_headers.native
@@ -1,15 +1,15 @@
 [Table [] [AlignDefault] [0.0]
  [[]]
- [[[Plain []]]
+ [[[]]
  ,[[Plain [Str "User\8217s",Space,Str "Guide"]]]
- ,[[Plain []]]
- ,[[Plain []]]
- ,[[Plain []]]
+ ,[[]]
+ ,[[]]
+ ,[[]]
  ,[[Plain [Str "11",Space,Str "August",Space,Str "2017"]]]
- ,[[Plain []]]
- ,[[Plain []]]
- ,[[Plain []]]
- ,[[Plain []]]]
+ ,[[]]
+ ,[[]]
+ ,[[]]
+ ,[[]]]
 ,Para [Str "CONTENTS"]
 ,Para [Strong [Str "Section",Space,Str "Page"]]
 ,Para [Str "FIGURES",Space,Str "iv"]
diff --git a/test/docx/comments.native b/test/docx/comments.native
index 8587b54dd..3357bc257 100644
--- a/test/docx/comments.native
+++ b/test/docx/comments.native
@@ -1,4 +1,4 @@
 [Para [Str "I",Space,Str "want",Space,Span ("",["comment-start"],[("id","0"),("author","Jesse Rosenthal"),("date","2016-05-09T16:13:00Z")]) [Str "I",Space,Str "left",Space,Str "a",Space,Str "comment."],Str "some",Space,Str "text",Space,Str "to",Space,Str "have",Space,Str "a",Space,Str "comment",Space,Span ("",["comment-end"],[("id","0")]) [],Str "on",Space,Str "it."]
 ,Para [Str "This",Space,Str "is",Space,Span ("",["comment-start"],[("id","1"),("author","Jesse Rosenthal"),("date","2016-05-09T16:13:00Z")]) [Str "A",Space,Str "comment",Space,Str "across",Space,Str "paragraphs."],Str "a",Space,Str "new",Space,Str "paragraph."]
 ,Para [Str "And",Space,Str "so",Span ("",["comment-end"],[("id","1")]) [],Space,Str "is",Space,Str "this."]
-,Para [Str "One",Space,Span ("",["comment-start"],[("id","2"),("author","Jesse Rosenthal"),("date","2016-05-09T16:14:00Z")]) [Str "This",Space,Str "one",Space,Str "has",Space,Str "multiple",Space,Str "paragraphs.",Space,Str "\182",Space,Str "\182",Space,Str "See?"],Str "more",Span ("",["comment-end"],[("id","2")]) [],Str ".",Space,Str "And",Space,Str "this",Space,Str "is",Space,Str "one",Space,Str "with",Space,Str "a",Space,Span ("",["comment-start"],[("id","3"),("author","Jesse Rosenthal"),("date","2016-06-22T14:35:00Z")]) [Str "Do",Space,Str "something."],Span ("",["comment-start"],[("id","4"),("author","Jesse Rosenthal"),("date","2016-06-22T14:36:00Z")]) [Str "Do",Space,Str "something",Space,Str "else."],Str "comment",Space,Str "in",Space,Str "a",Space,Str "comment",Span ("",["comment-end"],[("id","3")]) [Span ("",["comment-end"],[("id","4")]) []],Str "."]]
+,Para [Str "One",Space,Span ("",["comment-start"],[("id","2"),("author","Jesse Rosenthal"),("date","2016-05-09T16:14:00Z")]) [Str "This",Space,Str "one",Space,Str "has",Space,Str "multiple",Space,Str "paragraphs.",Space,Str "\182",Space,Str "See?"],Str "more",Span ("",["comment-end"],[("id","2")]) [],Str ".",Space,Str "And",Space,Str "this",Space,Str "is",Space,Str "one",Space,Str "with",Space,Str "a",Space,Span ("",["comment-start"],[("id","3"),("author","Jesse Rosenthal"),("date","2016-06-22T14:35:00Z")]) [Str "Do",Space,Str "something."],Span ("",["comment-start"],[("id","4"),("author","Jesse Rosenthal"),("date","2016-06-22T14:36:00Z")]) [Str "Do",Space,Str "something",Space,Str "else."],Str "comment",Space,Str "in",Space,Str "a",Space,Str "comment",Span ("",["comment-end"],[("id","3")]) [Span ("",["comment-end"],[("id","4")]) []],Str "."]]
diff --git a/test/docx/drop_cap_nostrip.native b/test/docx/drop_cap_nostrip.native
deleted file mode 100644
index ea6a244b5..000000000
--- a/test/docx/drop_cap_nostrip.native
+++ /dev/null
@@ -1,9 +0,0 @@
-[Para [Str "Drop",Space,Str "cap."]
-,Para []
-,Para [Str "Next",Space,Str "paragraph."]
-,Para []
-,Para [Str "Drop",Space,Str "cap",Space,Str "in",Space,Str "margin."]
-,Para []
-,Para []
-,Para []
-,Para [Str "Drop",Space,Str "cap",Space,Str "(not",Space,Str "really)."]]