tdf#35798: Hide empty Database fields' paragraphs (+ compat option)

With this change, Database fields that expand to empty values behave
as if they are "Hidden Paragraph" fields.

A compatibility option to enable this behaviour is added. The option is
enabled by default, and for any non-native documents (for compatibility
with other office suites). For existing (F)ODT documents, the option is
disabled for those documents that don't have this setting set, to keep
the layout of legacy documents.

Change-Id: Ic5e8cb15a3a7d1a765a984eef4b0d97666df7dfd
Reviewed-on: https://gerrit.libreoffice.org/54552Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>

filter/source/xmlfilteradaptor/XmlFilterAdaptor.cxx

@@ -90,12 +90,25 @@ bool XmlFilterAdaptor::importImpl( const Sequence< css::beans::PropertyValue >&
     PropertyMapEntry aImportInfoMap[] =
     {
         { OUString("BaseURI"), 0, ::cppu::UnoType<OUString>::get(), PropertyAttribute::MAYBEVOID, 0},
+        { OUString("DefaultDocumentSettings"), 0,
+          ::cppu::UnoType<Sequence<PropertyValue>>::get(), PropertyAttribute::MAYBEVOID, 0 },
         { OUString(), 0, css::uno::Type(), 0, 0 }
     };
 
     Reference< XPropertySet > xInfoSet(
             GenericPropertySet_CreateInstance( new PropertySetInfo( aImportInfoMap ) ) );
     xInfoSet->setPropertyValue( "BaseURI", makeAny( aBaseURI ));
+
+    OUString aFilterName;
+    auto It = aMediaMap.find(OUString("FilterName"));
+    if (It != aMediaMap.end() && (It->second >>= aFilterName)
+        && aFilterName == "OpenDocument Text Flat XML")
+    {
+        PropertyValue EmptyDbFieldHidesPara("EmptyDbFieldHidesPara", 0, Any(false),
+                                            PropertyState::PropertyState_DIRECT_VALUE);
+        Sequence<PropertyValue> aSettings{ EmptyDbFieldHidesPara };
+        xInfoSet->setPropertyValue("DefaultDocumentSettings", makeAny(aSettings));
+    }
     aAnys[0] <<= xInfoSet;
 
 

include/sal/log-areas.dox

@@ -519,6 +519,7 @@ certain functionality.
 @li @c sw.level2
 @li @c sw.mailmerge - Writer mail merge
 @li @c sw.pageframe - debug lifecycle of SwPageFrame
+@li @c sw.qa - Writer unit tests
 @li @c sw.rtf - .rtf export filter
 @li @c sw.tiled
 @li @c sw.ui

include/unotools/compatibility.hxx

@@ -60,6 +60,7 @@ class UNOTOOLS_DLLPUBLIC SvtCompatibilityEntry
             ProtectForm,
             MsWordTrailingBlanks,
             SubtractFlysAnchoredAtFlys,
+            EmptyDbFieldHidesPara,
 
             /* Should be at the end. Do not remove it. */
             INVALID

officecfg/registry/schema/org/openoffice/Office/Compatibility.xcs

@@ -122,6 +122,12 @@
         </info>
         <value>false</value>
       </prop>
+      <prop oor:name="EmptyDbFieldHidesPara" oor:type="xs:boolean" oor:nillable="false">
+        <info>
+          <desc>A database field (e.g., MailMerge) with empty value hides its paragraph</desc>
+        </info>
+        <value>true</value>
+      </prop>
     </group>
   </templates>
   <component>

sw/CppunitTest_sw_mailmerge.mk

@@ -45,7 +45,10 @@ $(eval $(call gb_CppunitTest_use_components,sw_mailmerge, \
     dbaccess/util/dba \
     embeddedobj/util/embobj \
     filter/source/config/cache/filterconfig1 \
+    filter/source/odfflatxml/odfflatxml \
     filter/source/storagefilterdetect/storagefd \
+    filter/source/xmlfilteradaptor/xmlfa \
+    filter/source/xmlfilterdetect/xmlfd \
     forms/util/frm \
     framework/util/fwk \
     i18npool/util/i18npool \
@@ -79,6 +82,7 @@ $(eval $(call gb_CppunitTest_use_components,sw_mailmerge, \
         ) \
     ) \
     xmloff/util/xo \
+    xmlscript/util/xmlscript \
 ))
 
 $(eval $(call gb_CppunitTest_use_configuration,sw_mailmerge))

sw/inc/IDocumentSettingAccess.hxx

@@ -83,6 +83,7 @@ enum class DocumentSettingId
     SUBTRACT_FLYS,
     // tdf#112443 disable off-page content positioning
     DISABLE_OFF_PAGE_POSITIONING,
+    EMPTY_DB_FIELD_HIDES_PARA,
     // COMPATIBILITY FLAGS END
     BROWSE_MODE,
     HTML_MODE,

sw/inc/doc.hxx

@@ -466,6 +466,9 @@ public:
 
     ::sw::DocumentFieldsManager & GetDocumentFieldsManager();
 
+    bool FieldCanHidePara(SwFieldIds eFieldId) const;
+    bool FieldHidesPara(const SwField& rField) const;
+
     // IDocumentContentOperations
     IDocumentContentOperations const & getIDocumentContentOperations() const;
     IDocumentContentOperations & getIDocumentContentOperations();

sw/inc/ndhints.hxx

@@ -77,6 +77,8 @@ class SwpHtEnd : public o3tl::sorted_vector<SwTextAttr*, CompareSwpHtEnd,
 class SwpHints
 {
 private:
+    const SwTextNode& m_rParent;
+
     // SAL_MAX_SIZE is used by GetStartOf to return
     // failure, so just allow SAL_MAX_SIZE-1 hints
     static const size_t MAX_HINTS = SAL_MAX_SIZE-1;
@@ -88,9 +90,11 @@ private:
 
     /// true: the Node is in Split and Frames are moved
     bool          m_bInSplitNode         : 1;
-    /// m_bHasHiddenParaField is invalid, call CalcHiddenParaField()
-    bool          m_bCalcHiddenParaField : 1;
-    bool          m_bHasHiddenParaField  : 1;   ///< HiddenParaField
+    // m_bHiddenByParaField is invalid, call CalcHiddenParaField()
+    mutable bool  m_bCalcHiddenParaField : 1;
+    // if all fields controlling visibility of the paragraph require to hide it
+    // (if there's no such fields, or if any field requires to show, then this is false)
+    mutable bool  m_bHiddenByParaField   : 1;
     bool          m_bFootnote            : 1;   ///< footnotes
     bool          m_bDDEFields           : 1;   ///< the TextNode has DDE fields
 
@@ -108,15 +112,15 @@ private:
     void Delete( SwTextAttr const * pTextHt );
 
     void SetInSplitNode(bool bInSplit) { m_bInSplitNode = bInSplit; }
-    void SetCalcHiddenParaField() { m_bCalcHiddenParaField = true; }
-    void SetHiddenParaField( const bool bNew ) { m_bHasHiddenParaField = bNew; }
-    bool HasHiddenParaField() const
+    void SetCalcHiddenParaField() const { m_bCalcHiddenParaField = true; }
+    void SetHiddenByParaField( const bool bNew ) const { m_bHiddenByParaField = bNew; }
+    bool IsHiddenByParaField() const
     {
         if ( m_bCalcHiddenParaField )
         {
-            const_cast<SwpHints*>(this)->CalcHiddenParaField();
+            CalcHiddenParaField();
         }
-        return m_bHasHiddenParaField;
+        return m_bHiddenByParaField;
     }
 
     void InsertNesting(SwTextAttrNesting & rNewHint);
@@ -144,7 +148,7 @@ private:
 #endif
 
 public:
-    SwpHints();
+    SwpHints(const SwTextNode& rParent);
 
     size_t Count() const { return m_HintsByStart.size(); }
     bool Contains( const SwTextAttr *pHt ) const;
@@ -179,8 +183,8 @@ public:
     bool HasFootnote() const          { return m_bFootnote; }
     bool IsInSplitNode() const   { return m_bInSplitNode; }
 
-    /// calc current value of m_bHasHiddenParaField, returns true iff changed
-    bool CalcHiddenParaField();
+    // calc current value of m_bHiddenByParaField, returns true iff changed
+    bool CalcHiddenParaField() const; // changes mutable state
 
     DECL_FIXEDMEMPOOL_NEWDEL(SwpHints)
 };

sw/inc/ndtxt.hxx

@@ -21,6 +21,7 @@
 
 #include <cppuhelper/weakref.hxx>
 
+#include "doc.hxx"
 #include "swdllapi.h"
 #include "node.hxx"
 #include "hintids.hxx"
@@ -701,8 +702,13 @@ public:
         { if (m_pSwpHints) m_pSwpHints->SetCalcHiddenParaField(); }
 
     /// is the paragraph visible?
-    bool HasHiddenParaField() const
-        { return m_pSwpHints && m_pSwpHints->HasHiddenParaField(); }
+    bool IsHiddenByParaField() const
+        { return m_pSwpHints && m_pSwpHints->IsHiddenByParaField(); }
+
+    bool FieldCanHidePara(SwFieldIds eFieldId) const
+        { return GetDoc()->FieldCanHidePara(eFieldId); }
+    bool FieldHidesPara(const SwField& rField) const
+        { return GetDoc()->FieldHidesPara(rField); }
 
     /// Hidden Paragraph Field:
 
@@ -811,7 +817,7 @@ inline SwpHints& SwTextNode::GetOrCreateSwpHints()
 {
     if ( !m_pSwpHints )
     {
-        m_pSwpHints.reset(new SwpHints);
+        m_pSwpHints.reset(new SwpHints(*this));
     }
     return *m_pSwpHints;
 }

sw/inc/viewsh.hxx

@@ -410,6 +410,8 @@ public:
 
     void SetSubtractFlysAnchoredAtFlys(bool bSubtractFlysAnchoredAtFlys);
 
+    void SetEmptyDbFieldHidesPara(bool bEmptyDbFieldHidesPara);
+
     // DOCUMENT COMPATIBILITY FLAGS END
 
     // Calls Idle-formatter of Layout.

sw/qa/extras/mailmerge/data/tdf35798-legacy.fodt

+<?xml version="1.0" encoding="UTF-8"?>
+
+<office:document xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" xmlns:number="urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0" xmlns:dr3d="urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0" xmlns:math="http://www.w3.org/1998/Math/MathML" xmlns:form="urn:oasis:names:tc:opendocument:xmlns:form:1.0" xmlns:script="urn:oasis:names:tc:opendocument:xmlns:script:1.0" xmlns:config="urn:oasis:names:tc:opendocument:xmlns:config:1.0" xmlns:ooo="http://openoffice.org/2004/office" xmlns:ooow="http://openoffice.org/2004/writer" xmlns:oooc="http://openoffice.org/2004/calc" xmlns:dom="http://www.w3.org/2001/xml-events" xmlns:xforms="http://www.w3.org/2002/xforms" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:rpt="http://openoffice.org/2005/report" xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:grddl="http://www.w3.org/2003/g/data-view#" xmlns:officeooo="http://openoffice.org/2009/office" xmlns:tableooo="http://openoffice.org/2009/table" xmlns:drawooo="http://openoffice.org/2010/draw" xmlns:calcext="urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0" xmlns:loext="urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0" xmlns:field="urn:openoffice:names:experimental:ooo-ms-interop:xmlns:field:1.0" xmlns:formx="urn:openoffice:names:experimental:ooxml-odf-interop:xmlns:form:1.0" xmlns:css3t="http://www.w3.org/TR/css3-text/" office:version="1.2" office:mimetype="application/vnd.oasis.opendocument.text">
+ <office:meta><meta:creation-date>2014-10-22T13:27:38.673154279</meta:creation-date><dc:date>2018-05-19T13:45:25.911000000</dc:date><meta:editing-duration>PT6M54S</meta:editing-duration><meta:editing-cycles>4</meta:editing-cycles><meta:generator>LibreOfficeDev/6.1.0.0.alpha1$Windows_X86_64 LibreOffice_project/dd1ab570d5791145c10a4e8f28b048ec8f70edb0</meta:generator><meta:document-statistic meta:table-count="0" meta:image-count="0" meta:object-count="0" meta:page-count="1" meta:paragraph-count="8" meta:word-count="37" meta:character-count="228"/></office:meta>
+ <office:settings>
+  <config:config-item-set config:name="ooo:configuration-settings">
+   <config:config-item config:name="CurrentDatabaseDataSource" config:type="string">5-with-blanks</config:config-item>
+   <config:config-item config:name="CurrentDatabaseCommand" config:type="string">names</config:config-item>
+   <config:config-item config:name="CurrentDatabaseCommandType" config:type="int">0</config:config-item>
+  </config:config-item-set>
+ </office:settings>
+ <office:scripts>
+  <office:script script:language="ooo:Basic">
+   <ooo:libraries xmlns:ooo="http://openoffice.org/2004/office" xmlns:xlink="http://www.w3.org/1999/xlink"/>
+  </office:script>
+ </office:scripts>
+ <office:automatic-styles>
+  <style:page-layout style:name="pm1">
+   <style:page-layout-properties fo:page-width="21cm" fo:page-height="29.7cm" style:print-orientation="portrait" fo:margin-top="2cm" fo:margin-bottom="2cm" fo:margin-left="2cm" fo:margin-right="2cm"/>
+  </style:page-layout>
+ </office:automatic-styles>
+ <office:master-styles>
+  <style:master-page style:name="Standard" style:page-layout-name="pm1"/>
+ </office:master-styles>
+ <office:body>
+  <office:text>
+   <text:p>Heading</text:p>
+   <text:p>Title: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Title" text:database-name="5-with-blanks">&lt;Title&gt;</text:database-display></text:p>
+   <text:p>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display></text:p>
+   <text:p>Last Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Last Name" text:database-name="5-with-blanks">&lt;Last Name&gt;</text:database-display></text:p>
+   <text:p>Title: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Title" text:database-name="5-with-blanks">&lt;Title&gt;</text:database-display><text:s/>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display></text:p>
+   <text:p>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display><text:s/>Last Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Last Name" text:database-name="5-with-blanks">&lt;Last Name&gt;</text:database-display></text:p>
+   <text:p>Title: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Title" text:database-name="5-with-blanks">&lt;Title&gt;</text:database-display><text:s/>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display><text:s/>Last Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Last Name" text:database-name="5-with-blanks">&lt;Last Name&gt;</text:database-display></text:p>
+   <text:p>Trailing text</text:p>
+  </office:text>
+ </office:body>
+</office:document>
\ No newline at end of file

sw/qa/extras/mailmerge/data/tdf35798-new.fodt

+<?xml version="1.0" encoding="UTF-8"?>
+
+<office:document xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" xmlns:number="urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0" xmlns:dr3d="urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0" xmlns:math="http://www.w3.org/1998/Math/MathML" xmlns:form="urn:oasis:names:tc:opendocument:xmlns:form:1.0" xmlns:script="urn:oasis:names:tc:opendocument:xmlns:script:1.0" xmlns:config="urn:oasis:names:tc:opendocument:xmlns:config:1.0" xmlns:ooo="http://openoffice.org/2004/office" xmlns:ooow="http://openoffice.org/2004/writer" xmlns:oooc="http://openoffice.org/2004/calc" xmlns:dom="http://www.w3.org/2001/xml-events" xmlns:xforms="http://www.w3.org/2002/xforms" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:rpt="http://openoffice.org/2005/report" xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:grddl="http://www.w3.org/2003/g/data-view#" xmlns:officeooo="http://openoffice.org/2009/office" xmlns:tableooo="http://openoffice.org/2009/table" xmlns:drawooo="http://openoffice.org/2010/draw" xmlns:calcext="urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0" xmlns:loext="urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0" xmlns:field="urn:openoffice:names:experimental:ooo-ms-interop:xmlns:field:1.0" xmlns:formx="urn:openoffice:names:experimental:ooxml-odf-interop:xmlns:form:1.0" xmlns:css3t="http://www.w3.org/TR/css3-text/" office:version="1.2" office:mimetype="application/vnd.oasis.opendocument.text">
+ <office:meta><meta:creation-date>2014-10-22T13:27:38.673154279</meta:creation-date><dc:date>2018-05-19T13:45:25.911000000</dc:date><meta:editing-duration>PT6M54S</meta:editing-duration><meta:editing-cycles>4</meta:editing-cycles><meta:generator>LibreOfficeDev/6.1.0.0.alpha1$Windows_X86_64 LibreOffice_project/dd1ab570d5791145c10a4e8f28b048ec8f70edb0</meta:generator><meta:document-statistic meta:table-count="0" meta:image-count="0" meta:object-count="0" meta:page-count="1" meta:paragraph-count="8" meta:word-count="37" meta:character-count="228"/></office:meta>
+ <office:settings>
+  <config:config-item-set config:name="ooo:configuration-settings">
+   <config:config-item config:name="CurrentDatabaseDataSource" config:type="string">5-with-blanks</config:config-item>
+   <config:config-item config:name="CurrentDatabaseCommand" config:type="string">names</config:config-item>
+   <config:config-item config:name="CurrentDatabaseCommandType" config:type="int">0</config:config-item>
+   <config:config-item config:name="EmptyDbFieldHidesPara" config:type="boolean">true</config:config-item>
+  </config:config-item-set>
+ </office:settings>
+ <office:scripts>
+  <office:script script:language="ooo:Basic">
+   <ooo:libraries xmlns:ooo="http://openoffice.org/2004/office" xmlns:xlink="http://www.w3.org/1999/xlink"/>
+  </office:script>
+ </office:scripts>
+ <office:automatic-styles>
+  <style:page-layout style:name="pm1">
+   <style:page-layout-properties fo:page-width="21cm" fo:page-height="29.7cm" style:print-orientation="portrait" fo:margin-top="2cm" fo:margin-bottom="2cm" fo:margin-left="2cm" fo:margin-right="2cm"/>
+  </style:page-layout>
+ </office:automatic-styles>
+ <office:master-styles>
+  <style:master-page style:name="Standard" style:page-layout-name="pm1"/>
+ </office:master-styles>
+ <office:body>
+  <office:text>
+   <text:p>Heading</text:p>
+   <text:p>Title: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Title" text:database-name="5-with-blanks">&lt;Title&gt;</text:database-display></text:p>
+   <text:p>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display></text:p>
+   <text:p>Last Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Last Name" text:database-name="5-with-blanks">&lt;Last Name&gt;</text:database-display></text:p>
+   <text:p>Title: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Title" text:database-name="5-with-blanks">&lt;Title&gt;</text:database-display><text:s/>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display></text:p>
+   <text:p>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display><text:s/>Last Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Last Name" text:database-name="5-with-blanks">&lt;Last Name&gt;</text:database-display></text:p>
+   <text:p>Title: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Title" text:database-name="5-with-blanks">&lt;Title&gt;</text:database-display><text:s/>First Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="First Name" text:database-name="5-with-blanks">&lt;First Name&gt;</text:database-display><text:s/>Last Name: <text:database-display text:table-name="names" text:table-type="table" text:column-name="Last Name" text:database-name="5-with-blanks">&lt;Last Name&gt;</text:database-display></text:p>
+   <text:p>Trailing text</text:p>
+  </office:text>
+ </office:body>
+</office:document>
\ No newline at end of file

sw/qa/extras/mailmerge/mailmerge.cxx

@@ -677,5 +677,145 @@ DECLARE_FILE_MAILMERGE_TEST(testTdf102010, "empty.odt", "10-testing-addresses.od
     loadMailMergeDocument( 1 );
 }
 
+namespace
+{
+constexpr char const* const EmptyValuesLegacyData[][8]
+    = { { "Heading", "Title: ", "First Name: firstname1", "Last Name: lastname1",
+          "Title:  First Name: firstname1", "First Name: firstname1 Last Name: lastname1",
+          "Title:  First Name: firstname1 Last Name: lastname1", "Trailing text" },
+        { "Heading", "Title: title2", "First Name: ", "Last Name: lastname2",
+          "Title: title2 First Name: ", "First Name:  Last Name: lastname2",
+          "Title: title2 First Name:  Last Name: lastname2", "Trailing text" },
+        { "Heading", "Title: title3", "First Name: firstname3", "Last Name: ",
+          "Title: title3 First Name: firstname3", "First Name: firstname3 Last Name: ",
+          "Title: title3 First Name: firstname3 Last Name: ", "Trailing text" },
+        { "Heading", "Title: ", "First Name: ", "Last Name: lastname4",
+          "Title:  First Name: ", "First Name:  Last Name: lastname4",
+          "Title:  First Name:  Last Name: lastname4", "Trailing text" },
+        { "Heading", "Title: title5", "First Name: ", "Last Name: ", "Title: title5 First Name: ",
+          "First Name:  Last Name: ", "Title: title5 First Name:  Last Name: ", "Trailing text" } };
+constexpr char const* const EmptyValuesNewData[][8]
+    = { { "Heading", "First Name: firstname1", "Last Name: lastname1",
+          "Title:  First Name: firstname1", "First Name: firstname1 Last Name: lastname1",
+          "Title:  First Name: firstname1 Last Name: lastname1", "Trailing text" },
+        { "Heading", "Title: title2", "Last Name: lastname2",
+          "Title: title2 First Name: ", "First Name:  Last Name: lastname2",
+          "Title: title2 First Name:  Last Name: lastname2", "Trailing text" },
+        { "Heading", "Title: title3", "First Name: firstname3",
+          "Title: title3 First Name: firstname3", "First Name: firstname3 Last Name: ",
+          "Title: title3 First Name: firstname3 Last Name: ", "Trailing text" },
+        { "Heading", "Last Name: lastname4", "First Name:  Last Name: lastname4",
+          "Title:  First Name:  Last Name: lastname4", "Trailing text" },
+        { "Heading", "Title: title5", "Title: title5 First Name: ",
+          "Title: title5 First Name:  Last Name: ", "Trailing text" } };
+}
+
+// The following four tests (testEmptyValuesLegacyODT, testEmptyValuesNewODT, (testEmptyValuesLegacyFODT, testEmptyValuesNewFODT)
+// check that for native documents without "EmptyDbFieldHidesPara" compatibility option, all paragraphs are exported visible,
+// while for documents with the option enabled, the paragraphs with all Database fields having empty values are hidden.
+
+DECLARE_FILE_MAILMERGE_TEST(testEmptyValuesLegacyODT, "tdf35798-legacy.odt", "5-with-blanks.ods",
+                            "names")
+{
+    executeMailMerge();
+    for (int doc = 0; doc < 5; ++doc)
+    {
+        loadMailMergeDocument(doc);
+        SwXTextDocument* pTextDoc = dynamic_cast<SwXTextDocument*>(mxComponent.get());
+        CPPUNIT_ASSERT(pTextDoc);
+        SwDoc* pDoc = pTextDoc->GetDocShell()->GetDoc();
+        pDoc->RemoveInvisibleContent();
+        CPPUNIT_ASSERT_EQUAL(1, getPages());
+        for (int i = 0; i < 8; ++i)
+        {
+            auto xPara = getParagraph(i+1);
+            SAL_WARN("sw.qa", "Testing doc " + OUString::number(doc) + " paragraph "
+                             + OUString::number(i + 1));
+            CPPUNIT_ASSERT_EQUAL(OUString::createFromAscii(EmptyValuesLegacyData[doc][i]),
+                                 xPara->getString());
+        }
+        CPPUNIT_ASSERT_EQUAL(8, getParagraphs());
+    }
+}
+
+DECLARE_FILE_MAILMERGE_TEST(testEmptyValuesNewODT, "tdf35798-new.odt", "5-with-blanks.ods",
+                            "names")
+{
+    executeMailMerge();
+    for (int doc = 0; doc < 5; ++doc)
+    {
+        loadMailMergeDocument(doc);
+        SwXTextDocument* pTextDoc = dynamic_cast<SwXTextDocument*>(mxComponent.get());
+        CPPUNIT_ASSERT(pTextDoc);
+        SwDoc* pDoc = pTextDoc->GetDocShell()->GetDoc();
+        pDoc->RemoveInvisibleContent();
+        CPPUNIT_ASSERT_EQUAL(1, getPages());
+        int i;
+        for (i = 0; i < 8; ++i)
+        {
+            const char* pExpected = EmptyValuesNewData[doc][i];
+            if (!pExpected)
+                break;
+            auto xPara = getParagraph(i + 1);
+            SAL_WARN("sw.qa", "Testing doc " + OUString::number(doc) + " paragraph "
+                             + OUString::number(i + 1));
+            CPPUNIT_ASSERT_EQUAL(OUString::createFromAscii(pExpected),
+                                 xPara->getString());
+        }
+        CPPUNIT_ASSERT_EQUAL(i, getParagraphs());
+    }
+}
+
+DECLARE_FILE_MAILMERGE_TEST(testEmptyValuesLegacyFODT, "tdf35798-legacy.fodt", "5-with-blanks.ods",
+                            "names")
+{
+    executeMailMerge();
+    for (int doc = 0; doc < 5; ++doc)
+    {
+        loadMailMergeDocument(doc);
+        SwXTextDocument* pTextDoc = dynamic_cast<SwXTextDocument*>(mxComponent.get());
+        CPPUNIT_ASSERT(pTextDoc);
+        SwDoc* pDoc = pTextDoc->GetDocShell()->GetDoc();
+        pDoc->RemoveInvisibleContent();
+        CPPUNIT_ASSERT_EQUAL(1, getPages());
+        for (int i = 0; i < 8; ++i)
+        {
+            auto xPara = getParagraph(i + 1);
+            SAL_WARN("sw.qa", "Testing doc " + OUString::number(doc) + " paragraph "
+                             + OUString::number(i + 1));
+            CPPUNIT_ASSERT_EQUAL(OUString::createFromAscii(EmptyValuesLegacyData[doc][i]),
+                                 xPara->getString());
+        }
+        CPPUNIT_ASSERT_EQUAL(8, getParagraphs());
+    }
+}
+
+DECLARE_FILE_MAILMERGE_TEST(testEmptyValuesNewFODT, "tdf35798-new.fodt", "5-with-blanks.ods",
+                            "names")
+{
+    executeMailMerge();
+    for (int doc = 0; doc < 5; ++doc)
+    {
+        loadMailMergeDocument(doc);
+        SwXTextDocument* pTextDoc = dynamic_cast<SwXTextDocument*>(mxComponent.get());
+        CPPUNIT_ASSERT(pTextDoc);
+        SwDoc* pDoc = pTextDoc->GetDocShell()->GetDoc();
+        pDoc->RemoveInvisibleContent();
+        CPPUNIT_ASSERT_EQUAL(1, getPages());
+        int i;
+        for (i = 0; i < 8; ++i)
+        {
+            const char* pExpected = EmptyValuesNewData[doc][i];
+            if (!pExpected)
+                break;
+            auto xPara = getParagraph(i + 1);
+            SAL_WARN("sw.qa", "Testing doc " + OUString::number(doc) + " paragraph "
+                             + OUString::number(i + 1));
+            CPPUNIT_ASSERT_EQUAL(OUString::createFromAscii(pExpected), xPara->getString());
+        }
+        CPPUNIT_ASSERT_EQUAL(i, getParagraphs());
+    }
+}
+
 CPPUNIT_PLUGIN_IMPLEMENT();
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

sw/source/core/doc/DocumentSettingManager.cxx

@@ -114,6 +114,8 @@ sw::DocumentSettingManager::DocumentSettingManager(SwDoc &rDoc)
         mbProtectForm                       = aOptions.GetDefault( SvtCompatibilityEntry::Index::ProtectForm );
         mbMsWordCompTrailingBlanks          = aOptions.GetDefault( SvtCompatibilityEntry::Index::MsWordTrailingBlanks );
         mbSubtractFlys                      = aOptions.GetDefault( SvtCompatibilityEntry::Index::SubtractFlysAnchoredAtFlys );
+        mbEmptyDbFieldHidesPara
+            = aOptions.GetDefault(SvtCompatibilityEntry::Index::EmptyDbFieldHidesPara);
     }
     else
     {
@@ -131,6 +133,7 @@ sw::DocumentSettingManager::DocumentSettingManager(SwDoc &rDoc)
         mbProtectForm                       = false;
         mbMsWordCompTrailingBlanks          = false;
         mbSubtractFlys                      = false;
+        mbEmptyDbFieldHidesPara             = true;
     }
 
     // COMPATIBILITY FLAGS END
@@ -204,6 +207,7 @@ bool sw::DocumentSettingManager::get(/*[in]*/ DocumentSettingId id) const
         case DocumentSettingId::EMBED_SYSTEM_FONTS: return mEmbedSystemFonts;
         case DocumentSettingId::APPLY_PARAGRAPH_MARK_FORMAT_TO_NUMBERING: return mApplyParagraphMarkFormatToNumbering;
         case DocumentSettingId::DISABLE_OFF_PAGE_POSITIONING: return mbDisableOffPagePositioning;
+        case DocumentSettingId::EMPTY_DB_FIELD_HIDES_PARA: return mbEmptyDbFieldHidesPara;
         default:
             OSL_FAIL("Invalid setting id");
     }
@@ -421,6 +425,9 @@ void sw::DocumentSettingManager::set(/*[in]*/ DocumentSettingId id, /*[in]*/ boo
         case DocumentSettingId::DISABLE_OFF_PAGE_POSITIONING:
             mbDisableOffPagePositioning = value;
             break;
+        case DocumentSettingId::EMPTY_DB_FIELD_HIDES_PARA:
+            mbEmptyDbFieldHidesPara = value;
+            break;
         default:
             OSL_FAIL("Invalid setting id");
     }
@@ -562,6 +569,7 @@ void sw::DocumentSettingManager::ReplaceCompatibilityOptions(const DocumentSetti
     mbTabRelativeToIndent = rSource.mbTabRelativeToIndent;
     mbTabAtLeftIndentForParagraphsInList = rSource.mbTabAtLeftIndentForParagraphsInList;
     mbMsWordCompTrailingBlanks = rSource.mbMsWordCompTrailingBlanks;
+    mbEmptyDbFieldHidesPara = rSource.mbEmptyDbFieldHidesPara;
 }
 
 sal_uInt32 sw::DocumentSettingManager::Getn32DummyCompatibilityOptions1() const

sw/source/core/doc/doc.cxx

@@ -1318,6 +1318,73 @@ void SwDoc::Summary( SwDoc* pExtDoc, sal_uInt8 nLevel, sal_uInt8 nPara, bool bIm
     }
 }
 
+namespace
+{
+void RemoveOrDeleteContents(SwTextNode* pTextNd, IDocumentContentOperations& xOperations)
+{
+    SwPaM aPam(*pTextNd, 0, *pTextNd, pTextNd->GetText().getLength());
+
+    // Remove hidden paragraph or delete contents:
+    // Delete contents if
+    // 1. removing the paragraph would result in an empty section or
+    // 2. if the paragraph is the last paragraph in the section and
+    //    there is no paragraph in front of the paragraph:
+    if ((2 == pTextNd->EndOfSectionIndex() - pTextNd->StartOfSectionIndex())
+        || (1 == pTextNd->EndOfSectionIndex() - pTextNd->GetIndex()
+            && !pTextNd->GetNodes()[pTextNd->GetIndex() - 1]->GetTextNode()))
+    {
+        xOperations.DeleteRange(aPam);
+    }
+    else
+    {
+        aPam.DeleteMark();
+        xOperations.DelFullPara(aPam);
+    }
+}
+// Returns if the data was actually modified
+bool HandleHidingField(SwFormatField& rFormatField, const SwNodes& rNodes,
+                       IDocumentContentOperations& xOperations)
+{
+    SwTextNode* pTextNd;
+    if (rFormatField.GetTextField()
+        && nullptr != (pTextNd = rFormatField.GetTextField()->GetpTextNode())
+        && pTextNd->GetpSwpHints() && pTextNd->IsHiddenByParaField()
+        && &pTextNd->GetNodes() == &rNodes)
+    {
+        RemoveOrDeleteContents(pTextNd, xOperations);
+        return true;
+    }
+    return false;
+}
+}
+
+bool SwDoc::FieldCanHidePara(SwFieldIds eFieldId) const
+{
+    switch (eFieldId)
+    {
+        case SwFieldIds::HiddenPara:
+            return true;
+        case SwFieldIds::Database:
+            return GetDocumentSettingManager().get(
+                DocumentSettingId::EMPTY_DB_FIELD_HIDES_PARA);
+        default:
+            return false;
+    }
+}
+
+bool SwDoc::FieldHidesPara(const SwField& rField) const
+{
+    switch (rField.GetTyp()->Which())
+    {
+        case SwFieldIds::HiddenPara:
+            return static_cast<const SwHiddenParaField&>(rField).IsHidden();
+        case SwFieldIds::Database:
+            return FieldCanHidePara(SwFieldIds::Database) && rField.ExpandField(true).isEmpty();
+        default:
+            return false;
+    }
+}
+
 /// Remove the invisible content from the document e.g. hidden areas, hidden paragraphs
 bool SwDoc::RemoveInvisibleContent()
 {
@@ -1325,35 +1392,22 @@ bool SwDoc::RemoveInvisibleContent()
     GetIDocumentUndoRedo().StartUndo( SwUndoId::UI_DELETE_INVISIBLECNTNT, nullptr );
 
     {
-        SwTextNode* pTextNd;
-        SwIterator<SwFormatField,SwFieldType> aIter( *getIDocumentFieldsAccess().GetSysFieldType( SwFieldIds::HiddenPara )  );
-        for( SwFormatField* pFormatField = aIter.First(); pFormatField;  pFormatField = aIter.Next() )
+        // Removing some nodes for one SwFieldIds::Database type might remove the type from
+        // document's field types, invalidating iterators. So, we need to create own list of
+        // matching types prior to processing them.
+        std::vector<const SwFieldType*> aHidingFieldTypes;
+        for (const auto* pType : *getIDocumentFieldsAccess().GetFieldTypes())
         {
-            if( pFormatField->GetTextField() &&
-                nullptr != ( pTextNd = pFormatField->GetTextField()->GetpTextNode() ) &&
-                pTextNd->GetpSwpHints() && pTextNd->HasHiddenParaField() &&
-                &pTextNd->GetNodes() == &GetNodes() )
-            {
-                bRet = true;
-                SwPaM aPam(*pTextNd, 0, *pTextNd, pTextNd->GetText().getLength());
-
-                // Remove hidden paragraph or delete contents:
-                // Delete contents if
-                // 1. removing the paragraph would result in an empty section or
-                // 2. if the paragraph is the last paragraph in the section and
-                //    there is no paragraph in front of the paragraph:
-                if ( ( 2 == pTextNd->EndOfSectionIndex() - pTextNd->StartOfSectionIndex() ) ||
-                     ( 1 == pTextNd->EndOfSectionIndex() - pTextNd->GetIndex() &&
-                       !GetNodes()[ pTextNd->GetIndex() - 1 ]->GetTextNode() ) )
-                {
-                    getIDocumentContentOperations().DeleteRange( aPam );
-                }
-                else
-                {
-                    aPam.DeleteMark();
-                    getIDocumentContentOperations().DelFullPara( aPam );
-                }
-            }
+            if (FieldCanHidePara(pType->Which()))
+                aHidingFieldTypes.push_back(pType);
+        }
+        for (const auto* pType : aHidingFieldTypes)
+        {
+            SwIterator<SwFormatField, SwFieldType> aIter(*pType);
+            for (SwFormatField* pFormatField = aIter.First(); pFormatField;
+                 pFormatField = aIter.Next())
+                bRet |= HandleHidingField(*pFormatField, GetNodes(),
+                                          getIDocumentContentOperations());
         }
     }
 
@@ -1369,23 +1423,7 @@ bool SwDoc::RemoveInvisibleContent()
             {
                 bRemoved = true;
                 bRet = true;
-
-                // Remove hidden paragraph or delete contents:
-                // Delete contents if
-                // 1. removing the paragraph would result in an empty section or
-                // 2. if the paragraph is the last paragraph in the section and
-                //    there is no paragraph in front of the paragraph:
-                if ( ( 2 == pTextNd->EndOfSectionIndex() - pTextNd->StartOfSectionIndex() ) ||
-                     ( 1 == pTextNd->EndOfSectionIndex() - pTextNd->GetIndex() &&
-                       !GetNodes()[ pTextNd->GetIndex() - 1 ]->GetTextNode() ) )
-                {
-                    getIDocumentContentOperations().DeleteRange( aPam );
-                }
-                else
-                {
-                    aPam.DeleteMark();
-                    getIDocumentContentOperations().DelFullPara( aPam );
-                }
+                RemoveOrDeleteContents(pTextNd, getIDocumentContentOperations());
             }
             else if ( pTextNd->HasHiddenCharAttribute( false ) )
             {

sw/source/core/doc/doctxm.cxx

@@ -1152,7 +1152,7 @@ void SwTOXBaseSection::UpdateMarks( const SwTOXInternational& rIntl,
                 pTOXSrc->GetText().getLength() && pTOXSrc->HasWriterListeners() &&
                 pTOXSrc->getLayoutFrame( pDoc->getIDocumentLayoutAccess().GetCurrentLayout() ) &&
                (!IsFromChapter() || ::lcl_FindChapterNode( *pTOXSrc ) == pOwnChapterNode ) &&
-               !pTOXSrc->HasHiddenParaField() &&
+               !pTOXSrc->IsHiddenByParaField() &&
                !SwScriptInfo::IsInHiddenRange( *pTOXSrc, pTextMark->GetStart() ) )
             {
                 SwTOXSortTabBase* pBase = nullptr;
@@ -1205,7 +1205,7 @@ void SwTOXBaseSection::UpdateOutline( const SwTextNode* pOwnChapterNode )
         if( pTextNd && pTextNd->Len() && pTextNd->HasWriterListeners() &&
             sal_uInt16( pTextNd->GetAttrOutlineLevel()) <= GetLevel() &&
             pTextNd->getLayoutFrame( pDoc->getIDocumentLayoutAccess().GetCurrentLayout() ) &&
-           !pTextNd->HasHiddenParaField() &&
+           !pTextNd->IsHiddenByParaField() &&
            !pTextNd->HasHiddenCharAttribute( true ) &&
             ( !IsFromChapter() ||
                ::lcl_FindChapterNode( *pTextNd ) == pOwnChapterNode ))

sw/source/core/inc/DocumentSettingManager.hxx

@@ -155,6 +155,7 @@ class DocumentSettingManager :
 
     bool mbLastBrowseMode                           : 1;
     bool mbDisableOffPagePositioning; // tdf#112443
+    bool mbEmptyDbFieldHidesPara;
 
 public:
 

sw/source/core/text/txtfrm.cxx

@@ -459,12 +459,11 @@ bool SwTextFrame::IsHiddenNow() const
         return true;
     }
 
-    const bool bHiddenCharsHidePara = GetTextNode()->HasHiddenCharAttribute( true );
-    const bool bHiddenParaField = GetTextNode()->HasHiddenParaField();
-    const SwViewShell* pVsh = getRootFrame()->GetCurrShell();
-
-    if ( pVsh && ( bHiddenCharsHidePara || bHiddenParaField ) )
+    if ( const SwViewShell* pVsh = getRootFrame()->GetCurrShell() )
     {
+        const bool bHiddenCharsHidePara = GetTextNode()->HasHiddenCharAttribute(true);
+        const bool bHiddenParaField = GetTextNode()->IsHiddenByParaField();
+
         if (
              ( bHiddenParaField &&
                ( !pVsh->GetViewOptions()->IsShowHiddenPara() &&

sw/source/core/txtnode/atrfld.cxx

@@ -380,18 +380,26 @@ void SwTextField::ExpandTextField(const bool bForceNotify) const
     const SwField* pField = GetFormatField().GetField();
     const OUString aNewExpand( pField->ExpandField(m_pTextNode->GetDoc()->IsClipBoard()) );
 
+    const SwFieldIds nWhich = pField->GetTyp()->Which();
+    const bool bSameExpandSimpleNotification
+        = SwFieldIds::Chapter != nWhich && SwFieldIds::PageNumber != nWhich
+          && SwFieldIds::RefPageGet != nWhich
+          // Page count fields to not use aExpand during formatting,
+          // therefore an invalidation of the text frame has to be triggered even if aNewExpand == aExpand:
+          && (SwFieldIds::DocStat != nWhich
+              || DS_PAGE != static_cast<const SwDocStatField*>(pField)->GetSubType())
+          && (SwFieldIds::GetExp != nWhich
+              || static_cast<const SwGetExpField*>(pField)->IsInBodyText());
+
+    bool bHiddenParaChanged = false;
+    if (aNewExpand != m_aExpand || bSameExpandSimpleNotification