ofz#8490 stack exhaustion

a linear loop builds a recursive structure, if it gets too deep then later
processing, e.g. releasing the tree, can exhaust stack

Change-Id: I4421b9bae62ac2b6ffe32531d1167a482103bfde
Reviewed-on: https://gerrit.libreoffice.org/54762Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

starmath/inc/parse.hxx

@@ -29,6 +29,8 @@
 #include "error.hxx"
 #include "node.hxx"
 
+#define DEPTH_LIMIT 1024
+
 class SmParser
 {
     OUString        m_aBufferString;
@@ -53,7 +55,7 @@ class SmParser
         {
             ++m_rParseDepth;
         }
-        bool TooDeep() const { return m_rParseDepth > 1024; }
+        bool TooDeep() const { return m_rParseDepth > DEPTH_LIMIT; }
         ~DepthProtect()
         {
             --m_rParseDepth;

starmath/source/parse.cxx

@@ -1103,8 +1103,16 @@ std::unique_ptr<SmNode> SmParser::DoProduct()
 
     auto xFirst = DoPower();
 
+    int nDepthLimit = 0;
+
     while (TokenInGroup(TG::Product))
     {
+        //this linear loop builds a recursive structure, if it gets
+        //too deep then later processing, e.g. releasing the tree,
+        //can exhaust stack
+        if (nDepthLimit > DEPTH_LIMIT)
+            throw std::range_error("parser depth limit");
+
         std::unique_ptr<SmStructureNode> xSNode;
         std::unique_ptr<SmNode> xOper;
         bool bSwitchArgs = false;
@@ -1169,6 +1177,7 @@ std::unique_ptr<SmNode> SmParser::DoProduct()
             xSNode->SetSubNodes(xFirst.release(), xOper.release(), xArg.release());
         }
         xFirst = std::move(xSNode);
+        ++nDepthLimit;
     }
     return xFirst;
 }