commit 4dc7c00d192dcf8f89e1f9f9e3bc5f9a2db52535
author Scott Cantor <cantor.2@osu.edu> Mon Oct 10 11:48:07 2022 -0400
committer Scott Cantor <cantor.2@osu.edu> Mon Oct 10 11:49:22 2022 -0400
tree fee56e57c3f99d55c3bf910e18471b2d9092d1b6
parent 472f612906ef860fab667af39fba51f01f2574db

XERCESC-2241 - Integer overflows in DFAContentModel class

https://issues.apache.org/jira/browse/XERCESC-2241

src/xercesc/validators/common/DFAContentModel.cpp

diff --git a/src/xercesc/validators/common/DFAContentModel.cpp b/src/xercesc/validators/common/DFAContentModel.cpp
index 856f88f..1b5dc59 100644
--- a/src/xercesc/validators/common/DFAContentModel.cpp
+++ b/src/xercesc/validators/common/DFAContentModel.cpp
@@ -42,6 +42,7 @@
 #include <xercesc/util/RefHashTableOf.hpp>
 #include <xercesc/util/XMLInteger.hpp>
 #include <math.h>
+#include <limits>
 
 namespace XERCES_CPP_NAMESPACE {
 
@@ -661,8 +662,15 @@
     //  in the fLeafCount member.
     //
     fLeafCount=countLeafNodes(curNode);
+    // Avoid integer overflow in below fLeafCount++ increment
+    if (fLeafCount > (std::numeric_limits<unsigned int>::max() - 1))
+        throw OutOfMemoryException();
     fEOCPos = fLeafCount++;
 
+    // Avoid integer overflow in below memory allocation
+    if (fLeafCount > (std::numeric_limits<size_t>::max() / sizeof(CMLeaf*)))
+        throw OutOfMemoryException();
+
     //  We need to build an array of references to the non-epsilon
     //  leaf nodes. We will put them in the array according to their position values
     //
@@ -1364,14 +1372,27 @@
         if(nLoopCount!=0)
         {
             count += countLeafNodes(cursor);
-            for(unsigned int i=0;i<nLoopCount;i++)
-                count += countLeafNodes(rightNode);
+            const unsigned int countRight = countLeafNodes(rightNode);
+            // Avoid integer overflow in below multiplication
+            if (countRight > (std::numeric_limits<unsigned int>::max() / nLoopCount))
+                throw OutOfMemoryException();
+            const unsigned int countRightMulLoopCount = nLoopCount * countRight;
+            // Avoid integer overflow in below addition
+            if (count > (std::numeric_limits<unsigned int>::max() - countRightMulLoopCount))
+                throw OutOfMemoryException();
+            count += countRightMulLoopCount;
             return count;
         }
         if(leftNode)
             count+=countLeafNodes(leftNode);
         if(rightNode)
-            count+=countLeafNodes(rightNode);
+        {
+            const unsigned int countRight = countLeafNodes(rightNode);
+            // Avoid integer overflow in below addition
+            if (count > (std::numeric_limits<unsigned int>::max() - countRight))
+                throw OutOfMemoryException();
+            count+=countRight;
+        }
     }
     return count;
 }