Prepare to handle OOXML Agile Encryption password hash as well

... that prepends the iteration count to the hash instead of
appending it

Change-Id: I090393e6337c110029e35baaa259b40ef4e5d416

comphelper/qa/unit/test_hash.cxx

@@ -99,7 +99,7 @@ void TestHash::testSHA512_NoSaltNoSpin()
     const char* const pInput = "";
     std::vector<unsigned char> calculate_hash =
         comphelper::Hash::calculateHash( reinterpret_cast<const unsigned char*>(pInput), 0,
-                nullptr, 0, 0, comphelper::HashType::SHA512);
+                nullptr, 0, 0, false, comphelper::HashType::SHA512);
     CPPUNIT_ASSERT_EQUAL(size_t(64), calculate_hash.size());
     std::string aStr("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e");
     CPPUNIT_ASSERT_EQUAL(aStr, tostring(calculate_hash));
@@ -112,7 +112,7 @@ void TestHash::testSHA512_saltspin()
     const OUString aPass("pwd");
     const OUString aAlgo("SHA-512");
     const OUString aSalt("876MLoKTq42+/DLp415iZQ==");
-    const OUString aHash = comphelper::DocPasswordHelper::GetOoxHashAsBase64( aPass, aSalt, 100000, aAlgo);
+    const OUString aHash = comphelper::DocPasswordHelper::GetOoxHashAsBase64( aPass, aSalt, 100000, false, aAlgo);
     const OUString aStr("5l3mgNHXpWiFaBPv5Yso1Xd/UifWvQWmlDnl/hsCYbFT2sJCzorjRmBCQ/3qeDu6Q/4+GIE8a1DsdaTwYh1q2g==");
     CPPUNIT_ASSERT_EQUAL(aStr, aHash);
 }

comphelper/source/misc/docpasswordhelper.cxx

@@ -263,12 +263,13 @@ css::uno::Sequence<sal_Int8> DocPasswordHelper::GetOoxHashAsSequence(
         const rtl::OUString& rPassword,
         const rtl::OUString& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         const rtl::OUString& rAlgorithmName)
 {
     comphelper::HashType eType;
-    if (rAlgorithmName == "SHA-512")
+    if (rAlgorithmName == "SHA-512" || rAlgorithmName == "SHA512")
         eType = comphelper::HashType::SHA512;
-    else if (rAlgorithmName == "SHA-256")
+    else if (rAlgorithmName == "SHA-256" || rAlgorithmName == "SHA256")
         eType = comphelper::HashType::SHA256;
     else if (rAlgorithmName == "SHA-1")
         eType = comphelper::HashType::SHA1;
@@ -285,7 +286,8 @@ css::uno::Sequence<sal_Int8> DocPasswordHelper::GetOoxHashAsSequence(
         aSaltVec = comphelper::sequenceToContainer<std::vector<unsigned char>>( aSaltSeq);
     }
 
-    std::vector<unsigned char> hash( comphelper::Hash::calculateHash( rPassword, aSaltVec, nSpinCount, eType));
+    std::vector<unsigned char> hash( comphelper::Hash::calculateHash( rPassword, aSaltVec, nSpinCount,
+                bPrependNotAppend, eType));
 
     return comphelper::containerToSequence<sal_Int8>( hash);
 }
@@ -294,9 +296,11 @@ OUString DocPasswordHelper::GetOoxHashAsBase64(
         const rtl::OUString& rPassword,
         const rtl::OUString& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         const rtl::OUString& rAlgorithmName)
 {
-    css::uno::Sequence<sal_Int8> aSeq( GetOoxHashAsSequence( rPassword, rSaltValue, nSpinCount, rAlgorithmName));
+    css::uno::Sequence<sal_Int8> aSeq( GetOoxHashAsSequence( rPassword, rSaltValue, nSpinCount,
+                bPrependNotAppend, rAlgorithmName));
 
     OUStringBuffer aBuf;
     comphelper::Base64::encode( aBuf, aSeq);

comphelper/source/misc/hash.cxx

@@ -157,6 +157,7 @@ std::vector<unsigned char> Hash::calculateHash(
         const unsigned char* pInput, size_t nLength,
         const unsigned char* pSalt, size_t nSaltLen,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         HashType eType)
 {
     if (!pSalt)
@@ -184,16 +185,11 @@ std::vector<unsigned char> Hash::calculateHash(
     {
         // https://msdn.microsoft.com/en-us/library/dd920692
         // says the iteration is concatenated after the hash.
-        // XXX NOTE: oox/source/crypto/AgileEngine.cxx
-        // AgileEngine::calculateHashFinal() prepends the iteration value, they
-        // do things differently for write protection and encryption passwords.
-        // https://msdn.microsoft.com/en-us/library/dd924776
-        /* TODO: maybe pass a flag whether to prepend or append, and then let
-         * AgileEngine::calculateHashFinal() call this function. */
-        const size_t nIterPos = hash.size();
-        const size_t nHashPos = 0;
-        //const size_t nIterPos = 0;
-        //const size_t nHashPos = 4;
+        // https://msdn.microsoft.com/en-us/library/dd924776 and
+        // https://msdn.microsoft.com/en-us/library/dd925430
+        // say the iteration is prepended to the hash.
+        const size_t nIterPos = (bPrependNotAppend ? 0 : hash.size());
+        const size_t nHashPos = (bPrependNotAppend ? 4 : 0);
         std::vector<unsigned char> data( hash.size() + 4, 0);
         for (sal_uInt32 i = 0; i < nSpinCount; ++i)
         {
@@ -222,11 +218,13 @@ std::vector<unsigned char> Hash::calculateHash(
         const OUString& rPassword,
         const std::vector<unsigned char>& rSaltValue,
         sal_uInt32 nSpinCount,
+        bool bPrependNotAppend,
         HashType eType)
 {
     const unsigned char* pPassBytes = reinterpret_cast<const unsigned char*>(rPassword.getStr());
     const size_t nPassBytesLen = rPassword.getLength() * 2;
-    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount, eType);
+    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount,
+            bPrependNotAppend, eType);
 }
 
 }

include/comphelper/docpasswordhelper.hxx

@@ -194,11 +194,21 @@ public:
         @param  nSpinCount
                 If >0 the number of repeated iterations.
 
+        @param  bPrependNotAppend
+                If <FALSE/>, append spin count in iterations as per
+                https://msdn.microsoft.com/en-us/library/dd920692
+                If <TRUE/>, prepend spin count in iterations as per
+                https://msdn.microsoft.com/en-us/library/dd924776 and
+                https://msdn.microsoft.com/en-us/library/dd925430
+
         @param  rAlgorithmName
-                One of "SHA-512", "SHA-256", ... as listed in
+                One of "SHA-512", "SHA-256", ... as listed for AlgorithmName in
                 https://msdn.microsoft.com/en-us/library/dd920692
-                that have a valid match in HashType. If not, an empty string is
-                returned. Not all algorithm names are supported.
+                or "SHA512", "SHA256", ... as listed for HashAlgorithm in
+                https://msdn.microsoft.com/en-us/library/dd925810
+                that have a valid match in comphelper::HashType. If not, an
+                empty sequence is returned. Not all algorithm names are
+                supported.
 
         @return the raw hash value as sal_Int8 sequence.
      */
@@ -206,6 +216,7 @@ public:
             const rtl::OUString& rPassword,
             const rtl::OUString& rSaltValue,
             sal_uInt32 nSpinCount,
+            bool bPrependNotAppend,
             const rtl::OUString& rAlgorithmName);
 
 
@@ -223,11 +234,21 @@ public:
         @param  nSpinCount
                 If >0 the number of repeated iterations.
 
+        @param  bPrependNotAppend
+                If <FALSE/>, append spin count in iterations as per
+                https://msdn.microsoft.com/en-us/library/dd920692
+                If <TRUE/>, prepend spin count in iterations as per
+                https://msdn.microsoft.com/en-us/library/dd924776 and
+                https://msdn.microsoft.com/en-us/library/dd925430
+
         @param  rAlgorithmName
-                One of "SHA-512", "SHA-256", ... as listed in
+                One of "SHA-512", "SHA-256", ... as listed for AlgorithmName in
                 https://msdn.microsoft.com/en-us/library/dd920692
-                that have a valid match in HashType. If not, an empty string is
-                returned. Not all algorithm names are supported.
+                or "SHA512", "SHA256", ... as listed for HashAlgorithm in
+                https://msdn.microsoft.com/en-us/library/dd925810
+                that have a valid match in comphelper::HashType. If not, an
+                empty sequence is returned. Not all algorithm names are
+                supported.
 
         @return the base64 encoded string of the hash value, that can be
                 compared against a stored base64 encoded hash value.
@@ -236,6 +257,7 @@ public:
             const rtl::OUString& rPassword,
             const rtl::OUString& rSaltValue,
             sal_uInt32 nSpinCount,
+            bool bPrependNotAppend,
             const rtl::OUString& rAlgorithmName);
 
 

include/comphelper/hash.hxx

@@ -54,18 +54,28 @@ public:
         (0-based, little endian) containing the number of the iteration
         appended to the hash value is the input for the next iteration.
 
-        This implements the algorithm as specified in
-        https://msdn.microsoft.com/en-us/library/dd920692
+        This implements the algorithms as specified in
+        https://msdn.microsoft.com/en-us/library/dd920692 or
+        https://msdn.microsoft.com/en-us/library/dd924776 and
+        https://msdn.microsoft.com/en-us/library/dd925430
 
         @param  pSalt
                 may be nullptr thus no salt prepended
 
+        @param  bPrependNotAppend
+                If <FALSE/>, append spin count in iterations as per
+                https://msdn.microsoft.com/en-us/library/dd920692
+                If <TRUE/>, prepend spin count in iterations as per
+                https://msdn.microsoft.com/en-us/library/dd924776 and
+                https://msdn.microsoft.com/en-us/library/dd925430
+
         @return the raw hash value
      */
     static std::vector<unsigned char> calculateHash(
             const unsigned char* pInput, size_t nLength,
             const unsigned char* pSalt, size_t nSaltLen,
             sal_uInt32 nSpinCount,
+            bool bPrependNotAppend,
             HashType eType);
 
     /** Convenience function to calculate a salted hash with iterations.
@@ -80,6 +90,7 @@ public:
             const rtl::OUString& rPassword,
             const std::vector<unsigned char>& rSaltValue,
             sal_uInt32 nSpinCount,
+            bool bPrependNotAppend,
             HashType eType);
 
     size_t getLength() const;

sc/source/core/data/tabprotection.cxx

@@ -95,7 +95,7 @@ bool ScOoxPasswordHash::verifyPassword( const OUString& aPassText ) const
         return false;
 
     const OUString aHash( comphelper::DocPasswordHelper::GetOoxHashAsBase64(
-                aPassText, maSaltValue, mnSpinCount, maAlgorithmName));
+                aPassText, maSaltValue, mnSpinCount, false, maAlgorithmName));
     if (aHash.isEmpty())
         // unsupported algorithm
         return false;