Implement OOXML password hashing algorithm, tdf#104250 prep

As per https://msdn.microsoft.com/en-us/library/dd920692

Change-Id: Iebacaf3549dab28fd3033f9c241130fd66782b25
Reviewed-on: https://gerrit.libreoffice.org/50259Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Eike Rathke <erack@redhat.com>

comphelper/qa/unit/test_hash.cxx

@@ -9,6 +9,7 @@
 
 #include <comphelper/hash.hxx>
 
+#include <rtl/ustring.hxx>
 #include <sal/log.hxx>
 #include <iomanip>
 
@@ -22,12 +23,16 @@ public:
     void testSHA1();
     void testSHA256();
     void testSHA512();
+    void testSHA512_NoSaltNoSpin();
+    void testSHA512_saltspin();
 
     CPPUNIT_TEST_SUITE(TestHash);
     CPPUNIT_TEST(testMD5);
     CPPUNIT_TEST(testSHA1);
     CPPUNIT_TEST(testSHA256);
     CPPUNIT_TEST(testSHA512);
+    CPPUNIT_TEST(testSHA512_NoSaltNoSpin);
+    CPPUNIT_TEST(testSHA512_saltspin);
     CPPUNIT_TEST_SUITE_END();
 };
 
@@ -87,6 +92,30 @@ void TestHash::testSHA512()
     CPPUNIT_ASSERT_EQUAL(aStr, tostring(calculate_hash));
 }
 
+// Must be identical to testSHA512()
+void TestHash::testSHA512_NoSaltNoSpin()
+{
+    const char* const pInput = "";
+    std::vector<unsigned char> calculate_hash =
+        comphelper::Hash::calculateHash( reinterpret_cast<const unsigned char*>(pInput), 0,
+                nullptr, 0, 0, comphelper::HashType::SHA512);
+    CPPUNIT_ASSERT_EQUAL(size_t(64), calculate_hash.size());
+    std::string aStr("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e");
+    CPPUNIT_ASSERT_EQUAL(aStr, tostring(calculate_hash));
+}
+
+// Password, salt, hash and spin count taken from OOXML sheetProtection of
+// tdf#104250 https://bugs.documentfoundation.org/attachment.cgi?id=129104
+void TestHash::testSHA512_saltspin()
+{
+    const OUString aPass("pwd");
+    const OUString aAlgo("SHA-512");
+    const OUString aSalt("876MLoKTq42+/DLp415iZQ==");
+    const OUString aHash = comphelper::Hash::calculateHash( aPass, aSalt, 100000, aAlgo);
+    OUString aStr("5l3mgNHXpWiFaBPv5Yso1Xd/UifWvQWmlDnl/hsCYbFT2sJCzorjRmBCQ/3qeDu6Q/4+GIE8a1DsdaTwYh1q2g==");
+    CPPUNIT_ASSERT_EQUAL(aStr, aHash);
+}
+
 CPPUNIT_TEST_SUITE_REGISTRATION(TestHash);
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

comphelper/source/misc/hash.cxx

@@ -8,6 +8,11 @@
  */
 
 #include <comphelper/hash.hxx>
+#include <comphelper/base64.hxx>
+#include <comphelper/sequence.hxx>
+#include <rtl/ustring.hxx>
+#include <rtl/alloc.h>
+#include <osl/endian.h>
 #include <config_oox.h>
 
 #if USE_TLS_NSS
@@ -150,6 +155,113 @@ std::vector<unsigned char> Hash::calculateHash(const unsigned char* pInput, size
     return aHash.finalize();
 }
 
+std::vector<unsigned char> Hash::calculateHash(
+        const unsigned char* pInput, size_t nLength,
+        const unsigned char* pSalt, size_t nSaltLen,
+        sal_uInt32 nSpinCount,
+        HashType eType)
+{
+    if (!pSalt)
+        nSaltLen = 0;
+
+    if (!nSaltLen && !nSpinCount)
+        return calculateHash( pInput, nLength, eType);
+
+    Hash aHash(eType);
+    std::vector<unsigned char> hash;
+    if (nSaltLen)
+    {
+        std::vector<unsigned char> initialData( nSaltLen + nLength);
+        std::copy( pSalt, pSalt + nSaltLen, initialData.begin());
+        std::copy( pInput, pInput + nLength, initialData.begin() + nSaltLen);
+        aHash.update( initialData.data(), initialData.size());
+        rtl_secureZeroMemory( initialData.data(), initialData.size());
+    }
+    else
+    {
+        aHash.update( pInput, nLength);
+    }
+    hash = aHash.finalize();
+
+    if (nSpinCount)
+    {
+        // https://msdn.microsoft.com/en-us/library/dd920692
+        // says the iteration is concatenated after the hash.
+        // XXX NOTE: oox/source/crypto/AgileEngine.cxx
+        // AgileEngine::calculateHashFinal() prepends the iteration value, they
+        // do things differently for write protection and encryption passwords.
+        // https://msdn.microsoft.com/en-us/library/dd924776
+        /* TODO: maybe pass a flag whether to prepend or append, and then let
+         * AgileEngine::calculateHashFinal() call this function. */
+        const size_t nIterPos = hash.size();
+        const size_t nHashPos = 0;
+        //const size_t nIterPos = 0;
+        //const size_t nHashPos = 4;
+        std::vector<unsigned char> data( hash.size() + 4, 0);
+        for (sal_uInt32 i = 0; i < nSpinCount; ++i)
+        {
+            std::copy( hash.begin(), hash.end(), data.begin() + nHashPos);
+#ifdef OSL_BIGENDIAN
+            sal_uInt32 be = i;
+            sal_uInt8* p = reinterpret_cast<sal_uInt8*>(&be);
+            std::swap( p[0], p[3] );
+            std::swap( p[1], p[2] );
+            memcpy( data.data() + nIterPos, &be, 4);
+#else
+            memcpy( data.data() + nIterPos, &i, 4);
+#endif
+            /* TODO: isn't there something better than
+             * creating/finalizing/destroying on each iteration? */
+            Hash aReHash(eType);
+            aReHash.update( data.data(), data.size());
+            hash = aReHash.finalize();
+        }
+    }
+
+    return hash;
+}
+
+std::vector<unsigned char> Hash::calculateHash(
+        const OUString& rPassword,
+        const std::vector<unsigned char>& rSaltValue,
+        sal_uInt32 nSpinCount,
+        HashType eType)
+{
+    const unsigned char* pPassBytes = reinterpret_cast<const unsigned char*>(rPassword.getStr());
+    const size_t nPassBytesLen = rPassword.getLength() * 2;
+    return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount, eType);
+}
+
+OUString Hash::calculateHash(
+        const rtl::OUString& rPassword,
+        const rtl::OUString& rSaltValue,
+        sal_uInt32 nSpinCount,
+        const rtl::OUString& rAlgorithmName)
+{
+    HashType eType;
+    if (rAlgorithmName == "SHA-512")
+        eType = HashType::SHA512;
+    else if (rAlgorithmName == "SHA-256")
+        eType = HashType::SHA256;
+    else if (rAlgorithmName == "SHA-1")
+        eType = HashType::SHA1;
+    else if (rAlgorithmName == "MD5")
+        eType = HashType::MD5;
+    else
+        return OUString();
+
+    css::uno::Sequence<sal_Int8> aSaltSeq;
+    comphelper::Base64::decode( aSaltSeq, rSaltValue);
+
+    std::vector<unsigned char> hash = calculateHash( rPassword,
+            comphelper::sequenceToContainer<std::vector<unsigned char>>( aSaltSeq),
+            nSpinCount, eType);
+
+    OUStringBuffer aBuf;
+    comphelper::Base64::encode( aBuf, comphelper::containerToSequence<sal_Int8>( hash));
+    return aBuf.makeStringAndClear();
+}
+
 }
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

include/comphelper/hash.hxx

@@ -15,6 +15,10 @@
 #include <memory>
 #include <vector>
 
+namespace rtl {
+    class OUString;
+}
+
 namespace comphelper {
 
 enum class HashType
@@ -43,6 +47,65 @@ public:
 
     static std::vector<unsigned char> calculateHash(const unsigned char* pInput, size_t length, HashType eType);
 
+    /** Calculate hash value with salt (pSalt,nSaltLen) prepended to password
+        (pInput,nLength) and repeated iterations run if nSpinCount>0.
+
+        For repeated iterations, each iteration's result plus a 4 byte value
+        (0-based, little endian) containing the number of the iteration
+        appended to the hash value is the input for the next iteration.
+
+        This implements the algorithm as specified in
+        https://msdn.microsoft.com/en-us/library/dd920692
+
+        @param  pSalt
+                may be nullptr thus no salt prepended
+
+        @return the raw hash value
+     */
+    static std::vector<unsigned char> calculateHash(
+            const unsigned char* pInput, size_t nLength,
+            const unsigned char* pSalt, size_t nSaltLen,
+            sal_uInt32 nSpinCount,
+            HashType eType);
+
+    /** Convenience function to calculate a salted hash with iterations.
+
+        @param  rPassword
+                UTF-16LE encoded string without leading BOM character
+
+        @param  rSaltValue
+                Salt that will be prepended to password data.
+     */
+    static std::vector<unsigned char> calculateHash(
+            const rtl::OUString& rPassword,
+            const std::vector<unsigned char>& rSaltValue,
+            sal_uInt32 nSpinCount,
+            HashType eType);
+
+    /** Convenience function to calculate a salted hash with iterations.
+
+        @param  rPassword
+                UTF-16LE encoded string without leading BOM character
+
+        @param  rSaltValue
+                Base64 encoded salt that will be decoded and prepended to password
+                data.
+
+        @param  rAlgorithmName
+                One of "SHA-512", "SHA-256", ... as listed in
+                https://msdn.microsoft.com/en-us/library/dd920692
+                that have a valid match in HashType. If not, an empty string is
+                returned. Not all algorithm names are supported.
+
+        @return the base64 encoded string of the hash value, that can be
+                compared against a stored base64 encoded hash value.
+     */
+    static rtl::OUString calculateHash(
+            const rtl::OUString& rPassword,
+            const rtl::OUString& rSaltValue,
+            sal_uInt32 nSpinCount,
+            const rtl::OUString& rAlgorithmName);
+
     size_t getLength() const;
 };