tdf#104734 Firebird improve XClob implementation

Create a more effective implementation of XClob::length() and
XClob::getSubString() methods, where string is read segment-by-segment
instead of reading the whole underlying blob. That way it is possible to
handle big texts which would not fit into memory.

Also allow reading Clob data from a resultset with getString() and
writing it in a prepared statement with setString().

Implement XPreparedStatement::setClob(). Also implement a private
version of setClob() for creating a clob from OUString:

Allow the creation of a clob column with GUI by adding a new type in
ODataBaseMetaData::getTypeInfo().

Change-Id: Ibcbbdd80e8eed5e2a3fe55b0fa196401f1bcbcdf
Reviewed-on: https://gerrit.libreoffice.org/47093Reviewed-by: Tamás Bunth <btomi96@gmail.com>
Tested-by: Tamás Bunth <btomi96@gmail.com>

connectivity/source/drivers/firebird/Blob.cxx

@@ -70,9 +70,14 @@ void Blob::ensureBlobIsOpened()
     m_nBlobPosition = 0;
 
     char aBlobItems[] = {
-        isc_info_blob_total_length
+        isc_info_blob_total_length,
+        isc_info_blob_max_segment
     };
-    char aResultBuffer[20];
+
+    // Assuming a data (e.g. legth of blob) is maximum 64 bit.
+    // That means we need 8 bytes for data + 2 for length of data + 1 for item
+    // identifier for each item.
+    char aResultBuffer[11 + 11];
 
     aErr = isc_blob_info(m_statusVector,
                   &m_blobHandle,
@@ -84,17 +89,63 @@ void Blob::ensureBlobIsOpened()
     if (aErr)
         evaluateStatusVector(m_statusVector, "isc_blob_info", *this);
 
-    if (*aResultBuffer == isc_info_blob_total_length)
+    char* pIt = aResultBuffer;
+    while( *pIt != isc_info_end ) // info is in clusters
     {
-        short aResultLength = (short) isc_vax_integer(aResultBuffer+1, 2);
-        m_nBlobLength =  isc_vax_integer(aResultBuffer+3, aResultLength);
+        char item = *pIt++;
+        short aResultLength = (short) isc_vax_integer(pIt, 2);
+
+        pIt += 2;
+        switch(item)
+        {
+            case isc_info_blob_total_length:
+                m_nBlobLength = isc_vax_integer(pIt, aResultLength);
+                break;
+            case isc_info_blob_max_segment:
+                m_nMaxSegmentSize = isc_vax_integer(pIt, aResultLength);
+                break;
+            default:
+                assert(false);
+                break;
+        }
+        pIt += aResultLength;
     }
-    else
+}
+
+sal_uInt16 Blob::getMaximumSegmentSize()
+{
+    ensureBlobIsOpened();
+
+    return m_nMaxSegmentSize;
+}
+
+bool Blob::readOneSegment(uno::Sequence< sal_Int8 >& rDataOut)
+{
+    checkDisposed(Blob_BASE::rBHelper.bDisposed);
+    ensureBlobIsOpened();
+
+    sal_uInt16 nMaxSize = getMaximumSegmentSize();
+
+    if(rDataOut.getLength() < nMaxSize)
+        rDataOut.realloc(nMaxSize);
+
+    sal_uInt16 nActualSize = 0;
+    ISC_STATUS aRet = isc_get_segment(m_statusVector,
+            &m_blobHandle,
+            &nActualSize,
+            nMaxSize,
+            reinterpret_cast<char*>(rDataOut.getArray()) );
+
+    if (aRet && aRet != isc_segstr_eof && IndicatesError(m_statusVector))
     {
-        assert(false);
+        OUString sError(StatusVectorToString(m_statusVector, "isc_get_segment"));
+        throw IOException(sError, *this);
     }
+    m_nBlobPosition += nActualSize;
+    return aRet == isc_segstr_eof;  // last segment read
 }
 
+
 void Blob::closeBlob()
 {
     MutexGuard aGuard(m_aMutex);

connectivity/source/drivers/firebird/Blob.hxx

@@ -41,6 +41,7 @@ namespace connectivity
 
             bool                m_bBlobOpened;
             sal_Int64           m_nBlobLength;
+            sal_uInt16          m_nMaxSegmentSize;
             sal_Int64           m_nBlobPosition;
 
             ISC_STATUS_ARRAY    m_statusVector;
@@ -54,12 +55,15 @@ namespace connectivity
              * @throws css::sdbc::SQLException
              */
             void closeBlob();
+            sal_uInt16 getMaximumSegmentSize();
 
         public:
             Blob(isc_db_handle* pDatabaseHandle,
                  isc_tr_handle* pTransactionHandle,
                  ISC_QUAD const & aBlobID);
 
+            bool readOneSegment(css::uno::Sequence< sal_Int8 >& rDataOut);
+
             // ---- XBlob ----------------------------------------------------
             virtual sal_Int64 SAL_CALL
                 length() override;

connectivity/source/drivers/firebird/Clob.cxx

@@ -28,7 +28,8 @@ Clob::Clob(isc_db_handle* pDatabaseHandle,
            isc_tr_handle* pTransactionHandle,
            ISC_QUAD const & aBlobID):
     Clob_BASE(m_aMutex),
-    m_aBlob(new connectivity::firebird::Blob(pDatabaseHandle, pTransactionHandle, aBlobID))
+    m_aBlob(new connectivity::firebird::Blob(pDatabaseHandle, pTransactionHandle, aBlobID)),
+    m_nCharCount(-1)
 {
 }
 
@@ -44,13 +45,27 @@ sal_Int64 SAL_CALL Clob::length()
     MutexGuard aGuard(m_aMutex);
     checkDisposed(Clob_BASE::rBHelper.bDisposed);
 
-    // read the entire blob
-    // TODO FIXME better solution?
-    uno::Sequence < sal_Int8 > aEntireBlob = m_aBlob->getBytes( 1, m_aBlob->length());
-    OUString sEntireClob (  reinterpret_cast< sal_Char *>( aEntireBlob.getArray() ),
-                            aEntireBlob.getLength(),
+    if( m_nCharCount >= 0 )
+        return m_nCharCount;
+    m_nCharCount = 0;
+
+    // Read each segment, and calculate it's size by interpreting it as a
+    // character stream. Assume that no characters are split by the segments.
+    bool bLastSegmRead = false;
+    do
+    {
+        uno::Sequence < sal_Int8 > aSegmentBytes;
+        bLastSegmRead = m_aBlob->readOneSegment( aSegmentBytes );
+        OUString sSegment ( reinterpret_cast< sal_Char *>( aSegmentBytes.getArray() ),
+                            aSegmentBytes.getLength(),
                             RTL_TEXTENCODING_UTF8 );
-    return sEntireClob.getLength();
+
+        if( !bLastSegmRead)
+            m_nCharCount += sSegment.getLength();
+    }while( !bLastSegmRead );
+
+    m_aBlob->closeInput(); // reset position
+    return m_nCharCount;
 }
 
 OUString SAL_CALL Clob::getSubString(sal_Int64 nPosition,
@@ -58,19 +73,58 @@ OUString SAL_CALL Clob::getSubString(sal_Int64 nPosition,
 {
     MutexGuard aGuard(m_aMutex);
     checkDisposed(Clob_BASE::rBHelper.bDisposed);
-
-    // read the entire blob
-    // TODO FIXME better solution?
-    // TODO FIXME Assume indexing of nPosition starts at position 1.
-    uno::Sequence < sal_Int8 > aEntireBlob = m_aBlob->getBytes( 1, m_aBlob->length());
-    OUString sEntireClob (  reinterpret_cast< sal_Char *>( aEntireBlob.getArray() ),
-                            aEntireBlob.getLength(),
+    // TODO do not reset position if it is not necessary
+    m_aBlob->closeInput(); // reset position
+
+    OUStringBuffer sSegmentBuffer;
+    sal_Int64 nActPos = 1;
+    sal_Int32 nActLen = 0;
+
+    // skip irrelevant parts
+    while( nActPos < nPosition )
+    {
+        uno::Sequence < sal_Int8 > aSegmentBytes;
+        bool bLastRead = m_aBlob->readOneSegment( aSegmentBytes );
+        if( bLastRead )
+            throw lang::IllegalArgumentException("nPosition out of range", *this, 0);
+
+        OUString sSegment ( reinterpret_cast< sal_Char *>( aSegmentBytes.getArray() ),
+                            aSegmentBytes.getLength(),
                             RTL_TEXTENCODING_UTF8 );
-
-    if( nPosition + nLength - 1 > sEntireClob.getLength() )
-        throw lang::IllegalArgumentException("nPosition out of range", *this, 0);
-
-    return sEntireClob.copy(nPosition - 1 , nLength);
+        sal_Int32 nStrLen = sSegment.getLength();
+        nActPos += nStrLen;
+        if( nActPos > nPosition )
+        {
+            sal_Int32 nCharsToCopy = static_cast<sal_Int32>(nActPos - nPosition);
+            if( nCharsToCopy > nLength )
+                nCharsToCopy = nLength;
+            // append relevant part of first segment
+            sSegmentBuffer.append( sSegment.copy(0, nCharsToCopy ) );
+            nActLen += sSegmentBuffer.getLength();
+        }
+    }
+
+    // read nLength characters
+    while( nActLen < nLength )
+    {
+        uno::Sequence < sal_Int8 > aSegmentBytes;
+        bool bLastRead = m_aBlob->readOneSegment( aSegmentBytes );
+
+        OUString sSegment ( reinterpret_cast< sal_Char *>( aSegmentBytes.getArray() ),
+                            aSegmentBytes.getLength(),
+                            RTL_TEXTENCODING_UTF8 );
+        sal_Int32 nStrLen = sSegment.getLength();
+        if( nActLen + nStrLen > nLength )
+            sSegmentBuffer.append(sSegment.copy(0, nLength - nActLen) );
+        else
+            sSegmentBuffer.append(sSegment);
+        nActLen += nStrLen;
+
+        if( bLastRead && nActLen < nLength )
+            throw lang::IllegalArgumentException("out of range", *this, 0);
+    }
+
+    return sSegmentBuffer.makeStringAndClear();
 }
 
 uno::Reference< XInputStream > SAL_CALL  Clob::getCharacterStream()

connectivity/source/drivers/firebird/Clob.hxx

@@ -38,6 +38,8 @@ namespace connectivity
              */
             rtl::Reference<connectivity::firebird::Blob> m_aBlob;
 
+            sal_Int64 m_nCharCount;
+
         public:
             Clob(isc_db_handle* pDatabaseHandle,
                  isc_tr_handle* pTransactionHandle,

connectivity/source/drivers/firebird/DatabaseMetaData.cxx

@@ -870,6 +870,15 @@ uno::Reference< XResultSet > SAL_CALL ODatabaseMetaData::getTypeInfo()
         aRow[6] = new ORowSetValueDecorator(OUString("length")); // Create Params
         aRow[9] = new ORowSetValueDecorator(
                 sal_Int16(ColumnSearch::NONE)); // Searchable
+
+        // Clob (SQL_BLOB)
+        aRow[1] = new ORowSetValueDecorator(OUString("BLOB")); // BLOB, with subtype 1
+        aRow[2] = new ORowSetValueDecorator(DataType::CLOB);
+        aRow[3] = new ORowSetValueDecorator(sal_Int16(2147483647)); // Precision = max length
+        aRow[6] = new ORowSetValueDecorator(); // Create Params
+        aRow[9] = new ORowSetValueDecorator(
+                sal_Int16(ColumnSearch::FULL)); // Searchable
+        aRow[12] = new ORowSetValueDecorator(false); // Autoincrement
         aRow[14] = ODatabaseMetaDataResultSet::get0Value(); // Minimum scale
         aRow[15] = ODatabaseMetaDataResultSet::get0Value(); // Max scale
         aResults.push_back(aRow);

connectivity/source/drivers/firebird/PreparedStatement.cxx

@@ -178,10 +178,10 @@ void SAL_CALL OPreparedStatement::disposing()
 }
 
 void SAL_CALL OPreparedStatement::setString(sal_Int32 nParameterIndex,
-                                            const OUString& x)
+                                            const OUString& sInput)
 {
     SAL_INFO("connectivity.firebird",
-             "setString(" << nParameterIndex << " , " << x << ")");
+             "setString(" << nParameterIndex << " , " << sInput << ")");
 
     MutexGuard aGuard( m_aMutex );
     checkDisposed(OStatementCommonBase_Base::rBHelper.bDisposed);
@@ -190,7 +190,7 @@ void SAL_CALL OPreparedStatement::setString(sal_Int32 nParameterIndex,
     checkParameterIndex(nParameterIndex);
     setParameterNull(nParameterIndex, false);
 
-    OString str = OUStringToOString(x , RTL_TEXTENCODING_UTF8 );
+    OString str = OUStringToOString(sInput , RTL_TEXTENCODING_UTF8 );
 
     XSQLVAR* pVar = m_pInSqlda->sqlvar + (nParameterIndex - 1);
 
@@ -219,6 +219,10 @@ void SAL_CALL OPreparedStatement::setString(sal_Int32 nParameterIndex,
         // Fill remainder with spaces
         memset(pVar->sqldata + str.getLength(), ' ', pVar->sqllen - str.getLength());
         break;
+    case SQL_BLOB: // Clob
+        assert( pVar->sqlsubtype == static_cast<short>(BlobSubtype::Clob) );
+        setClob(nParameterIndex, sInput );
+        break;
     default:
         ::dbtools::throwSQLException(
             "Incorrect type for setString",
@@ -504,21 +508,105 @@ void OPreparedStatement::closeBlobAfterWriting(isc_blob_handle& rBlobHandle)
     ISC_STATUS aErr;
 
     aErr = isc_close_blob(m_statusVector,
-                          &rBlobHandle);
+            &rBlobHandle);
     if (aErr)
     {
         evaluateStatusVector(m_statusVector,
-                             "isc_close_blob failed",
-                             *this);
+                "isc_close_blob failed",
+                *this);
+        assert(false);
+    }
+}
+
+void SAL_CALL OPreparedStatement::setClob(sal_Int32 nParameterIndex, const Reference< XClob >& xClob )
+{
+    ::osl::MutexGuard aGuard( m_aMutex );
+    checkDisposed(OStatementCommonBase_Base::rBHelper.bDisposed);
+
+#if SAL_TYPES_SIZEOFPOINTER == 8
+    isc_blob_handle aBlobHandle = 0;
+#else
+    isc_blob_handle aBlobHandle = nullptr;
+#endif
+    ISC_QUAD aBlobId;
+
+    openBlobForWriting(aBlobHandle, aBlobId);
+
+
+    // Max segment size is 2^16 == SAL_MAX_UINT16
+    // SAL_MAX_UINT16 / 4 is surely enough for UTF-8
+    // TODO apply max segment size to character encoding
+    sal_Int64 nCharWritten = 1; // XClob is indexed from 1
+    ISC_STATUS aErr = 0;
+    sal_Int64 nLen = xClob->length();
+    while ( nLen > nCharWritten )
+    {
+        sal_Int64 nCharRemain = nLen - nCharWritten;
+        constexpr sal_uInt16 MAX_SIZE = SAL_MAX_UINT16 / 4;
+        sal_uInt16 nWriteSize = (nCharRemain > MAX_SIZE) ? MAX_SIZE : nCharRemain;
+        OString sData = OUStringToOString(
+                xClob->getSubString(nCharWritten, nWriteSize),
+                RTL_TEXTENCODING_UTF8);
+        aErr = isc_put_segment( m_statusVector,
+                &aBlobHandle,
+                sData.getLength(),
+                sData.getStr() );
+        nCharWritten += nWriteSize;
+
+        if (aErr)
+            break;
+    }
+
+    // We need to make sure we close the Blob even if their are errors, hence evaluate
+    // errors after closing.
+    closeBlobAfterWriting(aBlobHandle);
+
+    if (aErr)
+    {
+        evaluateStatusVector(m_statusVector,
+                "isc_put_segment failed",
+                *this);
         assert(false);
     }
+
+    setValue< ISC_QUAD >(nParameterIndex, aBlobId, SQL_BLOB);
 }
 
-void SAL_CALL OPreparedStatement::setClob( sal_Int32, const Reference< XClob >& )
+void OPreparedStatement::setClob( sal_Int32 nParameterIndex, const OUString& rStr )
 {
     ::osl::MutexGuard aGuard( m_aMutex );
     checkDisposed(OStatementCommonBase_Base::rBHelper.bDisposed);
 
+#if SAL_TYPES_SIZEOFPOINTER == 8
+    isc_blob_handle aBlobHandle = 0;
+#else
+    isc_blob_handle aBlobHandle = nullptr;
+#endif
+    ISC_QUAD aBlobId;
+
+    openBlobForWriting(aBlobHandle, aBlobId);
+
+    OString sData = OUStringToOString(
+            rStr,
+            RTL_TEXTENCODING_UTF8);
+    ISC_STATUS aErr = isc_put_segment( m_statusVector,
+                            &aBlobHandle,
+                            sData.getLength(),
+                            sData.getStr() );
+
+    // We need to make sure we close the Blob even if their are errors, hence evaluate
+    // errors after closing.
+    closeBlobAfterWriting(aBlobHandle);
+
+    if (aErr)
+    {
+        evaluateStatusVector(m_statusVector,
+                             "isc_put_segment failed",
+                             *this);
+        assert(false);
+    }
+
+    setValue< ISC_QUAD >(nParameterIndex, aBlobId, SQL_BLOB);
 }
 
 void SAL_CALL OPreparedStatement::setBlob(sal_Int32 nParameterIndex,

connectivity/source/drivers/firebird/PreparedStatement.hxx

@@ -78,6 +78,7 @@ namespace connectivity
              * Assumes that all necessary mutexes have been taken.
              */
             void closeBlobAfterWriting(isc_blob_handle& rBlobHandle);
+            void setClob(sal_Int32 nParamIndex, const OUString& rStr);
 
         protected:
             virtual void SAL_CALL setFastPropertyValue_NoBroadcast(sal_Int32 nHandle,

connectivity/source/drivers/firebird/ResultSet.cxx

@@ -604,6 +604,11 @@ OUString OResultSet::retrieveValue(const sal_Int32 nColumnIndex, const ISC_SHORT
                 return OUString(); // never reached
         }
     }
+    else if(aSqlType == SQL_BLOB && aSqlSubType == static_cast<short>(BlobSubtype::Clob) )
+    {
+        uno::Reference<XClob> xClob = getClob(nColumnIndex);
+        return xClob->getSubString( 0, xClob->length() );
+    }
     else
     {
         return retrieveValue< ORowSetValue >(nColumnIndex, 0);

connectivity/source/drivers/firebird/Tables.cxx

@@ -106,6 +106,13 @@ OUString Tables::createStandardColumnPart(const Reference< XPropertySet >& xColP
             aSql.append(" ");
             aSql.append("CHARACTER SET OCTETS");
         }
+        else if(aType == DataType::CLOB)
+        {
+            // CLOB is a special type of blob in Firebird context.
+            // Subtype number 1 always refers to CLOB
+            aSql.append(" ");
+            aSql.append("SUB_TYPE 1");
+        }
     }
 
     if ( bIsAutoIncrement && !sAutoIncrementValue.isEmpty())

connectivity/source/drivers/firebird/Util.cxx

@@ -118,13 +118,14 @@ sal_Int32 firebird::ColumnTypeInfo::getSdbcType() const
     short aSubType = m_aSubType;
     if( m_nScale > 0 )
     {
-        // scale makes sense only for decimal and numeric types
-        assert(aType == SQL_SHORT || aType == SQL_LONG || aType == SQL_DOUBLE
-                || aType == SQL_INT64);
-
-        // if scale is set without subtype then imply numeric
-        if( static_cast<NumberSubType>(aSubType) == NumberSubType::Other )
-            aSubType = static_cast<short>(NumberSubType::Numeric);
+        // numeric / decimal
+        if(aType == SQL_SHORT || aType == SQL_LONG || aType == SQL_DOUBLE
+                || aType == SQL_INT64)
+        {
+            // if scale is set without subtype then imply numeric
+            if( static_cast<NumberSubType>(aSubType) == NumberSubType::Other )
+                aSubType = static_cast<short>(NumberSubType::Numeric);
+        }
     }
 
     switch (aType)