external/firebird: Support Clang ASan

First of all, LD_LIBRARY_PATH must not affect the make executable called for the
firebird build.  That would cause problems when that make executable depends on
system versions of dynamic libraries that also happen to be available as LO-
built versions on LD_LIBRARY_PATH:

> make: symbol lookup error: /data/sbergman/lo-san/core/instdir/program/libfreebl3.so: undefined symbol: __asan_option_detect_stack_use_after_return
> make: *** [/data/sbergman/lo-san/core/external/firebird/ExternalProject_firebird.mk:41: /data/sbergman/lo-san/core/workdir/ExternalProject/firebird/build] Error 127

For that, gb_Helper_extend_ld_path is now tunneled to the relevant places in the
firebird build, where executables are called that use firebird's
UnicodeUtil::getConversionICU
(workdir/UnpackedTarball/firebird/src/common/unicode_util.cpp) to dynamically
load icu libraries.  (Similar to 3a281828
"external/liblangtag: Tunnel LD_LIBRARY_PATH to where it's actually needed".)

Then, what appears to work is to disable firebird's global operators new/delete
(so that ASan's versions kick in).

MemPool::{alloc,release}Raw would still be called at start-up through
Firebird::GlobalPtr<Firebird::Mutex, (Firebird::InstanceControl::DtorPriority)3>
::GlobalPtr() calling Firebird::GlobalStorage::operator new(unsigned long), but
there is already provision to avoid those functions' heavy machinery for the
USE_VALGRIND case.

DEBUG_GDS_ALLOC (which would enable other inopportune overloads of global
operator new/delete) needs to be disabled in a somewhat contorted way, because
8ea07101 "external/firebird: Work around
operator new alignment violations" might explicitly set it from the outside,
which we need to undo for ASan builds.

`make check screenshot` encountered one alloc-dealloc-mismatch (already during
building firebird itself) that needs a patch to src/jrd/SimilarToMatcher.h.

Change-Id: I6bb68604095908189259be14440f72d3b23cbd4e
Reviewed-on: https://gerrit.libreoffice.org/52706Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

external/firebird/ExternalProject_firebird.mk

@@ -30,7 +30,7 @@ ifeq ($(COM_IS_CLANG),TRUE)
 firebird_NO_CXX11_NARROWING := -Wno-c++11-narrowing
 endif
 
-MAKE_PRE=$(call gb_Helper_extend_ld_path,$(call gb_UnpackedTarball_get_dir,icu)/source/lib) LC_ALL=C
+MAKE_PRE=LC_ALL=C
 
 MAKE_POST=$(if $(filter MACOSX,$(OS)),&& $(PERL) \
 			$(SRCDIR)/solenv/bin/macosx-change-install-names.pl shl OOO \
@@ -97,9 +97,9 @@ $(call gb_ExternalProject_get_state_target,firebird,build):
 							'<' 101200)), \
 					ac_cv_func_clock_gettime=no)) \
 		&& if [ -n "$${FB_CPU_ARG}" ]; then \
-			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) $(INVOKE_FPA) SHELL='$(SHELL)' $(MAKE_POST); \
+			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) $(INVOKE_FPA) SHELL='$(SHELL)' LIBO_TUNNEL_LIBRARY_PATH='$(subst ','\'',$(subst $$,$$$$,$(call gb_Helper_extend_ld_path,$(call gb_UnpackedTarball_get_dir,icu)/source/lib)))' $(MAKE_POST); \
 			else \
-			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) SHELL='$(SHELL)' $(MAKE_POST); \
+			   $(MAKE_PRE) $(MAKE) $(if $(ENABLE_DEBUG),Debug) SHELL='$(SHELL)' LIBO_TUNNEL_LIBRARY_PATH='$(subst ','\'',$(subst $$,$$$$,$(call gb_Helper_extend_ld_path,$(call gb_UnpackedTarball_get_dir,icu)/source/lib)))' $(MAKE_POST); \
 			fi \
 	)
 # vim: set noet sw=4 ts=4:

external/firebird/UnpackedTarball_firebird.mk

@@ -30,6 +30,7 @@ $(eval $(call gb_UnpackedTarball_add_patches,firebird,\
 		external/firebird/0002-Backported-fix-for-CORE-5452-Segfault-when-engine-s-.patch.1 \
 		external/firebird/c++17.patch \
 		external/firebird/ubsan.patch \
+		external/firebird/asan.patch \
 ))
 
 ifeq ($(OS),WNT)

external/firebird/asan.patch

+--- builds/posix/Makefile.in
++++ builds/posix/Makefile.in
+@@ -323,8 +323,8 @@
+ 
+ metadata.fdb:	$(RUN_ISQL) $(SRC_ROOT)/dbs/metadata.sql
+ 	-$(RM) $@
+-	$(RUN_ISQL) -q -i $(SRC_ROOT)/dbs/metadata.sql
+-	$(RUN_GFIX) -mode read_only $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL) -q -i $(SRC_ROOT)/dbs/metadata.sql
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -mode read_only $@
+ 	$(CHMOD) 0444 $@
+ 
+ $(HELP_FDB):	help.fdb
+@@ -333,7 +333,7 @@
+ 	$(CHMOD) 0444 $@
+ 
+ help.fdb:	$(BLD_ROOT)/misc/help.gbak
+-	$(RUN_GBAK) -MODE read_only -R $< $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GBAK) -MODE read_only -R $< $@
+ 	$(CHMOD) 0444 $@
+ 
+ $(SECURITY_FDB):	security.fdb
+@@ -343,18 +343,18 @@
+ security.fdb:	$(SRC_ROOT)/dbs/security.sql
+ 	-$(RM) $@
+ 	-$(RM) $(SECURITY_TMP)
+-	echo create database \'$(SECURITY_TMP)\'\; | $(RUN_ISQL)
+-	$(RUN_GFIX) -write async $(SECURITY_TMP)
+-	$(RUN_ISQL) -i $^ $(SECURITY_TMP)
++	echo create database \'$(SECURITY_TMP)\'\; | $(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL)
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -write async $(SECURITY_TMP)
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL) -i $^ $(SECURITY_TMP)
+ 	$(CHMOD) a=rw $(SECURITY_TMP)
+ 	$(CP) $(SECURITY_TMP) $@
+-	$(RUN_GFIX) -write sync $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -write sync $@
+ 
+ msg.timestamp:	$(MSG_FILES)
+ 	-$(RM) msg.fdb
+-	echo create database \'msg.fdb\'\; | $(RUN_ISQL)
+-	$(RUN_GFIX) -write async msg.fdb
+-	for sql in $(MSG_FILES); do (echo $$sql; $(RUN_ISQL) -i $$sql msg.fdb) || exit; done
++	echo create database \'msg.fdb\'\; | $(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL)
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_GFIX) -write async msg.fdb
++	for sql in $(MSG_FILES); do (echo $$sql; $(LIBO_TUNNEL_LIBRARY_PATH) $(RUN_ISQL) -i $$sql msg.fdb) || exit; done
+ 	$(TOUCH) $@
+ 
+ 
+@@ -560,7 +560,7 @@
+ message_file:	$(FIREBIRD_MSG)
+ 
+ $(FIREBIRD_MSG):	$(BUILD_FILE) msg.timestamp
+-	$(BUILD_FILE) -d msg.fdb -f $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(BUILD_FILE) -d msg.fdb -f $@
+ 	$(CHMOD_6) $@
+ 
+ $(BUILD_FILE):	$(BUILD_Objects) $(COMMON_LIB)
+--- builds/posix/make.rules
++++ builds/posix/make.rules
+@@ -68,17 +68,17 @@
+ .SUFFIXES: .c .e .epp .cpp
+ 
+ .e.c:
+-	$(GPRE_CURRENT) $(GPRE_FLAGS) $< $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(GPRE_FLAGS) $< $@
+ 
+ 
+ $(OBJ)/jrd/%.cpp: $(SRC_ROOT)/jrd/%.epp
+-	$(GPRE_CURRENT) $(JRD_GPRE_FLAGS) $(firstword $<) $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(JRD_GPRE_FLAGS) $(firstword $<) $@
+ 
+ $(OBJ)/isql/%.cpp: $(SRC_ROOT)/isql/%.epp
+-	$(GPRE_CURRENT) $(ISQL_GPRE_FLAGS) $< $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(ISQL_GPRE_FLAGS) $< $@
+ 
+ $(OBJ)/%.cpp: $(SRC_ROOT)/%.epp
+-	$(GPRE_CURRENT) $(GPRE_FLAGS) $(firstword $<) $@
++	$(LIBO_TUNNEL_LIBRARY_PATH) $(GPRE_CURRENT) $(GPRE_FLAGS) $(firstword $<) $@
+ 
+ 
+ .SUFFIXES: .lo .o .cpp .c
+--- src/common/classes/alloc.cpp
++++ src/common/classes/alloc.cpp
+@@ -2187,7 +2187,7 @@
+ 
+ void* MemPool::allocRaw(size_t size)
+ {
+-#ifndef USE_VALGRIND
++#if !(defined USE_VALGRIND || defined USE_ASAN)
+ 	if (size == DEFAULT_ALLOCATION)
+ 	{
+ 		MutexLockGuard guard(*cache_mutex, "MemPool::allocRaw");
+@@ -2267,7 +2267,7 @@
+ 
+ void MemPool::releaseRaw(bool destroying, void* block, size_t size, bool use_cache) throw ()
+ {
+-#ifndef USE_VALGRIND
++#if !(defined USE_VALGRIND || defined USE_ASAN)
+ 	if (use_cache && (size == DEFAULT_ALLOCATION))
+ 	{
+ 		MutexLockGuard guard(*cache_mutex, "MemPool::releaseRaw");
+@@ -2277,7 +2277,7 @@
+ 			return;
+ 		}
+ 	}
+-#else
++#elif defined USE_VALGRIND
+ 	// Set access protection for block to prevent memory from deleted pool being accessed
+ 	int handle = /* //VALGRIND_MAKE_NOACCESS */ VALGRIND_MAKE_MEM_DEFINED(block, size);
+ 
+--- src/common/classes/alloc.h
++++ src/common/classes/alloc.h
+@@ -295,40 +295,60 @@
+ 
+ // operators new and delete
+ 
++#if !defined USE_ASAN
+ inline void* operator new(size_t s ALLOC_PARAMS)
+ {
+ 	return MemoryPool::globalAlloc(s ALLOC_PASS_ARGS);
+ }
+ inline void* operator new[](size_t s ALLOC_PARAMS)
+ {
+ 	return MemoryPool::globalAlloc(s ALLOC_PASS_ARGS);
+ }
++#endif
+ 
+ inline void* operator new(size_t s, Firebird::MemoryPool& pool ALLOC_PARAMS)
+ {
++#if defined USE_ASAN
++	return operator new(s);
++#else
+ 	return pool.allocate(s ALLOC_PASS_ARGS);
++#endif
+ }
+ inline void* operator new[](size_t s, Firebird::MemoryPool& pool ALLOC_PARAMS)
+ {
++#if defined USE_ASAN
++	return operator new[](s);
++#else
+ 	return pool.allocate(s ALLOC_PASS_ARGS);
++#endif
+ }
+ 
++#if !defined USE_ASAN
+ inline void operator delete(void* mem ALLOC_PARAMS) throw()
+ {
+ 	MemoryPool::globalFree(mem);
+ }
+ inline void operator delete[](void* mem ALLOC_PARAMS) throw()
+ {
+ 	MemoryPool::globalFree(mem);
+ }
++#endif
+ 
+ inline void operator delete(void* mem, Firebird::MemoryPool& pool ALLOC_PARAMS) throw()
+ {
++#if defined USE_ASAN
++	return operator delete(mem);
++#else
+ 	MemoryPool::globalFree(mem);
++#endif
+ }
+ inline void operator delete[](void* mem, Firebird::MemoryPool& pool ALLOC_PARAMS) throw()
+ {
++#if defined USE_ASAN
++	return operator delete[](mem);
++#else
+ 	MemoryPool::globalFree(mem);
++#endif
+ }
+ 
+ #ifdef DEBUG_GDS_ALLOC
+--- src/include/firebird.h
++++ src/include/firebird.h
+@@ -38,10 +38,19 @@
+ #include "gen/autoconfig.h"
+ #endif
+ 
++#if defined __clang__ && defined __has_feature
++#if __has_feature(address_sanitizer)
++#define USE_ASAN
++#endif
++#endif
++
+ // Using our debugging code is pointless when we may use Valgrind features
+ #if defined(DEV_BUILD) && !defined(USE_VALGRIND)
+ #define DEBUG_GDS_ALLOC
+ #endif
++#if defined USE_ASAN
++#undef DEBUG_GDS_ALLOC
++#endif
+ 
+ #if defined(WIN_NT)
+ #define FB_DLL_EXPORT __declspec(dllexport)
+--- src/jrd/SimilarToMatcher.h
++++ src/jrd/SimilarToMatcher.h
+@@ -338,7 +338,7 @@
+ 		private:
+ 			static const unsigned INCREASE_FACTOR = 50;
+ 			unsigned size;
+-			AutoPtr<UCHAR> data;
++			AutoPtr<UCHAR, ArrayDelete<UCHAR>> data;
+ 			T* end;
+ 		};
+ #endif	// RECURSIVE_SIMILAR

external/firebird/firebird-cygwin-msvc.patch

@@ -10,8 +10,8 @@
  #include "gen/autoconfig.h"
 -#endif
  
- // Using our debugging code is pointless when we may use Valgrind features
- #if defined(DEV_BUILD) && !defined(USE_VALGRIND)
+ #if defined __clang__ && defined __has_feature
+ #if __has_feature(address_sanitizer)
 --- src/misc/writeBuildNum.sh	2016-07-07 15:57:04.538983200 +0200
 +++ src/misc/writeBuildNum.sh	2016-07-13 11:31:18.132820200 +0200
 @@ -95,9 +95,9 @@