Build environment: Enable bzip2, lzma and sqlite for Python on Linux

6 files changed, 158 insertions, 1 deletions

diff --git a/build_files/build_environment/CMakeLists.txt b/build_files/build_environment/CMakeLists.txt
index a334adc34ff..512d2567754 100644
--- a/build_files/build_environment/CMakeLists.txt
+++ b/build_files/build_environment/CMakeLists.txt
@@ -138,7 +138,10 @@ if(NOT WIN32 OR ENABLE_MINGW64)
 endif()
 
 if(UNIX AND NOT APPLE)
+	include(cmake/bzip2.cmake)
+	include(cmake/lzma.cmake)
 	include(cmake/ssl.cmake)
+	include(cmake/sqlite.cmake)
 endif()
 
 include(cmake/harvest.cmake)
diff --git a/build_files/build_environment/cmake/bzip2.cmake b/build_files/build_environment/cmake/bzip2.cmake
new file mode 100644
index 00000000000..6c515d10d61
--- /dev/null
+++ b/build_files/build_environment/cmake/bzip2.cmake
@@ -0,0 +1,39 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ***** END GPL LICENSE BLOCK *****
+
+set(BZIP2_PREFIX "${LIBDIR}/bzip2")
+set(BZIP2_CONFIGURE_ENV echo .)
+set(BZIP2_CONFIGURATION_ARGS)
+
+if(UNIX AND NOT APPLE)
+	set(BZIP2_LDFLAGS "-Wl,--as-needed")
+	set(BZIP2_CFLAGS "-fPIC -Wall -Winline -O2 -g -D_FILE_OFFSET_BITS=64")
+	set(BZIP2_CONFIGURE_ENV ${BZIP2_CONFIGURE_ENV} && export LDFLAGS=${BZIP2_LDFLAGS} && export CFLAGS=${BZIP2_CFLAGS}
+		&& export PREFIX=${BZIP2_PREFIX})
+endif()
+
+ExternalProject_Add(external_bzip2
+	URL ${BZIP2_URI}
+	DOWNLOAD_DIR ${DOWNLOAD_DIR}
+	URL_HASH SHA256=${BZIP2_HASH}
+	PREFIX ${BUILD_DIR}/bzip2
+	CONFIGURE_COMMAND echo .
+	BUILD_COMMAND ${BZIP2_CONFIGURE_ENV} && cd ${BUILD_DIR}/bzip2/src/external_bzip2/ && make CFLAGS=${BZIP2_CFLAGS} LDFLAGS=${BZIP2_LDFLAGS} -j${MAKE_THREADS}
+	INSTALL_COMMAND ${BZIP2_CONFIGURE_ENV} && cd ${BUILD_DIR}/bzip2/src/external_bzip2/ && make CFLAGS=${BZIP2_CFLAGS} LDFLAGS=${BZIP2_LDFLAGS} PREFIX=${BZIP2_PREFIX} install
+	INSTALL_DIR ${LIBDIR}/bzip2
+)
diff --git a/build_files/build_environment/cmake/lzma.cmake b/build_files/build_environment/cmake/lzma.cmake
new file mode 100644
index 00000000000..ef8adde2930
--- /dev/null
+++ b/build_files/build_environment/cmake/lzma.cmake
@@ -0,0 +1,32 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ***** END GPL LICENSE BLOCK *****
+
+set(LZMA_PATCH_CMD echo .)
+
+ExternalProject_Add(external_lzma
+	URL ${LZMA_URI}
+	DOWNLOAD_DIR ${DOWNLOAD_DIR}
+	URL_HASH SHA256=${LZMA_HASH}
+	PREFIX ${BUILD_DIR}/lzma
+	PATCH_COMMAND ${LZMA_PATCH_CMD}
+	CONFIGURE_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/lzma/src/external_lzma/ && ${CONFIGURE_COMMAND} --prefix=${LIBDIR}/lzma
+		--disable-shared
+	BUILD_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/lzma/src/external_lzma/ && make -j${MAKE_THREADS}
+	INSTALL_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/lzma/src/external_lzma/ && make install
+	INSTALL_DIR ${LIBDIR}/lzma
+)
diff --git a/build_files/build_environment/cmake/python.cmake b/build_files/build_environment/cmake/python.cmake
index 64ae515cb1b..0e3356e343d 100644
--- a/build_files/build_environment/cmake/python.cmake
+++ b/build_files/build_environment/cmake/python.cmake
@@ -88,10 +88,17 @@ else()
 		set(PYTHON_BINARY ${BUILD_DIR}/python/src/external_python/python.exe)
 		#set(PYTHON_PATCH ${PATCH_CMD} --verbose -p1 -d ${BUILD_DIR}/python/src/external_python < ${PATCH_DIR}/python_apple.diff)
 		set(PYTHON_PATCH echo .)
+		set(PYTHON_CONFIGURE_EXTRA_ENV echo .)
 		set(PYTHON_CONFIGURE_EXTRA_ARGS)
 	else()
 		set(PYTHON_CONFIGURE_EXTRA_ARGS "--with-openssl=${LIBDIR}/ssl")
 		set(PYTHON_CONFIGURE_ENV ${CONFIGURE_ENV})
+		set(PYTHON_CFLAGS "-I${LIBDIR}/sqlite/include -I${LIBDIR}/bzip2/include -I${LIBDIR}/lzma/include")
+		set(PYTHON_LDFLAGS "-L${LIBDIR}/sqlite/lib -L${LIBDIR}/bzip2/lib -L${LIBDIR}/lzma/lib")
+		set(PYTHON_CONFIGURE_EXTRA_ENV
+			export CFLAGS=${PYTHON_CFLAGS} &&
+			export CPPFLAGS=${PYTHON_CFLAGS} &&
+			export LDFLAGS=${PYTHON_LDFLAGS})
 		set(PYTHON_BINARY ${BUILD_DIR}/python/src/external_python/python)
 	endif()
 
@@ -101,7 +108,7 @@ else()
 		URL_HASH MD5=${PYTHON_HASH}
 		PREFIX ${BUILD_DIR}/python
 		PATCH_COMMAND ${PYTHON_PATCH}
-		CONFIGURE_COMMAND ${PYTHON_CONFIGURE_ENV} && cd ${BUILD_DIR}/python/src/external_python/ && ${CONFIGURE_COMMAND} --prefix=${LIBDIR}/python ${PYTHON_CONFIGURE_EXTRA_ARGS}
+		CONFIGURE_COMMAND ${PYTHON_CONFIGURE_ENV} && ${PYTHON_CONFIGURE_EXTRA_ENV} && cd ${BUILD_DIR}/python/src/external_python/ && ${CONFIGURE_COMMAND} --prefix=${LIBDIR}/python ${PYTHON_CONFIGURE_EXTRA_ARGS}
 		BUILD_COMMAND ${PYTHON_CONFIGURE_ENV} && cd ${BUILD_DIR}/python/src/external_python/ && make -j${MAKE_THREADS}
 		INSTALL_COMMAND ${PYTHON_CONFIGURE_ENV} && cd ${BUILD_DIR}/python/src/external_python/ && make install
 		INSTALL_DIR ${LIBDIR}/python)
@@ -168,6 +175,9 @@ endif()
 if(UNIX AND NOT APPLE)
 	add_dependencies(
 		external_python
+		external_bzip2
+		external_lzma
 		external_ssl
+		external_sqlite
 	)
 endif()
diff --git a/build_files/build_environment/cmake/sqlite.cmake b/build_files/build_environment/cmake/sqlite.cmake
new file mode 100644
index 00000000000..e5aa124a414
--- /dev/null
+++ b/build_files/build_environment/cmake/sqlite.cmake
@@ -0,0 +1,57 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ***** END GPL LICENSE BLOCK *****
+
+set(SQLITE_CONFIGURE_ENV echo .)
+set(SQLITE_CONFIGURATION_ARGS)
+
+if(UNIX AND NOT APPLE)
+	set(SQLITE_LDFLAGS -Wl,--as-needed)
+	set(SQLITE_CFLAGS
+		"-DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_COLUMN_METADATA \
+		-DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_PARENTHESIS \
+		-DSQLITE_ENABLE_RTREE=1 -DSQLITE_SOUNDEX=1 \
+		-DSQLITE_ENABLE_UNLOCK_NOTIFY \
+		-DSQLITE_OMIT_LOOKASIDE=1 -DSQLITE_ENABLE_DBSTAT_VTAB \
+		-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 \
+		-DSQLITE_ENABLE_LOAD_EXTENSION \
+		-DSQLITE_ENABLE_JSON1 \
+		-DSQLITE_LIKE_DOESNT_MATCH_BLOBS \
+		-DSQLITE_THREADSAFE=1 \
+		-DSQLITE_ENABLE_FTS3_TOKENIZER=1 \
+		-DSQLITE_MAX_SCHEMA_RETRY=25 \
+		-DSQLITE_ENABLE_PREUPDATE_HOOK \
+		-DSQLITE_ENABLE_SESSION \
+		-DSQLITE_ENABLE_STMTVTAB \
+		-DSQLITE_MAX_VARIABLE_NUMBER=250000 \
+		-fPIC")
+	set(SQLITE_CONFIGURE_ENV ${SQLITE_CONFIGURE_ENV} && export LDFLAGS=${SQLITE_LDFLAGS} && export CFLAGS=${SQLITE_CFLAGS})
+	set(SQLITE_CONFIGURATION_ARGS ${SQLITE_CONFIGURATION_ARGS} --enable-threadsafe --enable-load-extension --enable-json1 --enable-fts4 --enable-fts5
+	    --enable-shared=no)
+endif()
+
+ExternalProject_Add(external_sqlite
+	URL ${SQLITE_URI}
+	DOWNLOAD_DIR ${DOWNLOAD_DIR}
+	URL_HASH SHA1=${SQLITE_HASH}
+	PREFIX ${BUILD_DIR}/sqlite
+	PATCH_COMMAND ${SQLITE_PATCH_CMD}
+	CONFIGURE_COMMAND ${SQLITE_CONFIGURE_ENV} && cd ${BUILD_DIR}/sqlite/src/external_sqlite/ && ${CONFIGURE_COMMAND} --prefix=${LIBDIR}/sqlite ${SQLITE_CONFIGURATION_ARGS}
+	BUILD_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/sqlite/src/external_sqlite/ && make -j${MAKE_THREADS}
+	INSTALL_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/sqlite/src/external_sqlite/ && make install
+	INSTALL_DIR ${LIBDIR}/sqlite
+)
diff --git a/build_files/build_environment/cmake/versions.cmake b/build_files/build_environment/cmake/versions.cmake
index 3b3ee092ca5..a60c5109057 100644
--- a/build_files/build_environment/cmake/versions.cmake
+++ b/build_files/build_environment/cmake/versions.cmake
@@ -275,6 +275,22 @@ set(FLEXBISON_VERSION 2.5.5)
 set(FLEXBISON_URI http://prdownloads.sourceforge.net/winflexbison//win_flex_bison-2.5.5.zip)
 set(FLEXBISON_HASH d87a3938194520d904013abef3df10ce)
 
+# Libraries to keep Python modules static on Linux.
+
+# NOTE: bzip.org domain does no longer belong to BZip 2 project, so we download
+# sources from Debian packaging.
+set(BZIP2_VERSION 1.0.6)
+set(BZIP2_URI http://http.debian.net/debian/pool/main/b/bzip2/bzip2_${BZIP2_VERSION}.orig.tar.bz2)
+set(BZIP2_HASH d70a9ccd8bdf47e302d96c69fecd54925f45d9c7b966bb4ef5f56b770960afa7)
+
+set(LZMA_VERSION 5.2.4)
+set(LZMA_URI https://tukaani.org/xz/xz-${LZMA_VERSION}.tar.bz2)
+set(LZMA_HASH 3313fd2a95f43d88e44264e6b015e7d03053e681860b0d5d3f9baca79c57b7bf)
+
 set(SSL_VERSION 1.1.0i)
 set(SSL_URI https://www.openssl.org/source/openssl-${SSL_VERSION}.tar.gz)
 set(SSL_HASH ebbfc844a8c8cc0ea5dc10b86c9ce97f401837f3fa08c17b2cdadc118253cf99)
+
+set(SQLITE_VERSION 3.24.0)
+set(SQLITE_URI https://www.sqlite.org/2018/sqlite-src-3240000.zip)
+set(SQLITE_HASH fb558c49ee21a837713c4f1e7e413309aabdd9c7)