From 195a2cc4fbf06988a9b04f8985c538e202998ca3 Mon Sep 17 00:00:00 2001 From: Taku Kudo Date: Wed, 21 Oct 2020 19:28:09 +0900 Subject: support pip install src-pckage --- VERSION | 2 +- python/MANIFEST.in | 2 +- python/VERSION | 2 +- python/build_bundled.sh | 18 +++++++++ python/setup.py | 98 ++++++++++++++++++++++++++++++++----------------- 5 files changed, 86 insertions(+), 36 deletions(-) create mode 100755 python/build_bundled.sh diff --git a/VERSION b/VERSION index 7977b63..e982215 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.93 +0.1.94 diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 4d5c58a..9d9603f 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1,3 +1,3 @@ recursive-include test *.py *.model botchan.txt include *.i *.md VERSION - +include build_bundled.sh diff --git a/python/VERSION b/python/VERSION index 7977b63..e982215 100644 --- a/python/VERSION +++ b/python/VERSION @@ -1 +1 @@ -0.1.93 +0.1.94 diff --git a/python/build_bundled.sh b/python/build_bundled.sh new file mode 100755 index 0000000..496702a --- /dev/null +++ b/python/build_bundled.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +VERSION="$1" + +mkdir bundled +cd bundled +# Try taged version. Othewise, use head. +git clone https://github.com/google/sentencepiece.git \ + -b v"${VERSION}" --depth 1 || \ + git clone https://github.com/google/sentencepiece.git --depth 1 + +cd sentencepiece +mkdir build +cd build +cmake .. -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=../.. +make -j $(nproc) +make install +cd ../.. diff --git a/python/setup.py b/python/setup.py index 910128f..3120d86 100755 --- a/python/setup.py +++ b/python/setup.py @@ -15,6 +15,8 @@ # limitations under the License.! from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as _build_ext +from setuptools.command.build_py import build_py as _build_py import codecs import string import subprocess @@ -23,65 +25,95 @@ import os sys.path.append(os.path.join('.', 'test')) -with codecs.open('README.md', 'r', 'utf-8') as f: - long_description = f.read() -with codecs.open('VERSION', 'r', 'utf-8') as f: - version = f.read().rstrip() +def long_description(): + with codecs.open('README.md', 'r', 'utf-8') as f: + long_description = f.read() + return long_description -def cmd(line): +def version(): + with codecs.open('VERSION', 'r', 'utf-8') as f: + version = f.read().rstrip() + return version + + +def run_pkg_config(section, pkg_config_path=None): try: - output = subprocess.check_output(line, shell=True) + cmd = 'pkg-config sentencepiece --{}'.format(section) + if pkg_config_path: + cmd = 'env PKG_CONFIG_PATH={} {}'.format(pkg_config_path, cmd) + output = subprocess.check_output(cmd, shell=True) if sys.version_info >= (3, 0, 0): output = output.decode('utf-8') except subprocess.CalledProcessError: - sys.stderr.write('Failed to find sentencepiece pkgconfig\n') + sys.stderr.write('Failed to find sentencepiece pkg-config\n') sys.exit(1) return output.strip().split() -# Fix compile on some versions of Mac OSX -# See: https://github.com/neulab/xnmt/issues/199 -def cflags(): - if sys.platform == 'win32': - return ['/MT', '/I..\\build\\root\\include'] - args = ['-std=c++11'] - if sys.platform == 'darwin': - args.append('-mmacosx-version-min=10.9') - args = args + cmd('pkg-config sentencepiece --cflags') - return args +def is_sentencepiece_installed(): + return subprocess.run( + 'pkg-config sentencepiece --libs', shell=True, + check=False).returncode == 0 + + +class build_ext(_build_ext): + """Override build_extension to run cmake.""" + def build_extension(self, ext): + pkg_config_path = None + if not is_sentencepiece_installed(): + subprocess.run(['./build_bundled.sh', version()], check=True) + pkg_config_path = './bundled/lib/pkgconfig:./bundled/lib64/pkgconfig' -def libs(): - if sys.platform == 'win32': - return [ - '..\\build\\root\\lib\\sentencepiece.lib', - '..\\build\\root\\lib\\sentencepiece_train.lib' - ] + cflags = ['-std=c++11'] + # Fix compile on some versions of Mac OSX + # See: https://github.com/neulab/xnmt/issues/199 + if sys.platform == 'darwin': + cflags.append('-mmacosx-version-min=10.9') + cflags = cflags + run_pkg_config('cflags', pkg_config_path) + libs = run_pkg_config('libs', pkg_config_path) + print('## cflags={}'.format(' '.join(cflags))) + print('## libs={}'.format(' '.join(libs))) + ext.extra_compile_args = cflags + ext.extra_link_args = libs + _build_ext.build_extension(self, ext) - return cmd('pkg-config sentencepiece --libs') +if os.name == 'nt': + cflags = ['/MT', '/I..\\build\\root\\include'] + libs = [ + '..\\build\\root\\lib\\sentencepiece.lib', + '..\\build\\root\\lib\\sentencepiece_train.lib' + ] + SENTENCEPIECE_EXT = Extension( + '_sentencepiece', + sources=['sentencepiece_wrap.cxx'], + extra_compile_args=cflags, + extra_link_args=libs) + cmdclass = {} +else: + SENTENCEPIECE_EXT = Extension( + '_sentencepiece', sources=['sentencepiece_wrap.cxx']) + cmdclass = {'build_ext': build_ext} setup( name='sentencepiece', author='Taku Kudo', author_email='taku@google.com', description='SentencePiece python wrapper', - long_description=long_description, + long_description=long_description(), long_description_content_type='text/markdown', - version=version, + version=version(), url='https://github.com/google/sentencepiece', license='Apache', platforms='Unix', - py_modules=['sentencepiece'], - ext_modules=[ - Extension( - '_sentencepiece', - sources=['sentencepiece_wrap.cxx'], - extra_compile_args=cflags(), - extra_link_args=libs()) + py_modules=[ + 'sentencepiece', 'sentencepiece_model_pb2', 'sentencepiece_pb2' ], + ext_modules=[SENTENCEPIECE_EXT], + cmdclass=cmdclass, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', -- cgit v1.2.3