support pip install src-pckage

5 files changed, 86 insertions, 36 deletions

diff --git a/VERSION b/VERSION
index 7977b63..e982215 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.1.93
+0.1.94
diff --git a/python/MANIFEST.in b/python/MANIFEST.in
index 4d5c58a..9d9603f 100644
--- a/python/MANIFEST.in
+++ b/python/MANIFEST.in
@@ -1,3 +1,3 @@
 recursive-include test *.py *.model botchan.txt
 include *.i *.md VERSION
-
+include build_bundled.sh
diff --git a/python/VERSION b/python/VERSION
index 7977b63..e982215 100644
--- a/python/VERSION
+++ b/python/VERSION
@@ -1 +1 @@
-0.1.93
+0.1.94
diff --git a/python/build_bundled.sh b/python/build_bundled.sh
new file mode 100755
index 0000000..496702a
--- /dev/null
+++ b/python/build_bundled.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+VERSION="$1"
+
+mkdir bundled
+cd bundled
+# Try taged version. Othewise, use head.
+git clone https://github.com/google/sentencepiece.git \
+  -b v"${VERSION}" --depth 1 || \
+  git clone https://github.com/google/sentencepiece.git --depth 1
+
+cd sentencepiece
+mkdir build
+cd build
+cmake .. -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=../..
+make -j $(nproc)
+make install
+cd ../..
diff --git a/python/setup.py b/python/setup.py
index 910128f..3120d86 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -15,6 +15,8 @@
 # limitations under the License.!
 
 from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext as _build_ext
+from setuptools.command.build_py import build_py as _build_py
 import codecs
 import string
 import subprocess
@@ -23,65 +25,95 @@ import os
 
 sys.path.append(os.path.join('.', 'test'))
 
-with codecs.open('README.md', 'r', 'utf-8') as f:
-  long_description = f.read()
 
-with codecs.open('VERSION', 'r', 'utf-8') as f:
-  version = f.read().rstrip()
+def long_description():
+  with codecs.open('README.md', 'r', 'utf-8') as f:
+    long_description = f.read()
+  return long_description
 
 
-def cmd(line):
+def version():
+  with codecs.open('VERSION', 'r', 'utf-8') as f:
+    version = f.read().rstrip()
+    return version
+
+
+def run_pkg_config(section, pkg_config_path=None):
   try:
-    output = subprocess.check_output(line, shell=True)
+    cmd = 'pkg-config sentencepiece --{}'.format(section)
+    if pkg_config_path:
+      cmd = 'env PKG_CONFIG_PATH={} {}'.format(pkg_config_path, cmd)
+    output = subprocess.check_output(cmd, shell=True)
     if sys.version_info >= (3, 0, 0):
       output = output.decode('utf-8')
   except subprocess.CalledProcessError:
-    sys.stderr.write('Failed to find sentencepiece pkgconfig\n')
+    sys.stderr.write('Failed to find sentencepiece pkg-config\n')
     sys.exit(1)
   return output.strip().split()
 
 
-# Fix compile on some versions of Mac OSX
-# See: https://github.com/neulab/xnmt/issues/199
-def cflags():
-  if sys.platform == 'win32':
-    return ['/MT', '/I..\\build\\root\\include']
-  args = ['-std=c++11']
-  if sys.platform == 'darwin':
-    args.append('-mmacosx-version-min=10.9')
-  args = args + cmd('pkg-config sentencepiece --cflags')
-  return args
+def is_sentencepiece_installed():
+  return subprocess.run(
+      'pkg-config sentencepiece --libs', shell=True,
+      check=False).returncode == 0
+
+
+class build_ext(_build_ext):
+  """Override build_extension to run cmake."""
 
+  def build_extension(self, ext):
+    pkg_config_path = None
+    if not is_sentencepiece_installed():
+      subprocess.run(['./build_bundled.sh', version()], check=True)
+      pkg_config_path = './bundled/lib/pkgconfig:./bundled/lib64/pkgconfig'
 
-def libs():
-  if sys.platform == 'win32':
-    return [
-        '..\\build\\root\\lib\\sentencepiece.lib',
-        '..\\build\\root\\lib\\sentencepiece_train.lib'
-    ]
+    cflags = ['-std=c++11']
+    # Fix compile on some versions of Mac OSX
+    # See: https://github.com/neulab/xnmt/issues/199
+    if sys.platform == 'darwin':
+      cflags.append('-mmacosx-version-min=10.9')
+    cflags = cflags + run_pkg_config('cflags', pkg_config_path)
+    libs = run_pkg_config('libs', pkg_config_path)
+    print('## cflags={}'.format(' '.join(cflags)))
+    print('## libs={}'.format(' '.join(libs)))
+    ext.extra_compile_args = cflags
+    ext.extra_link_args = libs
+    _build_ext.build_extension(self, ext)
 
-  return cmd('pkg-config sentencepiece --libs')
 
+if os.name == 'nt':
+  cflags = ['/MT', '/I..\\build\\root\\include']
+  libs = [
+      '..\\build\\root\\lib\\sentencepiece.lib',
+      '..\\build\\root\\lib\\sentencepiece_train.lib'
+  ]
+  SENTENCEPIECE_EXT = Extension(
+      '_sentencepiece',
+      sources=['sentencepiece_wrap.cxx'],
+      extra_compile_args=cflags,
+      extra_link_args=libs)
+  cmdclass = {}
+else:
+  SENTENCEPIECE_EXT = Extension(
+      '_sentencepiece', sources=['sentencepiece_wrap.cxx'])
+  cmdclass = {'build_ext': build_ext}
 
 setup(
     name='sentencepiece',
     author='Taku Kudo',
     author_email='taku@google.com',
     description='SentencePiece python wrapper',
-    long_description=long_description,
+    long_description=long_description(),
     long_description_content_type='text/markdown',
-    version=version,
+    version=version(),
     url='https://github.com/google/sentencepiece',
     license='Apache',
     platforms='Unix',
-    py_modules=['sentencepiece'],
-    ext_modules=[
-        Extension(
-            '_sentencepiece',
-            sources=['sentencepiece_wrap.cxx'],
-            extra_compile_args=cflags(),
-            extra_link_args=libs())
+    py_modules=[
+        'sentencepiece', 'sentencepiece_model_pb2', 'sentencepiece_pb2'
     ],
+    ext_modules=[SENTENCEPIECE_EXT],
+    cmdclass=cmdclass,
     classifiers=[
         'Development Status :: 5 - Production/Stable', 'Environment :: Console',
         'Intended Audience :: Developers',