1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
#!/usr/bin/env python
# Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.!
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext as _build_ext
from setuptools.command.build_py import build_py as _build_py
import codecs
import string
import subprocess
import sys
import os
sys.path.append(os.path.join('.', 'test'))
def long_description():
with codecs.open('README.md', 'r', 'utf-8') as f:
long_description = f.read()
return long_description
def version():
with codecs.open('VERSION', 'r', 'utf-8') as f:
version = f.read().rstrip()
return version
def run_pkg_config(section, pkg_config_path=None):
try:
cmd = 'pkg-config sentencepiece --{}'.format(section)
if pkg_config_path:
cmd = 'env PKG_CONFIG_PATH={} {}'.format(pkg_config_path, cmd)
output = subprocess.check_output(cmd, shell=True)
if sys.version_info >= (3, 0, 0):
output = output.decode('utf-8')
except subprocess.CalledProcessError:
sys.stderr.write('Failed to find sentencepiece pkg-config\n')
sys.exit(1)
return output.strip().split()
def is_sentencepiece_installed():
try:
subprocess.check_call('pkg-config sentencepiece --libs', shell=True)
return True
except subprocess.CalledProcessError:
return False
class build_ext(_build_ext):
"""Override build_extension to run cmake."""
def build_extension(self, ext):
pkg_config_path = None
if not is_sentencepiece_installed():
subprocess.run(['./build_bundled.sh', version()], check=True)
pkg_config_path = './bundled/lib/pkgconfig:./bundled/lib64/pkgconfig'
cflags = ['-std=c++11']
# Fix compile on some versions of Mac OSX
# See: https://github.com/neulab/xnmt/issues/199
if sys.platform == 'darwin':
cflags.append('-mmacosx-version-min=10.9')
cflags = cflags + run_pkg_config('cflags', pkg_config_path)
libs = run_pkg_config('libs', pkg_config_path)
print('## cflags={}'.format(' '.join(cflags)))
print('## libs={}'.format(' '.join(libs)))
ext.extra_compile_args = cflags
ext.extra_link_args = libs
_build_ext.build_extension(self, ext)
if os.name == 'nt':
cflags = ['/MT', '/I..\\build\\root\\include']
libs = [
'..\\build\\root\\lib\\sentencepiece.lib',
'..\\build\\root\\lib\\sentencepiece_train.lib'
]
SENTENCEPIECE_EXT = Extension(
'sentencepiece._sentencepiece',
sources=['src/sentencepiece/sentencepiece_wrap.cxx'],
extra_compile_args=cflags,
extra_link_args=libs)
cmdclass = {}
else:
SENTENCEPIECE_EXT = Extension(
'sentencepiece._sentencepiece',
sources=['src/sentencepiece/sentencepiece_wrap.cxx'])
cmdclass = {'build_ext': build_ext}
setup(
name='sentencepiece',
author='Taku Kudo',
author_email='taku@google.com',
description='SentencePiece python wrapper',
long_description=long_description(),
long_description_content_type='text/markdown',
version=version(),
package_dir={'': 'src'},
url='https://github.com/google/sentencepiece',
license='Apache',
platforms='Unix',
py_modules=[
'sentencepiece/__init__', 'sentencepiece/sentencepiece_model_pb2',
'sentencepiece/sentencepiece_pb2'
],
ext_modules=[SENTENCEPIECE_EXT],
cmdclass=cmdclass,
classifiers=[
'Development Status :: 5 - Production/Stable', 'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: Apache Software License',
'Operating System :: Unix', 'Programming Language :: Python',
'Topic :: Text Processing :: Linguistic',
'Topic :: Software Development :: Libraries :: Python Modules'
],
test_suite='sentencepiece_test.suite')
|
