1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
#!/usr/bin/env python
# Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.!
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext as _build_ext
from setuptools.command.build_py import build_py as _build_py
import codecs
import string
import subprocess
import sys
import os
sys.path.append(os.path.join('.', 'test'))
def long_description():
with codecs.open('README.md', 'r', 'utf-8') as f:
long_description = f.read()
return long_description
def version():
with codecs.open('VERSION', 'r', 'utf-8') as f:
version = f.read().rstrip()
return version
def run_pkg_config(section, pkg_config_path=None):
try:
cmd = 'pkg-config sentencepiece --{}'.format(section)
if pkg_config_path:
cmd = 'env PKG_CONFIG_PATH={} {}'.format(pkg_config_path, cmd)
output = subprocess.check_output(cmd, shell=True)
if sys.version_info >= (3, 0, 0):
output = output.decode('utf-8')
except subprocess.CalledProcessError:
sys.stderr.write('Failed to find sentencepiece pkg-config\n')
sys.exit(1)
return output.strip().split()
def is_sentencepiece_installed():
return subprocess.run(
'pkg-config sentencepiece --libs', shell=True,
check=False).returncode == 0
class build_ext(_build_ext):
"""Override build_extension to run cmake."""
def build_extension(self, ext):
pkg_config_path = None
if not is_sentencepiece_installed():
subprocess.run(['./build_bundled.sh', version()], check=True)
pkg_config_path = './bundled/lib/pkgconfig:./bundled/lib64/pkgconfig'
cflags = ['-std=c++11']
# Fix compile on some versions of Mac OSX
# See: https://github.com/neulab/xnmt/issues/199
if sys.platform == 'darwin':
cflags.append('-mmacosx-version-min=10.9')
cflags = cflags + run_pkg_config('cflags', pkg_config_path)
libs = run_pkg_config('libs', pkg_config_path)
print('## cflags={}'.format(' '.join(cflags)))
print('## libs={}'.format(' '.join(libs)))
ext.extra_compile_args = cflags
ext.extra_link_args = libs
_build_ext.build_extension(self, ext)
if os.name == 'nt':
cflags = ['/MT', '/I..\\build\\root\\include']
libs = [
'..\\build\\root\\lib\\sentencepiece.lib',
'..\\build\\root\\lib\\sentencepiece_train.lib'
]
SENTENCEPIECE_EXT = Extension(
'sentencepiece._sentencepiece',
sources=['src/sentencepiece/sentencepiece_wrap.cxx'],
extra_compile_args=cflags,
extra_link_args=libs)
cmdclass = {}
else:
SENTENCEPIECE_EXT = Extension(
'sentencepiece._sentencepiece',
sources=['src/sentencepiece/sentencepiece_wrap.cxx'])
cmdclass = {'build_ext': build_ext}
setup(
name='sentencepiece',
author='Taku Kudo',
author_email='taku@google.com',
description='SentencePiece python wrapper',
long_description=long_description(),
long_description_content_type='text/markdown',
version=version(),
package_dir={'': 'src'},
url='https://github.com/google/sentencepiece',
license='Apache',
platforms='Unix',
py_modules=[
'sentencepiece/__init__', 'sentencepiece/sentencepiece_model_pb2',
'sentencepiece/sentencepiece_pb2'
],
ext_modules=[SENTENCEPIECE_EXT],
cmdclass=cmdclass,
classifiers=[
'Development Status :: 5 - Production/Stable', 'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: Apache Software License',
'Operating System :: Unix', 'Programming Language :: Python',
'Topic :: Text Processing :: Linguistic',
'Topic :: Software Development :: Libraries :: Python Modules'
],
test_suite='sentencepiece_test.suite')
|
