diff options
author | Kenneth Heafield <kheafiel@cluster01.lti.ece.cmu.local> | 2011-11-18 20:25:45 +0400 |
---|---|---|
committer | Kenneth Heafield <kheafiel@cluster01.lti.ece.cmu.local> | 2011-11-18 20:27:57 +0400 |
commit | f15eee0abf7e1d2795cbffd26b82992f23c1f36b (patch) | |
tree | 363e56067f157ce0bd1c4037e44412dc985a06c3 | |
parent | a5dea02f1896d9b742fc7f7e574ba4c6bd2394da (diff) |
Jam: Clean up boost library handling, assimilate mert
-rw-r--r-- | Jamroot | 24 | ||||
-rw-r--r-- | boost.jam | 321 | ||||
-rw-r--r-- | lm/Jamfile | 5 | ||||
-rw-r--r-- | mert/Jamfile | 43 | ||||
-rw-r--r-- | moses/src/Jamfile | 6 | ||||
-rw-r--r-- | moses/src/LM/Jamfile | 11 | ||||
-rw-r--r-- | util/Jamfile | 13 |
7 files changed, 397 insertions, 26 deletions
@@ -5,8 +5,14 @@ project : default-build <link>static ; -lib boost_thread : : <link>shared ; +#The linked Boost libraries we use so far. Link shared because most people don't have the static version. +import boost ; +boost.use-project ; +alias boost_thread : /boost//thread : <link>shared ; +alias boost_unit_test_framework : /boost//unit_test_framework : <link>shared ; +alias boost_program_options : /boost//program_options : <link>shared ; +import option ; if [ option.get "notrace" : no : yes ] = yes { trace = ; @@ -17,9 +23,21 @@ if [ option.get "notrace" : no : yes ] = yes project : requirements <threading>multi:<define>WITH_THREADS <threading>multi:<library>boost_thread - <define>_FILE_OFFSET_BITS=64 <define>_LARGE_FILES $(trace) + <define>_FILE_OFFSET_BITS=64 <define>_LARGE_FILES <define>USE_HYPO_POOL + $(trace) ; +# Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php +rule trim-nl ( str ) +{ + return [ MATCH "([^ +]*)" : $(str) ] ; +} +rule _shell ( cmd ) +{ + return [ trim-nl [ SHELL $(cmd) ] ] ; +} + path-constant TOP : . ; build-project lm ; @@ -29,6 +47,7 @@ build-project OnDiskPt/src ; build-project CreateOnDisk/src ; build-project moses-chart-cmd/src ; build-project moses-cmd/src ; +build-project mert ; install dist : lm//query @@ -36,4 +55,5 @@ install dist : moses-chart-cmd/src//moses_chart moses-cmd/src//programs CreateOnDisk/src//CreateOnDisk + mert//programs : <location>dist <install-type>EXE <install-dependencies>on <dll-path>$(TOP)/dist <link>shared:<install-type>LIB ; diff --git a/boost.jam b/boost.jam new file mode 100644 index 000000000..388ac4d15 --- /dev/null +++ b/boost.jam @@ -0,0 +1,321 @@ +# $Id: boost.jam 63913 2010-07-12 07:37:43Z vladimir_prus $ +# Copyright 2008 Roland Schwarz +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +# Boost library support module. +# +# This module allows to use the boost library from boost-build projects. +# The location of a boost source tree or the path to a pre-built +# version of the library can be configured from either site-config.jam +# or user-config.jam. If no location is configured the module looks for +# a BOOST_ROOT environment variable, which should point to a boost source +# tree. As a last resort it tries to use pre-built libraries from the standard +# search path of the compiler. +# +# If the location to a source tree is known, the module can be configured +# from the *-config.jam files: +# +# using boost : 1.35 : <root>/path-to-boost-root ; +# +# If the location to a pre-built version is known: +# +# using boost : 1.34 +# : <include>/usr/local/include/boost_1_34 +# <library>/usr/local/lib +# ; +# +# It is legal to configure more than one boost library version in the config +# files. The version identifier is used to disambiguate between them. +# The first configured version becomes the default. +# +# To use a boost library you need to put a 'use' statement into your +# Jamfile: +# +# import boost ; +# +# boost.use-project 1.35 ; +# +# If you don't care about a specific version you just can omit the version +# part, in which case the default is picked up: +# +# boost.use-project ; +# +# The library can be referenced with the project identifier '/boost'. To +# reference the program_options you would specify: +# +# exe myexe : mysrc.cpp : <library>/boost//program_options ; +# +# Note that the requirements are automatically transformed into suitable +# tags to find the correct pre-built library. +# + +import modules ; +import errors ; +import project ; +import string ; +import toolset ; +import property-set ; +import regex ; +import common ; +import option ; +import numbers ; + +.boost.auto_config = [ property-set.create <layout>system ] ; + +if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] +{ + .debug-configuration = true ; +} + +# Configuration of the boost library to use. +# +# This can either be a boost source tree or +# pre-built libraries. The 'version' parameter must be a valid boost +# version number, e.g. 1.35, if specifying a pre-built version with +# versioned layout. It may be a symbolic name, e.g. 'trunk' if specifying +# a source tree. The options are specified as named parameters (like +# properties). The following paramters are available: +# +# <root>/path-to-boost-root: Specify a source tree. +# +# <include>/path-to-include: The include directory to search. +# +# <library>/path-to-library: The library directory to search. +# +# <layout>system or <layout>versioned. +# +# <build-id>my_build_id: The custom build id to use. +# +rule init +( + version # Version identifier. + : options * # Set the option properties. +) +{ + if $(.boost.$(version)) { + errors.user-error + "Boost " $(version) "already configured." ; + } + else { + if $(.debug-configuration) { + if ! $(.boost_default) { + echo notice: configuring default boost library $(version) ; + } + echo notice: configuring boost library $(version) ; + } + .boost_default ?= $(version) ; # the first configured is default + .boost.$(version) = [ property-set.create $(options) ] ; + } +} + +# Use a certain version of the library. +# +# The use-project rule causes the module to define a boost project of +# searchable pre-built boost libraries, or references a source tree +# of the boost library. If the 'version' parameter is omitted either +# the configured default (first in config files) is used or an auto +# configuration will be attempted. +# +rule use-project +( + version ? # The version of the library to use. +) +{ + project.push-current [ project.current ] ; + version ?= $(.boost_default) ; + version ?= auto_config ; + + if $(.initialized) { + if $(.initialized) != $(version) { + errors.user-error + "Attempt to use" $(__name__) "with different parameters" ; + } + } + else { + if $(.boost.$(version)) { + local opt = $(.boost.$(version)) ; + local root = [ $(opt).get <root> ] ; + local inc = [ $(opt).get <include> ] ; + local lib = [ $(opt).get <library> ] ; + + if $(.debug-configuration) { + echo notice: using boost library $(version) [ $(opt).raw ] ; + } + + .layout = [ $(opt).get <layout> ] ; + .layout ?= versioned ; + .build_id = [ $(opt).get <build-id> ] ; + .version_tag = [ regex.replace $(version) "[*\\/:.\"\' ]" "_" ] ; + .initialized = $(version) ; + + if ( $(root) && $(inc) ) + || ( $(root) && $(lib) ) + || ( $(lib) && ! $(inc) ) + || ( ! $(lib) && $(inc) ) { + errors.user-error + "Ambiguous parameters," + "use either <root> or <inlude> with <library>." ; + } + else if ! $(root) && ! $(inc) { + root = [ modules.peek : BOOST_ROOT ] ; + } + + local prj = [ project.current ] ; + local mod = [ $(prj).project-module ] ; + + if $(root) { + modules.call-in $(mod) : use-project boost : $(root) ; + } + else { + project.initialize $(__name__) ; + # It is possible to overide the setup of the searched + # libraries per version. The (unlikely) tag 0.0.1 is + # meant as an example template only. + switch $(version) { + case 0.0.1 : boost_0_0_1 $(inc) $(lib) ; + case * : boost_std $(inc) $(lib) ; + } + } + } + else { + errors.user-error + "Reference to unconfigured boost version." ; + } + } + project.pop-current ; +} + +rule boost_std ( inc ? lib ? ) +{ +# The default definitions for pre-built libraries. + + project boost + : usage-requirements <include>$(inc) <define>BOOST_ALL_NO_LIB + : requirements <tag>@tag_std <search>$(lib) + ; + + alias headers ; + lib date_time : : : : + <link>shared:<define>BOOST_DATE_TIME_DYN_LINK ; + lib filesystem : : : : + <link>shared:<define>BOOST_FILE_SYSTEM_DYN_LINK ; + lib graph : : : : + <link>shared:<define>BOOST_GRAPH_DYN_LINK ; + lib graph_parallel : : : : + <link>shared:<define>BOOST_GRAPH_DYN_LINK ; + lib iostreams : : : : + <link>shared:<define>BOOST_IOSTREAMS_DYN_LINK ; + lib math_tr1 : : : : + <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ; + lib math_tr1f : : : : + <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ; + lib math_tr1l : : : : + <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ; + lib math_c99 : : : : + <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ; + lib math_c99f : : : : + <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ; + lib math_c99l : : : : + <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ; + lib mpi : : : : + <link>shared:<define>BOOST_MPI_DYN_LINK ; + lib program_options : : : : + <link>shared:<define>BOOST_PROGRAM_OPTIONS_DYN_LINK ; + lib python : : : : + <link>shared:<define>BOOST_PYTHON_DYN_LINK ; + lib random : : : : + <link>shared:<define>BOOST_RANDOM_DYN_LINK ; + lib regex : : : : + <link>shared:<define>BOOST_REGEX_DYN_LINK ; + lib serialization : : : : + <link>shared:<define>BOOST_SERIALIZATION_DYN_LINK ; + lib wserialization : : : : + <link>shared:<define>BOOST_SERIALIZATION_DYN_LINK ; + lib signals : : : : + <link>shared:<define>BOOST_SIGNALS_DYN_LINK ; + lib system : : : : + <link>shared:<define>BOOST_SYSTEM_DYN_LINK ; + lib unit_test_framework : : : : + <link>shared:<define>BOOST_TEST_DYN_LINK ; + lib prg_exec_monitor : : : : + <link>shared:<define>BOOST_TEST_DYN_LINK ; + lib test_exec_monitor : : : : + <link>shared:<define>BOOST_TEST_DYN_LINK ; + lib thread : : : : + <link>shared:<define>BOOST_THREAD_DYN_DLL ; + lib wave : : : : + <link>shared:<define>BOOST_WAVE_DYN_LINK ; +} + +rule boost_0_0_1 ( inc ? lib ? ) +{ + echo "You are trying to use an example placeholder for boost libs." ; + # Copy this template to another place (in the file boost.jam) + # and define a project and libraries modelled after the + # boost_std rule. Please note that it is also possible to have + # a per version taging rule in case they are different between + # versions. +} + +rule tag_std ( name : type ? : property-set ) +{ + name = boost_$(name) ; + if ( [ $(property-set).get <link> ] in static ) && + ( [ $(property-set).get <target-os> ] in windows ) + { + name = lib$(name) ; + } + + local result ; + if $(.layout) = system + { + local version = [ MATCH ^([0-9]+)_([0-9]+) : $(.version_tag) ] ; + if $(version[1]) = "1" && [ numbers.less $(version[2]) 39 ] + { + result = [ tag_tagged $(name) : $(type) : $(property-set) ] ; + } + else + { + result = [ tag_system $(name) : $(type) : $(property-set) ] ; + } + } + else if $(.layout) = tagged + { + result = [ tag_tagged $(name) : $(type) : $(property-set) ] ; + } + else if $(.layout) = versioned + { + result = [ tag_versioned $(name) : $(type) : $(property-set) ] ; + } + else + { + errors.error "Missing layout" ; + } + + return $(result) ; +} + +rule tag_system ( name : type ? : property-set ) +{ + return [ common.format-name + <base> + -$(.build_id) + : $(name) : $(type) : $(property-set) ] ; +} + +rule tag_tagged ( name : type ? : property-set ) +{ + return [ common.format-name + <base> <threading> <runtime> + -$(.build_id) + : $(name) : $(type) : $(property-set) ] ; +} + +rule tag_versioned ( name : type ? : property-set ) +{ + return [ common.format-name + <base> <toolset> <threading> <runtime> -$(.version_tag) + -$(.build_id) + : $(name) : $(type) : $(property-set) ] ; +} diff --git a/lm/Jamfile b/lm/Jamfile index bfec3a4c6..dbf18a556 100644 --- a/lm/Jamfile +++ b/lm/Jamfile @@ -1,10 +1,9 @@ lib lm : bhiksha.cc binary_format.cc config.cc lm_exception.cc model.cc quantize.cc read_arpa.cc search_hashed.cc search_trie.cc trie.cc trie_sort.cc virtual_interface.cc vocab.cc ../util//util : <include>.. : : <include>.. ; import testing ; -lib boost_unit_test_framework : : <link>shared : : <link>shared:<define>BOOST_TEST_DYN_LINK ; -run left_test.cc lm boost_unit_test_framework : : test.arpa ; -run model_test.cc lm boost_unit_test_framework : : test.arpa test_nounk.arpa ; +run left_test.cc lm ..//boost_unit_test_framework : : test.arpa ; +run model_test.cc lm ..//boost_unit_test_framework : : test.arpa test_nounk.arpa ; exe query : ngram_query.cc lm ; exe build_binary : build_binary.cc lm ; diff --git a/mert/Jamfile b/mert/Jamfile new file mode 100644 index 000000000..f2e8b14a1 --- /dev/null +++ b/mert/Jamfile @@ -0,0 +1,43 @@ +lib m ; +lib z ; + +lib mert_lib : +Util.cpp +FileStream.cpp +Timer.cpp +ScoreStats.cpp ScoreArray.cpp ScoreData.cpp +ScoreDataIterator.cpp +FeatureStats.cpp FeatureArray.cpp FeatureData.cpp +FeatureDataIterator.cpp +Data.cpp +BleuScorer.cpp +Point.cpp +PerScorer.cpp +Scorer.cpp +ScorerFactory.cpp +Optimizer.cpp +TERsrc/alignmentStruct.cpp +TERsrc/hashMap.cpp +TERsrc/hashMapStringInfos.cpp +TERsrc/stringHasher.cpp +TERsrc/terAlignment.cpp +TERsrc/terShift.cpp +TERsrc/hashMapInfos.cpp +TERsrc/infosHasher.cpp +TERsrc/stringInfosHasher.cpp +TERsrc/tercalc.cpp +TERsrc/tools.cpp +TerScorer.cpp +CderScorer.cpp +MergeScorer.cpp +../util//util m z ; + +exe mert : mert.cpp mert_lib ../moses/src//ThreadPool ; + +exe extractor : extractor.cpp mert_lib ; + +exe evaluator : evaluator.cpp mert_lib ; + +exe pro : pro.cpp mert_lib ..//boost_program_options ; + +alias programs : mert extractor evaluator pro ; diff --git a/moses/src/Jamfile b/moses/src/Jamfile index a5b0e2210..f2ec7a481 100644 --- a/moses/src/Jamfile +++ b/moses/src/Jamfile @@ -3,6 +3,8 @@ alias headers : ../../util//util : : : <include>. ; lib z ; alias InputFileStream : InputFileStream.cpp z headers ; +alias ThreadPool : ThreadPool.cpp ; + if [ option.get "with-synlm" : no : yes ] = yes { lib m ; @@ -103,7 +105,6 @@ SquareMatrix.cpp StaticData.cpp TargetPhrase.cpp TargetPhraseCollection.cpp -ThreadPool.cpp Timer.cpp TranslationOption.cpp TranslationOptionCollection.cpp @@ -122,5 +123,4 @@ WordsBitmap.cpp WordsRange.cpp XmlOption.cpp -synlm -InputFileStream LM//LM headers ; +synlm ThreadPool InputFileStream LM//LM headers ; diff --git a/moses/src/LM/Jamfile b/moses/src/LM/Jamfile index dd25775a2..0c9fe8c4e 100644 --- a/moses/src/LM/Jamfile +++ b/moses/src/LM/Jamfile @@ -1,16 +1,5 @@ import option ; -# Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php -rule trim-nl ( str ) -{ - return [ MATCH "([^ -]*)" : $(str) ] ; -} -rule _shell ( cmd ) -{ - return [ trim-nl [ SHELL $(cmd) ] ] ; -} - with-irstlm = [ option.get "with-irstlm" ] ; if $(with-irstlm) != "" { diff --git a/util/Jamfile b/util/Jamfile index b69883781..ac5417a79 100644 --- a/util/Jamfile +++ b/util/Jamfile @@ -3,11 +3,10 @@ lib z ; lib util : bit_packing.cc ersatz_progress.cc exception.cc file.cc file_piece.cc mmap.cc murmur_hash.cc z : <include>.. : : <include>.. ; import testing ; -lib boost_unit_test_framework : : <link>shared : : <link>shared:<define>BOOST_TEST_DYN_LINK ; -unit-test bit_packing_test : bit_packing_test.cc util boost_unit_test_framework ; -run file_piece_test.cc util boost_unit_test_framework : : file_piece.cc ; -unit-test joint_sort_test : joint_sort_test.cc util boost_unit_test_framework ; -unit-test probing_hash_table_test : probing_hash_table_test.cc util boost_unit_test_framework ; -unit-test sorted_uniform_test : sorted_uniform_test.cc util boost_unit_test_framework ; -unit-test tokenize_piece_test : tokenize_piece_test.cc util boost_unit_test_framework ; +unit-test bit_packing_test : bit_packing_test.cc util ..//boost_unit_test_framework ; +run file_piece_test.cc util ..//boost_unit_test_framework : : file_piece.cc ; +unit-test joint_sort_test : joint_sort_test.cc util ..//boost_unit_test_framework ; +unit-test probing_hash_table_test : probing_hash_table_test.cc util ..//boost_unit_test_framework ; +unit-test sorted_uniform_test : sorted_uniform_test.cc util ..//boost_unit_test_framework ; +unit-test tokenize_piece_test : tokenize_piece_test.cc util ..//boost_unit_test_framework ; |