Replace scripts build system.

6 files changed, 31 insertions, 338 deletions

diff --git a/scripts/Jamfile b/scripts/Jamfile
index 72bf0402e..c20827bd8 100644
--- a/scripts/Jamfile
+++ b/scripts/Jamfile
@@ -1,14 +1,38 @@
+#MOSES SCRIPTS INSTRUCTIONS
+#
+#--bindir=/path/to/bindir where GIZA++ binaries live
+#
+#--install=/path/to/install to set a path to install (default dist)
+#--git appends the git revision to the install directory
+#
+#--with-boost=/path/to/boost to specify a non-standard Boost installation
+#
+#See also: bjam --help one level up.  
+
 import option ;
 
 build-project training ;
 
 location = [ option.get "install" : $(TOP)/scripts/dist ] ;
+location = $(location)$(GITTAG) ;
 
-constant BINDIR : [ option.get "bindir" ] ;
-
-if ! $(BINDIR) {
-  exit "Pass --bindir=/path/to/bindir to indicate where GIZA++, snt2cooc.out, and mkcls live." : 1 ;
+bindir = [ option.get "bindir" ] ;
+if ! $(bindir) {
+  echo "Pass --bindir=/path/to/bindir" ;
+  echo "It should contain GIZA++, snt2cooc.out, and mkcls." ;
+  echo "These are available from http://www.fjoch.com/GIZA++.html and" ;
+  echo "http://www-i6.informatik.rwth-aachen.de/Colleagues/och/software/mkcls.html ." ;
+  exit : 1 ;
+}
+rule check-for-bin ( name ) {
+  if ! [ FILE_OPEN $(bindir)/$(name) : "r" ] {
+    exit "Did not find $(bindir)/$(name)." : 1 ;
+  }
 }
+check-for-bin GIZA++ ;
+check-for-bin snt2cooc.out ;
+check-for-bin mkcls ;
+constant BINDIR : $(bindir) ;
 
 #These two used to live in a tools directory.  
 install ghkm : training/phrase-extract/extract-ghkm//extract-ghkm : <location>$(location)/training/phrase-extract/extract-ghkm/tools ;
diff --git a/scripts/Makefile b/scripts/Makefile
deleted file mode 100644
index 084172a0b..000000000
--- a/scripts/Makefile
+++ /dev/null
@@ -1,185 +0,0 @@
-# This makefile is here to simplify the automatic releases (and tests!!!)
-# of the scripts
-
-
-TS?=$(shell date '+%Y%m%d-%H%M')
-DS?=$(shell date '+%Y%m%d')
-
-# Set TARGETDIR to directory where you want the compiled scripts to be copied
-# to.
-# Set BINDIR to the directory where GIZA++ and other tools are installed.
-TARGETDIR=/home/bhaddow/moses.github
-BINDIR=/home/bhaddow/moses/bin
-
-MAIN_SCRIPTS_TARGET_DIR=$(TARGETDIR)
-# MAIN_SCRIPTS_TARGET_DIR=$(shell echo `pwd`/temp)
-
-RELEASEDIR=$(TARGETDIR)/scripts-$(TS)
-# RELEASEDIR=$(shell echo `pwd`/temp)
-
-
-
-## Rules to compile parts that need compilation
-
-
-all: compile
-
-SUBDIRS=training/phrase-extract training/symal training/lexical-reordering ems/biconcor
-SUBDIRS_CLEAN=$(SUBDIRS) training/compact-rule-table
-
-compile: compile-compact-rule-table compile-extract-ghkm
-	touch release-exclude # No files excluded by default
-	pwd=`pwd`; \
-	for subdir in $(SUBDIRS); do \
-	  $(MAKE) -C $$subdir || exit 1; \
-	  echo "### Compiler $$subdir"; \
-	  cd $$pwd; \
-	done
-	## All files that need compilation were compiled
-
-compile-compact-rule-table:
-	# Building the compact rule table tools may fail e.g. if boost is not available.
-	# We ignore this because the tools aren't used by default.
-	cd training/compact-rule-table ; \
-	  ./configure && $(MAKE) \
-	  || ( echo "WARNING: Building compact rule table tools failed."; \
-	       echo 'training/compact-rule-table/tools/compactify' >> ../../release-exclude )
-
-compile-extract-ghkm:
-	# Building the GHKM rule extractor may fail e.g. if certain boost libraries are not available.
-	# We ignore this because the tool isn't used by default.
-	cd training/phrase-extract/extract-ghkm ; \
-	  if [ ! -f configure ]; then \
-	    autoreconf -i \
-	      || ( echo "WARNING: Bootstrapping extract-ghkm build environment failed."; \
-	           echo 'training/phrase-extract/extract-ghkm/tools/extract-ghkm' >> ../../release-exclude ); \
-          fi; \
-	  ./configure && $(MAKE) \
-	  || ( echo "WARNING: Building extract-ghkm failed."; \
-	       echo 'training/phrase-extract/extract-ghkm/tools/extract-ghkm' >> ../../release-exclude )
-
-clean:
-	pwd=`pwd`; \
-	for subdir in $(SUBDIRS_CLEAN); do \
-	  $(MAKE) -C $$subdir clean || exit 1; \
-	  echo "### Compiler $$subdir"; \
-	  cd $$pwd; \
-	done
-	## All files that need compilation were compiled
-
-
-### "MAIN" scripts are scripts that have a Philipp-like name, too
-## That is for each script (listed below in MAIN_SCRIPTS),
-## we create a date-stamped version in MAIN_SCRIPTS_TARGET_DIR
-
-MAIN_TRAINING_SCRIPTS_NAMES=filter-model-given-input.pl mert-moses-multi.pl  mert-moses.pl train-model.perl clean-corpus-n.perl
-# Make trick to add directory name to all of them:
-MAIN_TRAINING_SCRIPTS=$(MAIN_TRAINING_SCRIPTS_NAMES:%=training/%)
-
-MAIN_GENERIC_SCRIPTS_NAMES= moses-parallel.pl
-# Make trick to add directory name to all of them:
-MAIN_GENERIC_SCRIPTS=$(MAIN_GENERIC_SCRIPTS_NAMES:%=generic/%)
-
-# the list of all scripts that should be released
-MAIN_SCRIPTS= $(MAIN_TRAINING_SCRIPTS) $(MAIN_GENERIC_SCRIPTS)
-
-
-release:
-	# Compile the parts
-	$(MAKE) all
-	@./check-dependencies.pl "$(HOME)" "$(TARGETDIR)" "$(RELEASEDIR)" "$(BINDIR)"
-	mkdir -p $(RELEASEDIR)
-	cat ./released-files \
-	| grep -v -x -f release-exclude \
-	| rsync -r --files-from - . $(RELEASEDIR)/
-	sed 's#^my \$$BINDIR\s*=.*#my \$$BINDIR="$(BINDIR)";#' training/train-model.perl > $(RELEASEDIR)/training/train-model.perl
-	@echo "####### Do not forget to:" >> $(RELEASEDIR)/README
-	@echo "  export SCRIPTS_ROOTDIR=$(RELEASEDIR)" >> $(RELEASEDIR)/README
-	## Remember, only files listed in released-files are released!!
-	## Don't forget to set your SCRIPTS_ROOTDIR with:
-	@echo "   export SCRIPTS_ROOTDIR=$(RELEASEDIR)"
-
-generate_wrappers:
-	## And for each script, create/rewrite the daily release
-	export TARGET
-	@for s in $(MAIN_SCRIPTS); do \
-	  bn=`basename $$s`; \
-	  echo '#!/bin/bash' > $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS) || exit 1; \
-	  echo "export SCRIPTS_ROOTDIR=$(RELEASEDIR); $(RELEASEDIR)/$$s "'"$$@"; exit $$?' >> $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS) || exit 1; \
-	  chmod 775 $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS); \
-	done
-	
-
-MOSESRELEASE=$(TARGETDIR)/moses.$(DS)
-## This is a handy goal to release moses binary, too
-releasemoses:
-	if [ -z "$(TARGETDIR)" ]; then \
-	  echo "Please specify a TARGETDIR." ; \
-	  echo "  For custom releases issue: "; \
-	  echo "   TARGETDIR=$(HOME)/releases make releasemoses"; \
-	  echo "  For official releases: "; \
-	  echo "   TARGETDIR=/export/ws06osmt make releasemoses"; \
-	  exit 1; \
-	fi
-	if [ -e $(MOSESRELEASE) ]; then echo "Moses release exists! Not touching it! $(MOSESRELEASE)"; exit 1; fi
-	if [ ! -e ../moses-cmd/src/moses ]; then echo "Moses (../moses-cmd/src/moses) does not exist, nothing to release"; ecit 1; fi
-	if file ../moses-cmd/src/moses | grep -q 'dynamicall' ; then echo "Moses (../moses-cmd/src/moses) is dynamically linked, not releasing."; ecit 1; fi
-	cp ../moses-cmd/src/moses $(MOSESRELEASE)
-	## Your current version of moses:
-	@echo "   $(MOSESRELEASE)"
-
-
-## This goal lists all files you might have wanted to release
-# but forgot to mention in released-files
-missed:
-	### These might be intended for release
-	find . -type f \
-	| grep -v '/CVS/' \
-	| grep -v /tests/ \
-	| sed 's/^\.\///' \
-	| grep -F -x -v -f released-files
-
-
-### Tests, applicable only at JHU environment due to data dependencies
-export WORKSPACE=$(shell pwd)/../
-
-.PHONY: tests
-tests:
-	export SCRIPTS_ROOTDIR=`pwd`; \
-	cd tests; \
-	ts=`date '+%Y%m%d-%H%M%N'`; \
-	for test in *.test; do  \
-	  mkdir $$test.$$ts; \
-	  cd $$test.$$ts; \
-	  echo "Running $$test  in  tests/$$test.$$ts"; \
-	  ../$$test > log 2>&1 || exit 1; \
-	  cd ..; \
-	done
-	## All tests passed
-
-## Run just one test in the background
-tests/%.test.run: tests/%.test
-	export SCRIPTS_ROOTDIR=`pwd`; \
-	ts=`date '+%Y%m%d-%H%M%N'`; \
-	cd tests; \
-	  test=$*.test; \
-	  mkdir $$test.$$ts; \
-	  cd $$test.$$ts; \
-	  echo "Running $$test  in  tests/$$test.$$ts"; \
-	  ( nohup ../$$test > log 2>&1 & ) || exit 1; \
-	  echo "Observe tests/$$test.$$ts/log"; \
-	cd ..
-
-## Run just one test in the foreground
-tests/%.test.runfg: tests/%.test
-	export SCRIPTS_ROOTDIR=`pwd`; \
-	ts=`date '+%Y%m%d-%H%M%N'`; \
-	cd tests; \
-	  test=$*.test; \
-	  mkdir $$test.$$ts; \
-	  cd $$test.$$ts; \
-	  echo "Running $$test  in  tests/$$test.$$ts"; \
-	  ../$$test 2>&1 | tee log ; \
-	  echo "Log saved to tests/$$test.$$ts/log"; \
-	cd ..
-	
diff --git a/scripts/README b/scripts/README
index 609352bcc..35dac9dd0 100644
--- a/scripts/README
+++ b/scripts/README
@@ -8,7 +8,7 @@ This directory should contain all multi-purpose scripts for:
 - lib         ... perl modules used by various scripts
 
 
-The Makefile then takes care of proper 'release' from your CVS directory to
+The Jamfile then takes care of proper 'release' from your git directory to
 the shared directories.
 
 The released scripts should remain in the *same directory structure*.
diff --git a/scripts/check-dependencies.pl b/scripts/check-dependencies.pl
deleted file mode 100755
index e93520c8d..000000000
--- a/scripts/check-dependencies.pl
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/perl
-
-# $Id$
-
-my ($home, $target_dir, $release_dir, $bin_dir) = @ARGV;
-
-#print "HOME: $home\nTARGET_DIR: $target_dir\nRELEASE_DIR: $release_dir\n";
-
-if ($target_dir eq '' || -z $target_dir) {
-  print <<EOT;
-Please specify a TARGETDIR.
-
-  For development releases you probably want the following:
-  TARGETDIR=$home/releases make release
-
-  For shared environments, you will want to set TARGETDIR to
-  some appropriately common directory.
-
-EOT
-  exit 1;
-}
-
-if (-e $release_dir) {
-  print "Targetdir exists! Not touching it! $release_dir";
-  exit 1;
-}
-
-unless (-x "$bin_dir/GIZA++" && -x "$bin_dir/snt2cooc.out" && -x "$bin_dir/mkcls" ) {
-  print <<EOT;
-Please specify a BINDIR.
-
-  The BINDIR directory must contain GIZA++, snt2cooc.out and mkcls executables.
-  These are available from http://www.fjoch.com/GIZA++.html and
-  http://www-i6.informatik.rwth-aachen.de/Colleagues/och/software/mkcls.html .
-EOT
-  exit 1;
-}
-
-
-exit 0;
-
diff --git a/scripts/released-files b/scripts/released-files
deleted file mode 100644
index 2e50c4a38..000000000
--- a/scripts/released-files
+++ /dev/null
@@ -1,107 +0,0 @@
-analysis/README
-analysis/sentence-by-sentence.pl
-ems/experiment.machines
-ems/experiment.meta
-ems/experiment.perl
-ems/example/config.basic
-ems/example/config.factored
-ems/example/config.hierarchical
-ems/example/config.syntax
-ems/example/config.toy
-ems/example/data/nc-5k.en
-ems/example/data/nc-5k.fr
-ems/example/data/test-ref.en.sgm
-ems/example/data/test-src.fr.sgm
-ems/support/analysis.perl
-ems/support/berkeley-process.sh
-ems/support/berkeley-train.sh
-ems/support/consolidate-training-data.perl
-ems/support/generic-multicore-parallelizer.perl
-ems/support/generic-parallelizer.perl
-ems/support/input-from-sgm.perl
-ems/support/interpolate-lm.perl
-ems/support/reference-from-sgm.perl
-ems/support/remove-segmenation-markup.perl
-ems/support/report-experiment-scores.perl
-ems/support/reuse-weights.perl
-ems/support/run-command-on-multiple-refsets.perl
-ems/support/wrap-xml.perl
-ems/web/analysis.php
-ems/web/analysis_diff.php
-ems/web/comment.php
-ems/web/diff.php
-ems/web/index.php
-ems/web/lib.php
-ems/web/overview.php
-ems/web/setup
-ems/web/javascripts/builder.js
-ems/web/javascripts/controls.js
-ems/web/javascripts/dragdrop.js
-ems/web/javascripts/effects.js
-ems/web/javascripts/prototype.js
-ems/web/javascripts/scriptaculous.js
-ems/web/javascripts/slider.js
-ems/web/javascripts/sound.js
-ems/web/javascripts/unittest.js
-generic/compound-splitter.perl
-generic/extract-factors.pl
-generic/lopar2pos.pl
-generic/moses-parallel.pl
-generic/mteval-v12.pl
-generic/multi-bleu.perl
-generic/qsub-wrapper.pl
-Makefile
-README
-released-files
-tokenizer/detokenizer.perl
-tokenizer/tokenizer.perl
-tokenizer/lowercase.perl
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ru
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ro
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.de
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.fr
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.el
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.is
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pt
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.nl
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.it
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sl
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sk
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pl
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sv
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.es
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.en
-tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ca
-training/absolutize_moses_model.pl
-training/build-generation-table.perl
-training/clean-corpus-n.perl
-training/clone_moses_model.pl
-training/compact-rule-table/tools/compactify
-training/filter-model-given-input.pl
-training/filter-rule-table.py
-training/lexical-reordering/score
-training/zmert-moses.pl
-training/mert-moses.pl
-training/mert-moses-multi.pl
-training/phrase-extract/extract
-training/phrase-extract/extract-ghkm/tools/extract-ghkm
-training/phrase-extract/extract-rules
-training/phrase-extract/score
-training/phrase-extract/consolidate
-training/postprocess-lopar.perl
-training/reduce_combine.pl
-training/combine_factors.pl
-training/train-model.perl
-training/symal/symal
-training/symal/giza2bal.pl
-training/wrappers/parse-de-bitpar.perl
-training/wrappers/parse-en-collins.perl
-training/wrappers/make-factor-en-pos.mxpost.perl
-training/wrappers/make-factor-pos.tree-tagger.perl
-training/wrappers/make-factor-stem.perl
-recaser/train-recaser.perl
-recaser/recase.perl
-recaser/truecase.perl
-recaser/detruecase.perl
-recaser/train-truecaser.perl
-
diff --git a/scripts/training/Jamfile b/scripts/training/Jamfile
index 0444af8d1..2de16d05c 100644
--- a/scripts/training/Jamfile
+++ b/scripts/training/Jamfile
@@ -7,3 +7,5 @@ make train-model.perl : train-model.perl.missing_bin_dir : @missing_bin_dir ;
 actions missing_bin_dir {
   sed 's#^my \$BINDIR\s*=.*#my\ \$BINDIR=\"$(BINDIR)\";#' $(>) >$(<)
 }
+
+install legacy : train-model.perl : <location>. ;