diff options
author | Barry Haddow <barry.haddow@gmail.com> | 2011-11-23 18:48:50 +0400 |
---|---|---|
committer | Barry Haddow <barry.haddow@gmail.com> | 2011-11-23 18:48:50 +0400 |
commit | 5e2731619c2c3ffd9f1b518f268c711087b8ec32 (patch) | |
tree | e89520dded0044835dd7a6439444525c606c7b60 /contrib | |
parent | 6de8928d656ad5c00736d4a4d971a3a9e3ef7ebc (diff) |
Move projects from top level to contrib
server build probably broken
Diffstat (limited to 'contrib')
109 files changed, 36272 insertions, 0 deletions
diff --git a/contrib/lmserver/AUTHORS b/contrib/lmserver/AUTHORS new file mode 100644 index 000000000..4d9dfd0cd --- /dev/null +++ b/contrib/lmserver/AUTHORS @@ -0,0 +1 @@ +Chris Dyer <redpony AT UMD dot EDU> diff --git a/contrib/lmserver/BUILD b/contrib/lmserver/BUILD new file mode 100755 index 000000000..2f4d9ace4 --- /dev/null +++ b/contrib/lmserver/BUILD @@ -0,0 +1,6 @@ +g++ srilm.cc -c -I/fs/clip-software/srilm-1.5.6-PIC/include -O2 + +make + +g++ -g -O2 -L/fs/clip-software/libevent-1.4.8-stable/lib -o memcached memcached-memcached.o memcached-slabs.o memcached-items.o memcached-assoc.o memcached-thread.o memcached-stats.o srilm.o -levent -L/fs/clip-software/srilm-1.5.6-PIC/lib/i686 -loolm -ldstruct -lmisc + diff --git a/contrib/lmserver/COPYING b/contrib/lmserver/COPYING new file mode 100644 index 000000000..cdfc50626 --- /dev/null +++ b/contrib/lmserver/COPYING @@ -0,0 +1,33 @@ +Copyright (c) 2009, Chris Dyer + +Portions of this software are +Copyright (c) 2003, Danga Interactive, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + + * Neither the name of the Danga Interactive nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/lmserver/ChangeLog b/contrib/lmserver/ChangeLog new file mode 100644 index 000000000..4083f59b8 --- /dev/null +++ b/contrib/lmserver/ChangeLog @@ -0,0 +1,4 @@ +2009-01-21 [Version 1.0 checked in] + + * Branch from memcached-1.2.6-rc1 + diff --git a/contrib/lmserver/INSTALL b/contrib/lmserver/INSTALL new file mode 120000 index 000000000..81fa6ffa4 --- /dev/null +++ b/contrib/lmserver/INSTALL @@ -0,0 +1 @@ +/usr/share/automake-1.9/INSTALL
\ No newline at end of file diff --git a/contrib/lmserver/Makefile.am b/contrib/lmserver/Makefile.am new file mode 100644 index 000000000..ec06d0da2 --- /dev/null +++ b/contrib/lmserver/Makefile.am @@ -0,0 +1,18 @@ +bin_PROGRAMS = lmserver lmserver-debug + +lmserver_SOURCES = lmserver.c lmserver.h thread.c srilm.cc +lmserver_debug_SOURCES = $(lmserver_SOURCES) +lmserver_CPPFLAGS = -DNDEBUG +lmserver_LDADD = @DAEMON_OBJ@ +lmserver_debug_LDADD = @DAEMON_OBJ@ +lmserver_DEPENDENCIES = @DAEMON_OBJ@ +lmserver_debug_DEPENDENCIES = @DAEMON_OBJ@ + +DIST_DIRS = examples +EXTRA_DIST = examples daemon.c + +dist-hook: + rm -rf $(distdir)/doc/.svn/ + rm -rf $(distdir)/scripts/.svn/ + rm -rf $(distdir)/t/.svn/ + rm -rf $(distdir)/t/lib/.svn/ diff --git a/contrib/lmserver/Makefile.in b/contrib/lmserver/Makefile.in new file mode 100644 index 000000000..3d62eb5fd --- /dev/null +++ b/contrib/lmserver/Makefile.in @@ -0,0 +1,645 @@ +# Makefile.in generated by automake 1.9.2 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004 Free Software Foundation, Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +SOURCES = $(lmserver_SOURCES) $(lmserver_debug_SOURCES) + +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +top_builddir = . +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +INSTALL = @INSTALL@ +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +bin_PROGRAMS = lmserver$(EXEEXT) lmserver-debug$(EXEEXT) +DIST_COMMON = README $(am__configure_deps) $(srcdir)/Makefile.am \ + $(srcdir)/Makefile.in $(srcdir)/config.h.in \ + $(top_srcdir)/configure AUTHORS COPYING ChangeLog INSTALL NEWS \ + compile config.guess config.sub depcomp install-sh missing +subdir = . +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \ + configure.lineno configure.status.lineno +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = config.h +CONFIG_CLEAN_FILES = +am__installdirs = "$(DESTDIR)$(bindir)" +binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) +PROGRAMS = $(bin_PROGRAMS) +am_lmserver_OBJECTS = lmserver-lmserver.$(OBJEXT) \ + lmserver-thread.$(OBJEXT) lmserver-srilm.$(OBJEXT) +lmserver_OBJECTS = $(am_lmserver_OBJECTS) +am__objects_1 = lmserver.$(OBJEXT) thread.$(OBJEXT) srilm.$(OBJEXT) +am_lmserver_debug_OBJECTS = $(am__objects_1) +lmserver_debug_OBJECTS = $(am_lmserver_debug_OBJECTS) +DEFAULT_INCLUDES = -I. -I$(srcdir) -I. +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +CCLD = $(CC) +LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +CXXLD = $(CXX) +CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +SOURCES = $(lmserver_SOURCES) $(lmserver_debug_SOURCES) +DIST_SOURCES = $(lmserver_SOURCES) $(lmserver_debug_SOURCES) +ETAGS = etags +CTAGS = ctags +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +distdir = $(PACKAGE)-$(VERSION) +top_distdir = $(distdir) +am__remove_distdir = \ + { test ! -d $(distdir) \ + || { find $(distdir) -type d ! -perm -200 -exec chmod u+w {} ';' \ + && rm -fr $(distdir); }; } +DIST_ARCHIVES = $(distdir).tar.gz +GZIP_ENV = --best +distuninstallcheck_listfiles = find . -type f -print +distcleancheck_listfiles = find . -type f -print +ACLOCAL = @ACLOCAL@ +AMDEP_FALSE = @AMDEP_FALSE@ +AMDEP_TRUE = @AMDEP_TRUE@ +AMTAR = @AMTAR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DAEMON_OBJ = @DAEMON_OBJ@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +OBJEXT = @OBJEXT@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SRI_LM_FALSE = @SRI_LM_FALSE@ +SRI_LM_TRUE = @SRI_LM_TRUE@ +STRIP = @STRIP@ +VERSION = @VERSION@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_STRIP = @ac_ct_STRIP@ +am__fastdepCC_FALSE = @am__fastdepCC_FALSE@ +am__fastdepCC_TRUE = @am__fastdepCC_TRUE@ +am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@ +am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +datadir = @datadir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +lmserver_SOURCES = lmserver.c lmserver.h thread.c srilm.cc +lmserver_debug_SOURCES = $(lmserver_SOURCES) +lmserver_CPPFLAGS = -DNDEBUG +lmserver_LDADD = @DAEMON_OBJ@ +lmserver_debug_LDADD = @DAEMON_OBJ@ +lmserver_DEPENDENCIES = @DAEMON_OBJ@ +lmserver_debug_DEPENDENCIES = @DAEMON_OBJ@ +DIST_DIRS = examples +EXTRA_DIST = examples daemon.c +all: config.h + $(MAKE) $(AM_MAKEFLAGS) all-am + +.SUFFIXES: +.SUFFIXES: .c .cc .o .obj +am--refresh: + @: +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + echo ' cd $(srcdir) && $(AUTOMAKE) --gnu '; \ + cd $(srcdir) && $(AUTOMAKE) --gnu \ + && exit 0; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu Makefile'; \ + cd $(top_srcdir) && \ + $(AUTOMAKE) --gnu Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + echo ' $(SHELL) ./config.status'; \ + $(SHELL) ./config.status;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + $(SHELL) ./config.status --recheck + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(srcdir) && $(AUTOCONF) +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(srcdir) && $(ACLOCAL) $(ACLOCAL_AMFLAGS) + +config.h: stamp-h1 + @if test ! -f $@; then \ + rm -f stamp-h1; \ + $(MAKE) stamp-h1; \ + else :; fi + +stamp-h1: $(srcdir)/config.h.in $(top_builddir)/config.status + @rm -f stamp-h1 + cd $(top_builddir) && $(SHELL) ./config.status config.h +$(srcdir)/config.h.in: $(am__configure_deps) + cd $(top_srcdir) && $(AUTOHEADER) + rm -f stamp-h1 + touch $@ + +distclean-hdr: + -rm -f config.h stamp-h1 +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(mkdir_p) "$(DESTDIR)$(bindir)" + @list='$(bin_PROGRAMS)'; for p in $$list; do \ + p1=`echo $$p|sed 's/$(EXEEXT)$$//'`; \ + if test -f $$p \ + ; then \ + f=`echo "$$p1" | sed 's,^.*/,,;$(transform);s/$$/$(EXEEXT)/'`; \ + echo " $(INSTALL_PROGRAM_ENV) $(binPROGRAMS_INSTALL) '$$p' '$(DESTDIR)$(bindir)/$$f'"; \ + $(INSTALL_PROGRAM_ENV) $(binPROGRAMS_INSTALL) "$$p" "$(DESTDIR)$(bindir)/$$f" || exit 1; \ + else :; fi; \ + done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; for p in $$list; do \ + f=`echo "$$p" | sed 's,^.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/'`; \ + echo " rm -f '$(DESTDIR)$(bindir)/$$f'"; \ + rm -f "$(DESTDIR)$(bindir)/$$f"; \ + done + +clean-binPROGRAMS: + -test -z "$(bin_PROGRAMS)" || rm -f $(bin_PROGRAMS) +lmserver$(EXEEXT): $(lmserver_OBJECTS) $(lmserver_DEPENDENCIES) + @rm -f lmserver$(EXEEXT) + $(CXXLINK) $(lmserver_LDFLAGS) $(lmserver_OBJECTS) $(lmserver_LDADD) $(LIBS) +lmserver-debug$(EXEEXT): $(lmserver_debug_OBJECTS) $(lmserver_debug_DEPENDENCIES) + @rm -f lmserver-debug$(EXEEXT) + $(CXXLINK) $(lmserver_debug_LDFLAGS) $(lmserver_debug_OBJECTS) $(lmserver_debug_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lmserver-lmserver.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lmserver-srilm.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lmserver-thread.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lmserver.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/srilm.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/thread.Po@am__quote@ + +.c.o: +@am__fastdepCC_TRUE@ if $(COMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c $< + +.c.obj: +@am__fastdepCC_TRUE@ if $(COMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ `$(CYGPATH_W) '$<'`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'` + +lmserver-lmserver.o: lmserver.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT lmserver-lmserver.o -MD -MP -MF "$(DEPDIR)/lmserver-lmserver.Tpo" -c -o lmserver-lmserver.o `test -f 'lmserver.c' || echo '$(srcdir)/'`lmserver.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/lmserver-lmserver.Tpo" "$(DEPDIR)/lmserver-lmserver.Po"; else rm -f "$(DEPDIR)/lmserver-lmserver.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='lmserver.c' object='lmserver-lmserver.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o lmserver-lmserver.o `test -f 'lmserver.c' || echo '$(srcdir)/'`lmserver.c + +lmserver-lmserver.obj: lmserver.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT lmserver-lmserver.obj -MD -MP -MF "$(DEPDIR)/lmserver-lmserver.Tpo" -c -o lmserver-lmserver.obj `if test -f 'lmserver.c'; then $(CYGPATH_W) 'lmserver.c'; else $(CYGPATH_W) '$(srcdir)/lmserver.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/lmserver-lmserver.Tpo" "$(DEPDIR)/lmserver-lmserver.Po"; else rm -f "$(DEPDIR)/lmserver-lmserver.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='lmserver.c' object='lmserver-lmserver.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o lmserver-lmserver.obj `if test -f 'lmserver.c'; then $(CYGPATH_W) 'lmserver.c'; else $(CYGPATH_W) '$(srcdir)/lmserver.c'; fi` + +lmserver-thread.o: thread.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT lmserver-thread.o -MD -MP -MF "$(DEPDIR)/lmserver-thread.Tpo" -c -o lmserver-thread.o `test -f 'thread.c' || echo '$(srcdir)/'`thread.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/lmserver-thread.Tpo" "$(DEPDIR)/lmserver-thread.Po"; else rm -f "$(DEPDIR)/lmserver-thread.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='thread.c' object='lmserver-thread.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o lmserver-thread.o `test -f 'thread.c' || echo '$(srcdir)/'`thread.c + +lmserver-thread.obj: thread.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT lmserver-thread.obj -MD -MP -MF "$(DEPDIR)/lmserver-thread.Tpo" -c -o lmserver-thread.obj `if test -f 'thread.c'; then $(CYGPATH_W) 'thread.c'; else $(CYGPATH_W) '$(srcdir)/thread.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/lmserver-thread.Tpo" "$(DEPDIR)/lmserver-thread.Po"; else rm -f "$(DEPDIR)/lmserver-thread.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='thread.c' object='lmserver-thread.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o lmserver-thread.obj `if test -f 'thread.c'; then $(CYGPATH_W) 'thread.c'; else $(CYGPATH_W) '$(srcdir)/thread.c'; fi` + +.cc.o: +@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ `$(CYGPATH_W) '$<'`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +lmserver-srilm.o: srilm.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT lmserver-srilm.o -MD -MP -MF "$(DEPDIR)/lmserver-srilm.Tpo" -c -o lmserver-srilm.o `test -f 'srilm.cc' || echo '$(srcdir)/'`srilm.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/lmserver-srilm.Tpo" "$(DEPDIR)/lmserver-srilm.Po"; else rm -f "$(DEPDIR)/lmserver-srilm.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='srilm.cc' object='lmserver-srilm.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o lmserver-srilm.o `test -f 'srilm.cc' || echo '$(srcdir)/'`srilm.cc + +lmserver-srilm.obj: srilm.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT lmserver-srilm.obj -MD -MP -MF "$(DEPDIR)/lmserver-srilm.Tpo" -c -o lmserver-srilm.obj `if test -f 'srilm.cc'; then $(CYGPATH_W) 'srilm.cc'; else $(CYGPATH_W) '$(srcdir)/srilm.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/lmserver-srilm.Tpo" "$(DEPDIR)/lmserver-srilm.Po"; else rm -f "$(DEPDIR)/lmserver-srilm.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='srilm.cc' object='lmserver-srilm.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(lmserver_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o lmserver-srilm.obj `if test -f 'srilm.cc'; then $(CYGPATH_W) 'srilm.cc'; else $(CYGPATH_W) '$(srcdir)/srilm.cc'; fi` +uninstall-info-am: + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + tags=; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + if test -z "$(ETAGS_ARGS)$$tags$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$tags $$unique; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + tags=; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + test -z "$(CTAGS_ARGS)$$tags$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$tags $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && cd $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) $$here + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + $(am__remove_distdir) + mkdir $(distdir) + @srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \ + list='$(DISTFILES)'; for file in $$list; do \ + case $$file in \ + $(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \ + $(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \ + esac; \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test "$$dir" != "$$file" && test "$$dir" != "."; then \ + dir="/$$dir"; \ + $(mkdir_p) "$(distdir)$$dir"; \ + else \ + dir=''; \ + fi; \ + if test -d $$d/$$file; then \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \ + fi; \ + cp -pR $$d/$$file $(distdir)$$dir || exit 1; \ + else \ + test -f $(distdir)/$$file \ + || cp -p $$d/$$file $(distdir)/$$file \ + || exit 1; \ + fi; \ + done + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$(top_distdir)" distdir="$(distdir)" \ + dist-hook + -find $(distdir) -type d ! -perm -777 -exec chmod a+rwx {} \; -o \ + ! -type d ! -perm -444 -links 1 -exec chmod a+r {} \; -o \ + ! -type d ! -perm -400 -exec chmod a+r {} \; -o \ + ! -type d ! -perm -444 -exec $(SHELL) $(install_sh) -c -m a+r {} {} \; \ + || chmod -R a+r $(distdir) +dist-gzip: distdir + tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz + $(am__remove_distdir) + +dist-bzip2: distdir + tardir=$(distdir) && $(am__tar) | bzip2 -9 -c >$(distdir).tar.bz2 + $(am__remove_distdir) + +dist-tarZ: distdir + tardir=$(distdir) && $(am__tar) | compress -c >$(distdir).tar.Z + $(am__remove_distdir) + +dist-shar: distdir + shar $(distdir) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).shar.gz + $(am__remove_distdir) + +dist-zip: distdir + -rm -f $(distdir).zip + zip -rq $(distdir).zip $(distdir) + $(am__remove_distdir) + +dist dist-all: distdir + tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz + $(am__remove_distdir) + +# This target untars the dist file and tries a VPATH configuration. Then +# it guarantees that the distribution is self-contained by making another +# tarfile. +distcheck: dist + case '$(DIST_ARCHIVES)' in \ + *.tar.gz*) \ + GZIP=$(GZIP_ENV) gunzip -c $(distdir).tar.gz | $(am__untar) ;;\ + *.tar.bz2*) \ + bunzip2 -c $(distdir).tar.bz2 | $(am__untar) ;;\ + *.tar.Z*) \ + uncompress -c $(distdir).tar.Z | $(am__untar) ;;\ + *.shar.gz*) \ + GZIP=$(GZIP_ENV) gunzip -c $(distdir).shar.gz | unshar ;;\ + *.zip*) \ + unzip $(distdir).zip ;;\ + esac + chmod -R a-w $(distdir); chmod a+w $(distdir) + mkdir $(distdir)/_build + mkdir $(distdir)/_inst + chmod a-w $(distdir) + dc_install_base=`$(am__cd) $(distdir)/_inst && pwd | sed -e 's,^[^:\\/]:[\\/],/,'` \ + && dc_destdir="$${TMPDIR-/tmp}/am-dc-$$$$/" \ + && cd $(distdir)/_build \ + && ../configure --srcdir=.. --prefix="$$dc_install_base" \ + $(DISTCHECK_CONFIGURE_FLAGS) \ + && $(MAKE) $(AM_MAKEFLAGS) \ + && $(MAKE) $(AM_MAKEFLAGS) dvi \ + && $(MAKE) $(AM_MAKEFLAGS) check \ + && $(MAKE) $(AM_MAKEFLAGS) install \ + && $(MAKE) $(AM_MAKEFLAGS) installcheck \ + && $(MAKE) $(AM_MAKEFLAGS) uninstall \ + && $(MAKE) $(AM_MAKEFLAGS) distuninstallcheck_dir="$$dc_install_base" \ + distuninstallcheck \ + && chmod -R a-w "$$dc_install_base" \ + && ({ \ + (cd ../.. && umask 077 && mkdir "$$dc_destdir") \ + && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" install \ + && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" uninstall \ + && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" \ + distuninstallcheck_dir="$$dc_destdir" distuninstallcheck; \ + } || { rm -rf "$$dc_destdir"; exit 1; }) \ + && rm -rf "$$dc_destdir" \ + && $(MAKE) $(AM_MAKEFLAGS) dist \ + && rm -rf $(DIST_ARCHIVES) \ + && $(MAKE) $(AM_MAKEFLAGS) distcleancheck + $(am__remove_distdir) + @(echo "$(distdir) archives ready for distribution: "; \ + list='$(DIST_ARCHIVES)'; for i in $$list; do echo $$i; done) | \ + sed -e '1{h;s/./=/g;p;x;}' -e '$${p;x;}' +distuninstallcheck: + @cd $(distuninstallcheck_dir) \ + && test `$(distuninstallcheck_listfiles) | wc -l` -le 1 \ + || { echo "ERROR: files left after uninstall:" ; \ + if test -n "$(DESTDIR)"; then \ + echo " (check DESTDIR support)"; \ + fi ; \ + $(distuninstallcheck_listfiles) ; \ + exit 1; } >&2 +distcleancheck: distclean + @if test '$(srcdir)' = . ; then \ + echo "ERROR: distcleancheck can only run from a VPATH build" ; \ + exit 1 ; \ + fi + @test `$(distcleancheck_listfiles) | wc -l` -eq 0 \ + || { echo "ERROR: files left in build directory after distclean:" ; \ + $(distcleancheck_listfiles) ; \ + exit 1; } >&2 +check-am: all-am +check: check-am +all-am: Makefile $(PROGRAMS) config.h +installdirs: + for dir in "$(DESTDIR)$(bindir)"; do \ + test -z "$$dir" || $(mkdir_p) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-binPROGRAMS clean-generic mostlyclean-am + +distclean: distclean-am + -rm -f $(am__CONFIG_DISTCLEAN_FILES) + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-hdr distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +info: info-am + +info-am: + +install-data-am: + +install-exec-am: install-binPROGRAMS + +install-info: install-info-am + +install-man: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f $(am__CONFIG_DISTCLEAN_FILES) + -rm -rf $(top_srcdir)/autom4te.cache + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-binPROGRAMS uninstall-info-am + +.PHONY: CTAGS GTAGS all all-am am--refresh check check-am clean \ + clean-binPROGRAMS clean-generic ctags dist dist-all dist-bzip2 \ + dist-gzip dist-hook dist-shar dist-tarZ dist-zip distcheck \ + distclean distclean-compile distclean-generic distclean-hdr \ + distclean-tags distcleancheck distdir distuninstallcheck dvi \ + dvi-am html html-am info info-am install install-am \ + install-binPROGRAMS install-data install-data-am install-exec \ + install-exec-am install-info install-info-am install-man \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic pdf pdf-am ps ps-am \ + tags uninstall uninstall-am uninstall-binPROGRAMS \ + uninstall-info-am + + +dist-hook: + rm -rf $(distdir)/doc/.svn/ + rm -rf $(distdir)/scripts/.svn/ + rm -rf $(distdir)/t/.svn/ + rm -rf $(distdir)/t/lib/.svn/ +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/contrib/lmserver/NEWS b/contrib/lmserver/NEWS new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/contrib/lmserver/NEWS diff --git a/contrib/lmserver/README b/contrib/lmserver/README new file mode 100644 index 000000000..dd9dde134 --- /dev/null +++ b/contrib/lmserver/README @@ -0,0 +1,31 @@ +This software is based on pieces of the memcached server. + +To start an LM server: + + ./lmserver -x /tmp/moses-reg-test-data-2/lm/europarl.en.srilm.gz -o 3 + +-o specifies the order, -x specifies the file. + + +The following was taken from the memcached README: + +Dependencies: + + -- libevent, http://www.monkey.org/~provos/libevent/ (libevent-dev) + +If using Linux, you need a kernel with epoll. Sure, libevent will +work with normal select, but it sucks. + +epoll isn't in Linux 2.4 yet, but there's a backport at: + + http://www.xmailserver.org/linux-patches/nio-improve.html + +You want the epoll-lt patch (level-triggered). + +If you're using MacOS, you'll want libevent 1.1 or higher to deal with +a kqueue bug. + +The memcached website is at: + + http://www.danga.com/memcached/ + diff --git a/contrib/lmserver/aclocal.m4 b/contrib/lmserver/aclocal.m4 new file mode 100644 index 000000000..01ee4a076 --- /dev/null +++ b/contrib/lmserver/aclocal.m4 @@ -0,0 +1,1084 @@ +# generated automatically by aclocal 1.9.2 -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004 +# Free Software Foundation, Inc. +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +# -*- Autoconf -*- +# Copyright (C) 2002, 2003 Free Software Foundation, Inc. +# Generated from amversion.in; do not edit by hand. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + +# AM_AUTOMAKE_VERSION(VERSION) +# ---------------------------- +# Automake X.Y traces this macro to ensure aclocal.m4 has been +# generated from the m4 files accompanying Automake X.Y. +AC_DEFUN([AM_AUTOMAKE_VERSION], [am__api_version="1.9"]) + +# AM_SET_CURRENT_AUTOMAKE_VERSION +# ------------------------------- +# Call AM_AUTOMAKE_VERSION so it can be traced. +# This function is AC_REQUIREd by AC_INIT_AUTOMAKE. +AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION], + [AM_AUTOMAKE_VERSION([1.9.2])]) + +# AM_AUX_DIR_EXPAND + +# Copyright (C) 2001, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# For projects using AC_CONFIG_AUX_DIR([foo]), Autoconf sets +# $ac_aux_dir to `$srcdir/foo'. In other projects, it is set to +# `$srcdir', `$srcdir/..', or `$srcdir/../..'. +# +# Of course, Automake must honor this variable whenever it calls a +# tool from the auxiliary directory. The problem is that $srcdir (and +# therefore $ac_aux_dir as well) can be either absolute or relative, +# depending on how configure is run. This is pretty annoying, since +# it makes $ac_aux_dir quite unusable in subdirectories: in the top +# source directory, any form will work fine, but in subdirectories a +# relative path needs to be adjusted first. +# +# $ac_aux_dir/missing +# fails when called from a subdirectory if $ac_aux_dir is relative +# $top_srcdir/$ac_aux_dir/missing +# fails if $ac_aux_dir is absolute, +# fails when called from a subdirectory in a VPATH build with +# a relative $ac_aux_dir +# +# The reason of the latter failure is that $top_srcdir and $ac_aux_dir +# are both prefixed by $srcdir. In an in-source build this is usually +# harmless because $srcdir is `.', but things will broke when you +# start a VPATH build or use an absolute $srcdir. +# +# So we could use something similar to $top_srcdir/$ac_aux_dir/missing, +# iff we strip the leading $srcdir from $ac_aux_dir. That would be: +# am_aux_dir='\$(top_srcdir)/'`expr "$ac_aux_dir" : "$srcdir//*\(.*\)"` +# and then we would define $MISSING as +# MISSING="\${SHELL} $am_aux_dir/missing" +# This will work as long as MISSING is not called from configure, because +# unfortunately $(top_srcdir) has no meaning in configure. +# However there are other variables, like CC, which are often used in +# configure, and could therefore not use this "fixed" $ac_aux_dir. +# +# Another solution, used here, is to always expand $ac_aux_dir to an +# absolute PATH. The drawback is that using absolute paths prevent a +# configured tree to be moved without reconfiguration. + +AC_DEFUN([AM_AUX_DIR_EXPAND], +[dnl Rely on autoconf to set up CDPATH properly. +AC_PREREQ([2.50])dnl +# expand $ac_aux_dir to an absolute path +am_aux_dir=`cd $ac_aux_dir && pwd` +]) + +# AM_CONDITIONAL -*- Autoconf -*- + +# Copyright (C) 1997, 2000, 2001, 2003, 2004 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 6 + +# AM_CONDITIONAL(NAME, SHELL-CONDITION) +# ------------------------------------- +# Define a conditional. +AC_DEFUN([AM_CONDITIONAL], +[AC_PREREQ(2.52)dnl + ifelse([$1], [TRUE], [AC_FATAL([$0: invalid condition: $1])], + [$1], [FALSE], [AC_FATAL([$0: invalid condition: $1])])dnl +AC_SUBST([$1_TRUE]) +AC_SUBST([$1_FALSE]) +if $2; then + $1_TRUE= + $1_FALSE='#' +else + $1_TRUE='#' + $1_FALSE= +fi +AC_CONFIG_COMMANDS_PRE( +[if test -z "${$1_TRUE}" && test -z "${$1_FALSE}"; then + AC_MSG_ERROR([[conditional "$1" was never defined. +Usually this means the macro was only invoked conditionally.]]) +fi])]) + +# serial 7 -*- Autoconf -*- + +# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004 +# Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + + +# There are a few dirty hacks below to avoid letting `AC_PROG_CC' be +# written in clear, in which case automake, when reading aclocal.m4, +# will think it sees a *use*, and therefore will trigger all it's +# C support machinery. Also note that it means that autoscan, seeing +# CC etc. in the Makefile, will ask for an AC_PROG_CC use... + + + +# _AM_DEPENDENCIES(NAME) +# ---------------------- +# See how the compiler implements dependency checking. +# NAME is "CC", "CXX", "GCJ", or "OBJC". +# We try a few techniques and use that to set a single cache variable. +# +# We don't AC_REQUIRE the corresponding AC_PROG_CC since the latter was +# modified to invoke _AM_DEPENDENCIES(CC); we would have a circular +# dependency, and given that the user is not expected to run this macro, +# just rely on AC_PROG_CC. +AC_DEFUN([_AM_DEPENDENCIES], +[AC_REQUIRE([AM_SET_DEPDIR])dnl +AC_REQUIRE([AM_OUTPUT_DEPENDENCY_COMMANDS])dnl +AC_REQUIRE([AM_MAKE_INCLUDE])dnl +AC_REQUIRE([AM_DEP_TRACK])dnl + +ifelse([$1], CC, [depcc="$CC" am_compiler_list=], + [$1], CXX, [depcc="$CXX" am_compiler_list=], + [$1], OBJC, [depcc="$OBJC" am_compiler_list='gcc3 gcc'], + [$1], GCJ, [depcc="$GCJ" am_compiler_list='gcc3 gcc'], + [depcc="$$1" am_compiler_list=]) + +AC_CACHE_CHECK([dependency style of $depcc], + [am_cv_$1_dependencies_compiler_type], +[if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_$1_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n ['s/^#*\([a-zA-Z0-9]*\))$/\1/p'] < ./depcomp` + fi + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + case $depmode in + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + none) break ;; + esac + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. + if depmode=$depmode \ + source=sub/conftest.c object=sub/conftest.${OBJEXT-o} \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c -o sub/conftest.${OBJEXT-o} sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftest.${OBJEXT-o} sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_$1_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_$1_dependencies_compiler_type=none +fi +]) +AC_SUBST([$1DEPMODE], [depmode=$am_cv_$1_dependencies_compiler_type]) +AM_CONDITIONAL([am__fastdep$1], [ + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_$1_dependencies_compiler_type" = gcc3]) +]) + + +# AM_SET_DEPDIR +# ------------- +# Choose a directory name for dependency files. +# This macro is AC_REQUIREd in _AM_DEPENDENCIES +AC_DEFUN([AM_SET_DEPDIR], +[AC_REQUIRE([AM_SET_LEADING_DOT])dnl +AC_SUBST([DEPDIR], ["${am__leading_dot}deps"])dnl +]) + + +# AM_DEP_TRACK +# ------------ +AC_DEFUN([AM_DEP_TRACK], +[AC_ARG_ENABLE(dependency-tracking, +[ --disable-dependency-tracking speeds up one-time build + --enable-dependency-tracking do not reject slow dependency extractors]) +if test "x$enable_dependency_tracking" != xno; then + am_depcomp="$ac_aux_dir/depcomp" + AMDEPBACKSLASH='\' +fi +AM_CONDITIONAL([AMDEP], [test "x$enable_dependency_tracking" != xno]) +AC_SUBST([AMDEPBACKSLASH]) +]) + +# Generate code to set up dependency tracking. -*- Autoconf -*- + +# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004 +# Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +#serial 2 + +# _AM_OUTPUT_DEPENDENCY_COMMANDS +# ------------------------------ +AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS], +[for mf in $CONFIG_FILES; do + # Strip MF so we end up with the name of the file. + mf=`echo "$mf" | sed -e 's/:.*$//'` + # Check whether this is an Automake generated Makefile or not. + # We used to match only the files named `Makefile.in', but + # some people rename them; so instead we look at the file content. + # Grep'ing the first line is not enough: some people post-process + # each Makefile.in and add a new line on top of each file to say so. + # So let's grep whole file. + if grep '^#.*generated by automake' $mf > /dev/null 2>&1; then + dirpart=`AS_DIRNAME("$mf")` + else + continue + fi + # Extract the definition of DEPDIR, am__include, and am__quote + # from the Makefile without running `make'. + DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` + test -z "$DEPDIR" && continue + am__include=`sed -n 's/^am__include = //p' < "$mf"` + test -z "am__include" && continue + am__quote=`sed -n 's/^am__quote = //p' < "$mf"` + # When using ansi2knr, U may be empty or an underscore; expand it + U=`sed -n 's/^U = //p' < "$mf"` + # Find all dependency output files, they are included files with + # $(DEPDIR) in their names. We invoke sed twice because it is the + # simplest approach to changing $(DEPDIR) to its actual value in the + # expansion. + for file in `sed -n " + s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ + sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do + # Make sure the directory exists. + test -f "$dirpart/$file" && continue + fdir=`AS_DIRNAME(["$file"])` + AS_MKDIR_P([$dirpart/$fdir]) + # echo "creating $dirpart/$file" + echo '# dummy' > "$dirpart/$file" + done +done +])# _AM_OUTPUT_DEPENDENCY_COMMANDS + + +# AM_OUTPUT_DEPENDENCY_COMMANDS +# ----------------------------- +# This macro should only be invoked once -- use via AC_REQUIRE. +# +# This code is only required when automatic dependency tracking +# is enabled. FIXME. This creates each `.P' file that we will +# need in order to bootstrap the dependency handling code. +AC_DEFUN([AM_OUTPUT_DEPENDENCY_COMMANDS], +[AC_CONFIG_COMMANDS([depfiles], + [test x"$AMDEP_TRUE" != x"" || _AM_OUTPUT_DEPENDENCY_COMMANDS], + [AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"]) +]) + +# Like AC_CONFIG_HEADER, but automatically create stamp file. -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 2000, 2001, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 7 + +# AM_CONFIG_HEADER is obsolete. It has been replaced by AC_CONFIG_HEADERS. +AU_DEFUN([AM_CONFIG_HEADER], [AC_CONFIG_HEADERS($@)]) + +# Do all the work for Automake. -*- Autoconf -*- + +# This macro actually does too much some checks are only needed if +# your package does certain things. But this isn't really a big deal. + +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004 +# Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 11 + +# AM_INIT_AUTOMAKE(PACKAGE, VERSION, [NO-DEFINE]) +# AM_INIT_AUTOMAKE([OPTIONS]) +# ----------------------------------------------- +# The call with PACKAGE and VERSION arguments is the old style +# call (pre autoconf-2.50), which is being phased out. PACKAGE +# and VERSION should now be passed to AC_INIT and removed from +# the call to AM_INIT_AUTOMAKE. +# We support both call styles for the transition. After +# the next Automake release, Autoconf can make the AC_INIT +# arguments mandatory, and then we can depend on a new Autoconf +# release and drop the old call support. +AC_DEFUN([AM_INIT_AUTOMAKE], +[AC_PREREQ([2.58])dnl +dnl Autoconf wants to disallow AM_ names. We explicitly allow +dnl the ones we care about. +m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl +AC_REQUIRE([AM_SET_CURRENT_AUTOMAKE_VERSION])dnl +AC_REQUIRE([AC_PROG_INSTALL])dnl +# test to see if srcdir already configured +if test "`cd $srcdir && pwd`" != "`pwd`" && + test -f $srcdir/config.status; then + AC_MSG_ERROR([source directory already configured; run "make distclean" there first]) +fi + +# test whether we have cygpath +if test -z "$CYGPATH_W"; then + if (cygpath --version) >/dev/null 2>/dev/null; then + CYGPATH_W='cygpath -w' + else + CYGPATH_W=echo + fi +fi +AC_SUBST([CYGPATH_W]) + +# Define the identity of the package. +dnl Distinguish between old-style and new-style calls. +m4_ifval([$2], +[m4_ifval([$3], [_AM_SET_OPTION([no-define])])dnl + AC_SUBST([PACKAGE], [$1])dnl + AC_SUBST([VERSION], [$2])], +[_AM_SET_OPTIONS([$1])dnl + AC_SUBST([PACKAGE], ['AC_PACKAGE_TARNAME'])dnl + AC_SUBST([VERSION], ['AC_PACKAGE_VERSION'])])dnl + +_AM_IF_OPTION([no-define],, +[AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE", [Name of package]) + AC_DEFINE_UNQUOTED(VERSION, "$VERSION", [Version number of package])])dnl + +# Some tools Automake needs. +AC_REQUIRE([AM_SANITY_CHECK])dnl +AC_REQUIRE([AC_ARG_PROGRAM])dnl +AM_MISSING_PROG(ACLOCAL, aclocal-${am__api_version}) +AM_MISSING_PROG(AUTOCONF, autoconf) +AM_MISSING_PROG(AUTOMAKE, automake-${am__api_version}) +AM_MISSING_PROG(AUTOHEADER, autoheader) +AM_MISSING_PROG(MAKEINFO, makeinfo) +AM_PROG_INSTALL_SH +AM_PROG_INSTALL_STRIP +AC_REQUIRE([AM_PROG_MKDIR_P])dnl +# We need awk for the "check" target. The system "awk" is bad on +# some platforms. +AC_REQUIRE([AC_PROG_AWK])dnl +AC_REQUIRE([AC_PROG_MAKE_SET])dnl +AC_REQUIRE([AM_SET_LEADING_DOT])dnl +_AM_IF_OPTION([tar-ustar], [_AM_PROG_TAR([ustar])], + [_AM_IF_OPTION([tar-pax], [_AM_PROG_TAR([pax])], + [_AM_PROG_TAR([v7])])]) +_AM_IF_OPTION([no-dependencies],, +[AC_PROVIDE_IFELSE([AC_PROG_CC], + [_AM_DEPENDENCIES(CC)], + [define([AC_PROG_CC], + defn([AC_PROG_CC])[_AM_DEPENDENCIES(CC)])])dnl +AC_PROVIDE_IFELSE([AC_PROG_CXX], + [_AM_DEPENDENCIES(CXX)], + [define([AC_PROG_CXX], + defn([AC_PROG_CXX])[_AM_DEPENDENCIES(CXX)])])dnl +]) +]) + + +# When config.status generates a header, we must update the stamp-h file. +# This file resides in the same directory as the config header +# that is generated. The stamp files are numbered to have different names. + +# Autoconf calls _AC_AM_CONFIG_HEADER_HOOK (when defined) in the +# loop where config.status creates the headers, so we can generate +# our stamp files there. +AC_DEFUN([_AC_AM_CONFIG_HEADER_HOOK], +[# Compute $1's index in $config_headers. +_am_stamp_count=1 +for _am_header in $config_headers :; do + case $_am_header in + $1 | $1:* ) + break ;; + * ) + _am_stamp_count=`expr $_am_stamp_count + 1` ;; + esac +done +echo "timestamp for $1" >`AS_DIRNAME([$1])`/stamp-h[]$_am_stamp_count]) + +# AM_PROG_INSTALL_SH +# ------------------ +# Define $install_sh. + +# Copyright (C) 2001, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +AC_DEFUN([AM_PROG_INSTALL_SH], +[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +install_sh=${install_sh-"$am_aux_dir/install-sh"} +AC_SUBST(install_sh)]) + +# -*- Autoconf -*- +# Copyright (C) 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 1 + +# Check whether the underlying file-system supports filenames +# with a leading dot. For instance MS-DOS doesn't. +AC_DEFUN([AM_SET_LEADING_DOT], +[rm -rf .tst 2>/dev/null +mkdir .tst 2>/dev/null +if test -d .tst; then + am__leading_dot=. +else + am__leading_dot=_ +fi +rmdir .tst 2>/dev/null +AC_SUBST([am__leading_dot])]) + +# Check to see how 'make' treats includes. -*- Autoconf -*- + +# Copyright (C) 2001, 2002, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 2 + +# AM_MAKE_INCLUDE() +# ----------------- +# Check to see how make treats includes. +AC_DEFUN([AM_MAKE_INCLUDE], +[am_make=${MAKE-make} +cat > confinc << 'END' +am__doit: + @echo done +.PHONY: am__doit +END +# If we don't find an include directive, just comment out the code. +AC_MSG_CHECKING([for style of include used by $am_make]) +am__include="#" +am__quote= +_am_result=none +# First try GNU make style include. +echo "include confinc" > confmf +# We grep out `Entering directory' and `Leaving directory' +# messages which can occur if `w' ends up in MAKEFLAGS. +# In particular we don't look at `^make:' because GNU make might +# be invoked under some other name (usually "gmake"), in which +# case it prints its new name instead of `make'. +if test "`$am_make -s -f confmf 2> /dev/null | grep -v 'ing directory'`" = "done"; then + am__include=include + am__quote= + _am_result=GNU +fi +# Now try BSD make style include. +if test "$am__include" = "#"; then + echo '.include "confinc"' > confmf + if test "`$am_make -s -f confmf 2> /dev/null`" = "done"; then + am__include=.include + am__quote="\"" + _am_result=BSD + fi +fi +AC_SUBST([am__include]) +AC_SUBST([am__quote]) +AC_MSG_RESULT([$_am_result]) +rm -f confinc confmf +]) + +# serial 2 + +# AM_PROG_CC_C_O +# -------------- +# Like AC_PROG_CC_C_O, but changed for automake. + +# Copyright (C) 1999, 2000, 2001, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +AC_DEFUN([AM_PROG_CC_C_O], +[AC_REQUIRE([AC_PROG_CC_C_O])dnl +AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +# FIXME: we rely on the cache variable name because +# there is no other way. +set dummy $CC +ac_cc=`echo $[2] | sed ['s/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/']` +if eval "test \"`echo '$ac_cv_prog_cc_'${ac_cc}_c_o`\" != yes"; then + # Losing compiler, so override with the script. + # FIXME: It is wrong to rewrite CC. + # But if we don't then we get into trouble of one sort or another. + # A longer-term fix would be to have automake use am__CC in this case, + # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" + CC="$am_aux_dir/compile $CC" +fi +]) + +# -*- Autoconf -*- + + +# Copyright (C) 1997, 1999, 2000, 2001, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 3 + +# AM_MISSING_PROG(NAME, PROGRAM) +# ------------------------------ +AC_DEFUN([AM_MISSING_PROG], +[AC_REQUIRE([AM_MISSING_HAS_RUN]) +$1=${$1-"${am_missing_run}$2"} +AC_SUBST($1)]) + + +# AM_MISSING_HAS_RUN +# ------------------ +# Define MISSING if not defined so far and test if it supports --run. +# If it does, set am_missing_run to use it, otherwise, to nothing. +AC_DEFUN([AM_MISSING_HAS_RUN], +[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +test x"${MISSING+set}" = xset || MISSING="\${SHELL} $am_aux_dir/missing" +# Use eval to expand $SHELL +if eval "$MISSING --run true"; then + am_missing_run="$MISSING --run " +else + am_missing_run= + AC_MSG_WARN([`missing' script is too old or missing]) +fi +]) + +# AM_PROG_MKDIR_P +# --------------- +# Check whether `mkdir -p' is supported, fallback to mkinstalldirs otherwise. + +# Copyright (C) 2003, 2004 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# Automake 1.8 used `mkdir -m 0755 -p --' to ensure that directories +# created by `make install' are always world readable, even if the +# installer happens to have an overly restrictive umask (e.g. 077). +# This was a mistake. There are at least two reasons why we must not +# use `-m 0755': +# - it causes special bits like SGID to be ignored, +# - it may be too restrictive (some setups expect 775 directories). +# +# Do not use -m 0755 and let people choose whatever they expect by +# setting umask. +# +# We cannot accept any implementation of `mkdir' that recognizes `-p'. +# Some implementations (such as Solaris 8's) are not thread-safe: if a +# parallel make tries to run `mkdir -p a/b' and `mkdir -p a/c' +# concurrently, both version can detect that a/ is missing, but only +# one can create it and the other will error out. Consequently we +# restrict ourselves to GNU make (using the --version option ensures +# this.) +AC_DEFUN([AM_PROG_MKDIR_P], +[if mkdir -p --version . >/dev/null 2>&1 && test ! -d ./--version; then + # We used to keeping the `.' as first argument, in order to + # allow $(mkdir_p) to be used without argument. As in + # $(mkdir_p) $(somedir) + # where $(somedir) is conditionally defined. However this is wrong + # for two reasons: + # 1. if the package is installed by a user who cannot write `.' + # make install will fail, + # 2. the above comment should most certainly read + # $(mkdir_p) $(DESTDIR)$(somedir) + # so it does not work when $(somedir) is undefined and + # $(DESTDIR) is not. + # To support the latter case, we have to write + # test -z "$(somedir)" || $(mkdir_p) $(DESTDIR)$(somedir), + # so the `.' trick is pointless. + mkdir_p='mkdir -p --' +else + # On NextStep and OpenStep, the `mkdir' command does not + # recognize any option. It will interpret all options as + # directories to create, and then abort because `.' already + # exists. + for d in ./-p ./--version; + do + test -d $d && rmdir $d + done + # $(mkinstalldirs) is defined by Automake if mkinstalldirs exists. + if test -f "$ac_aux_dir/mkinstalldirs"; then + mkdir_p='$(mkinstalldirs)' + else + mkdir_p='$(install_sh) -d' + fi +fi +AC_SUBST([mkdir_p])]) + +# Helper functions for option handling. -*- Autoconf -*- + +# Copyright (C) 2001, 2002, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 2 + +# _AM_MANGLE_OPTION(NAME) +# ----------------------- +AC_DEFUN([_AM_MANGLE_OPTION], +[[_AM_OPTION_]m4_bpatsubst($1, [[^a-zA-Z0-9_]], [_])]) + +# _AM_SET_OPTION(NAME) +# ------------------------------ +# Set option NAME. Presently that only means defining a flag for this option. +AC_DEFUN([_AM_SET_OPTION], +[m4_define(_AM_MANGLE_OPTION([$1]), 1)]) + +# _AM_SET_OPTIONS(OPTIONS) +# ---------------------------------- +# OPTIONS is a space-separated list of Automake options. +AC_DEFUN([_AM_SET_OPTIONS], +[AC_FOREACH([_AM_Option], [$1], [_AM_SET_OPTION(_AM_Option)])]) + +# _AM_IF_OPTION(OPTION, IF-SET, [IF-NOT-SET]) +# ------------------------------------------- +# Execute IF-SET if OPTION is set, IF-NOT-SET otherwise. +AC_DEFUN([_AM_IF_OPTION], +[m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])]) + +# +# Check to make sure that the build environment is sane. +# + +# Copyright (C) 1996, 1997, 2000, 2001, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 3 + +# AM_SANITY_CHECK +# --------------- +AC_DEFUN([AM_SANITY_CHECK], +[AC_MSG_CHECKING([whether build environment is sane]) +# Just in case +sleep 1 +echo timestamp > conftest.file +# Do `set' in a subshell so we don't clobber the current shell's +# arguments. Must try -L first in case configure is actually a +# symlink; some systems play weird games with the mod time of symlinks +# (eg FreeBSD returns the mod time of the symlink's containing +# directory). +if ( + set X `ls -Lt $srcdir/configure conftest.file 2> /dev/null` + if test "$[*]" = "X"; then + # -L didn't work. + set X `ls -t $srcdir/configure conftest.file` + fi + rm -f conftest.file + if test "$[*]" != "X $srcdir/configure conftest.file" \ + && test "$[*]" != "X conftest.file $srcdir/configure"; then + + # If neither matched, then we have a broken ls. This can happen + # if, for instance, CONFIG_SHELL is bash and it inherits a + # broken ls alias from the environment. This has actually + # happened. Such a system could not be considered "sane". + AC_MSG_ERROR([ls -t appears to fail. Make sure there is not a broken +alias in your environment]) + fi + + test "$[2]" = conftest.file + ) +then + # Ok. + : +else + AC_MSG_ERROR([newly created file is older than distributed files! +Check your system clock]) +fi +AC_MSG_RESULT(yes)]) + +# AM_PROG_INSTALL_STRIP + +# Copyright (C) 2001, 2003 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# One issue with vendor `install' (even GNU) is that you can't +# specify the program used to strip binaries. This is especially +# annoying in cross-compiling environments, where the build's strip +# is unlikely to handle the host's binaries. +# Fortunately install-sh will honor a STRIPPROG variable, so we +# always use install-sh in `make install-strip', and initialize +# STRIPPROG with the value of the STRIP variable (set by the user). +AC_DEFUN([AM_PROG_INSTALL_STRIP], +[AC_REQUIRE([AM_PROG_INSTALL_SH])dnl +# Installed binaries are usually stripped using `strip' when the user +# run `make install-strip'. However `strip' might not be the right +# tool to use in cross-compilation environments, therefore Automake +# will honor the `STRIP' environment variable to overrule this program. +dnl Don't test for $cross_compiling = yes, because it might be `maybe'. +if test "$cross_compiling" != no; then + AC_CHECK_TOOL([STRIP], [strip], :) +fi +INSTALL_STRIP_PROGRAM="\${SHELL} \$(install_sh) -c -s" +AC_SUBST([INSTALL_STRIP_PROGRAM])]) + +# Check how to create a tarball. -*- Autoconf -*- + +# Copyright (C) 2004 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# serial 1 + + +# _AM_PROG_TAR(FORMAT) +# -------------------- +# Check how to create a tarball in format FORMAT. +# FORMAT should be one of `v7', `ustar', or `pax'. +# +# Substitute a variable $(am__tar) that is a command +# writing to stdout a FORMAT-tarball containing the directory +# $tardir. +# tardir=directory && $(am__tar) > result.tar +# +# Substitute a variable $(am__untar) that extract such +# a tarball read from stdin. +# $(am__untar) < result.tar +AC_DEFUN([_AM_PROG_TAR], +[# Always define AMTAR for backward compatibility. +AM_MISSING_PROG([AMTAR], [tar]) +m4_if([$1], [v7], + [am__tar='${AMTAR} chof - "$$tardir"'; am__untar='${AMTAR} xf -'], + [m4_case([$1], [ustar],, [pax],, + [m4_fatal([Unknown tar format])]) +AC_MSG_CHECKING([how to create a $1 tar archive]) +# Loop over all known methods to create a tar archive until one works. +_am_tools='gnutar m4_if([$1], [ustar], [plaintar]) pax cpio none' +_am_tools=${am_cv_prog_tar_$1-$_am_tools} +# Do not fold the above two line into one, because Tru64 sh and +# Solaris sh will not grok spaces in the rhs of `-'. +for _am_tool in $_am_tools +do + case $_am_tool in + gnutar) + for _am_tar in tar gnutar gtar; + do + AM_RUN_LOG([$_am_tar --version]) && break + done + am__tar="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$$tardir"' + am__tar_="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$tardir"' + am__untar="$_am_tar -xf -" + ;; + plaintar) + # Must skip GNU tar: if it does not support --format= it doesn't create + # ustar tarball either. + (tar --version) >/dev/null 2>&1 && continue + am__tar='tar chf - "$$tardir"' + am__tar_='tar chf - "$tardir"' + am__untar='tar xf -' + ;; + pax) + am__tar='pax -L -x $1 -w "$$tardir"' + am__tar_='pax -L -x $1 -w "$tardir"' + am__untar='pax -r' + ;; + cpio) + am__tar='find "$$tardir" -print | cpio -o -H $1 -L' + am__tar_='find "$tardir" -print | cpio -o -H $1 -L' + am__untar='cpio -i -H $1 -d' + ;; + none) + am__tar=false + am__tar_=false + am__untar=false + ;; + esac + + # If the value was cached, stop now. We just wanted to have am__tar + # and am__untar set. + test -n "${am_cv_prog_tar_$1}" && break + + # tar/untar a dummy directory, and stop if the command works + rm -rf conftest.dir + mkdir conftest.dir + echo GrepMe > conftest.dir/file + AM_RUN_LOG([tardir=conftest.dir && eval $am__tar_ >conftest.tar]) + rm -rf conftest.dir + if test -s conftest.tar; then + AM_RUN_LOG([$am__untar <conftest.tar]) + grep GrepMe conftest.dir/file >/dev/null 2>&1 && break + fi +done +rm -rf conftest.dir + +AC_CACHE_VAL([am_cv_prog_tar_$1], [am_cv_prog_tar_$1=$_am_tool]) +AC_MSG_RESULT([$am_cv_prog_tar_$1])]) +AC_SUBST([am__tar]) +AC_SUBST([am__untar]) +]) # _AM_PROG_TAR + diff --git a/contrib/lmserver/compile b/contrib/lmserver/compile new file mode 100755 index 000000000..1b1d23216 --- /dev/null +++ b/contrib/lmserver/compile @@ -0,0 +1,142 @@ +#! /bin/sh +# Wrapper for compilers which do not understand `-c -o'. + +scriptversion=2005-05-14.22 + +# Copyright (C) 1999, 2000, 2003, 2004, 2005 Free Software Foundation, Inc. +# Written by Tom Tromey <tromey@cygnus.com>. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to <bug-automake@gnu.org> or send patches to +# <automake-patches@gnu.org>. + +case $1 in + '') + echo "$0: No command. Try \`$0 --help' for more information." 1>&2 + exit 1; + ;; + -h | --h*) + cat <<\EOF +Usage: compile [--help] [--version] PROGRAM [ARGS] + +Wrapper for compilers which do not understand `-c -o'. +Remove `-o dest.o' from ARGS, run PROGRAM with the remaining +arguments, and rename the output as expected. + +If you are trying to build a whole package this is not the +right script to run: please start by reading the file `INSTALL'. + +Report bugs to <bug-automake@gnu.org>. +EOF + exit $? + ;; + -v | --v*) + echo "compile $scriptversion" + exit $? + ;; +esac + +ofile= +cfile= +eat= + +for arg +do + if test -n "$eat"; then + eat= + else + case $1 in + -o) + # configure might choose to run compile as `compile cc -o foo foo.c'. + # So we strip `-o arg' only if arg is an object. + eat=1 + case $2 in + *.o | *.obj) + ofile=$2 + ;; + *) + set x "$@" -o "$2" + shift + ;; + esac + ;; + *.c) + cfile=$1 + set x "$@" "$1" + shift + ;; + *) + set x "$@" "$1" + shift + ;; + esac + fi + shift +done + +if test -z "$ofile" || test -z "$cfile"; then + # If no `-o' option was seen then we might have been invoked from a + # pattern rule where we don't need one. That is ok -- this is a + # normal compilation that the losing compiler can handle. If no + # `.c' file was seen then we are probably linking. That is also + # ok. + exec "$@" +fi + +# Name of file we expect compiler to create. +cofile=`echo "$cfile" | sed -e 's|^.*/||' -e 's/\.c$/.o/'` + +# Create the lock directory. +# Note: use `[/.-]' here to ensure that we don't use the same name +# that we are using for the .o file. Also, base the name on the expected +# object file name, since that is what matters with a parallel build. +lockdir=`echo "$cofile" | sed -e 's|[/.-]|_|g'`.d +while true; do + if mkdir "$lockdir" >/dev/null 2>&1; then + break + fi + sleep 1 +done +# FIXME: race condition here if user kills between mkdir and trap. +trap "rmdir '$lockdir'; exit 1" 1 2 15 + +# Run the compile. +"$@" +ret=$? + +if test -f "$cofile"; then + mv "$cofile" "$ofile" +elif test -f "${cofile}bj"; then + mv "${cofile}bj" "$ofile" +fi + +rmdir "$lockdir" +exit $ret + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/contrib/lmserver/config.guess b/contrib/lmserver/config.guess new file mode 100755 index 000000000..2313a174e --- /dev/null +++ b/contrib/lmserver/config.guess @@ -0,0 +1,1545 @@ +#! /bin/sh +# Attempt to guess a canonical system name. +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, +# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 +# Free Software Foundation, Inc. + +timestamp='2008-01-23' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA +# 02110-1301, USA. +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + + +# Originally written by Per Bothner <per@bothner.com>. +# Please send patches to <config-patches@gnu.org>. Submit a context +# diff and a properly formatted ChangeLog entry. +# +# This script attempts to guess a canonical system name similar to +# config.sub. If it succeeds, it prints the system name on stdout, and +# exits with 0. Otherwise, it exits with 1. +# +# The plan is that this can be called by configure scripts if you +# don't specify an explicit build system type. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] + +Output the configuration name of the system \`$me' is run on. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to <config-patches@gnu.org>." + +version="\ +GNU config.guess ($timestamp) + +Originally written by Per Bothner. +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, +2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" >&2 + exit 1 ;; + * ) + break ;; + esac +done + +if test $# != 0; then + echo "$me: too many arguments$help" >&2 + exit 1 +fi + +trap 'exit 1' 1 2 15 + +# CC_FOR_BUILD -- compiler used by this script. Note that the use of a +# compiler to aid in system detection is discouraged as it requires +# temporary files to be created and, as you can see below, it is a +# headache to deal with in a portable fashion. + +# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still +# use `HOST_CC' if defined, but it is deprecated. + +# Portable tmp directory creation inspired by the Autoconf team. + +set_cc_for_build=' +trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; +trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; +: ${TMPDIR=/tmp} ; + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; +dummy=$tmp/dummy ; +tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; +case $CC_FOR_BUILD,$HOST_CC,$CC in + ,,) echo "int x;" > $dummy.c ; + for c in cc gcc c89 c99 ; do + if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then + CC_FOR_BUILD="$c"; break ; + fi ; + done ; + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found ; + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; +esac ; set_cc_for_build= ;' + +# This is needed to find uname on a Pyramid OSx when run in the BSD universe. +# (ghazi@noc.rutgers.edu 1994-08-24) +if (test -f /.attbin/uname) >/dev/null 2>&1 ; then + PATH=$PATH:/.attbin ; export PATH +fi + +UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown +UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown +UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown +UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown + +if [ "${UNAME_SYSTEM}" = "Linux" ] ; then + eval $set_cc_for_build + cat << EOF > $dummy.c + #include <features.h> + #ifdef __UCLIBC__ + # ifdef __UCLIBC_CONFIG_VERSION__ + LIBC=uclibc __UCLIBC_CONFIG_VERSION__ + # else + LIBC=uclibc + # endif + #else + LIBC=gnu + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep LIBC= | sed -e 's: ::g'` +fi + +# Note: order is significant - the case branches are not exclusive. + +case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in + *:NetBSD:*:*) + # NetBSD (nbsd) targets should (where applicable) match one or + # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*, + # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently + # switched to ELF, *-*-netbsd* would select the old + # object file format. This provides both forward + # compatibility and a consistent mechanism for selecting the + # object file format. + # + # Note: NetBSD doesn't particularly care about the vendor + # portion of the name. We always set it to "unknown". + sysctl="sysctl -n hw.machine_arch" + UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + case "${UNAME_MACHINE_ARCH}" in + armeb) machine=armeb-unknown ;; + arm*) machine=arm-unknown ;; + sh3el) machine=shl-unknown ;; + sh3eb) machine=sh-unknown ;; + sh5el) machine=sh5le-unknown ;; + *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + esac + # The Operating System including object format, if it has switched + # to ELF recently, or will in the future. + case "${UNAME_MACHINE_ARCH}" in + arm*|i386|m68k|ns32k|sh3*|sparc|vax) + eval $set_cc_for_build + if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep __ELF__ >/dev/null + then + # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). + # Return netbsd for either. FIX? + os=netbsd + else + os=netbsdelf + fi + ;; + *) + os=netbsd + ;; + esac + # The OS release + # Debian GNU/NetBSD machines have a different userland, and + # thus, need a distinct triplet. However, they do not need + # kernel version information, so it can be replaced with a + # suitable tag, in the style of linux-gnu. + case "${UNAME_VERSION}" in + Debian*) + release='-gnu' + ;; + *) + release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + ;; + esac + # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: + # contains redundant information, the shorter form: + # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. + echo "${machine}-${os}${release}" + exit ;; + *:OpenBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + exit ;; + *:ekkoBSD:*:*) + echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + exit ;; + *:SolidBSD:*:*) + echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + exit ;; + macppc:MirBSD:*:*) + echo powerpc-unknown-mirbsd${UNAME_RELEASE} + exit ;; + *:MirBSD:*:*) + echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + exit ;; + alpha:OSF1:*:*) + case $UNAME_RELEASE in + *4.0) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + ;; + *5.*) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + ;; + esac + # According to Compaq, /usr/sbin/psrinfo has been available on + # OSF/1 and Tru64 systems produced since 1995. I hope that + # covers most systems running today. This code pipes the CPU + # types through head -n 1, so we only detect the type of CPU 0. + ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + case "$ALPHA_CPU_TYPE" in + "EV4 (21064)") + UNAME_MACHINE="alpha" ;; + "EV4.5 (21064)") + UNAME_MACHINE="alpha" ;; + "LCA4 (21066/21068)") + UNAME_MACHINE="alpha" ;; + "EV5 (21164)") + UNAME_MACHINE="alphaev5" ;; + "EV5.6 (21164A)") + UNAME_MACHINE="alphaev56" ;; + "EV5.6 (21164PC)") + UNAME_MACHINE="alphapca56" ;; + "EV5.7 (21164PC)") + UNAME_MACHINE="alphapca57" ;; + "EV6 (21264)") + UNAME_MACHINE="alphaev6" ;; + "EV6.7 (21264A)") + UNAME_MACHINE="alphaev67" ;; + "EV6.8CB (21264C)") + UNAME_MACHINE="alphaev68" ;; + "EV6.8AL (21264B)") + UNAME_MACHINE="alphaev68" ;; + "EV6.8CX (21264D)") + UNAME_MACHINE="alphaev68" ;; + "EV6.9A (21264/EV69A)") + UNAME_MACHINE="alphaev69" ;; + "EV7 (21364)") + UNAME_MACHINE="alphaev7" ;; + "EV7.9 (21364A)") + UNAME_MACHINE="alphaev79" ;; + esac + # A Pn.n version is a patched version. + # A Vn.n version is a released version. + # A Tn.n version is a released field test version. + # A Xn.n version is an unreleased experimental baselevel. + # 1.2 uses "1.2" for uname -r. + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + exit ;; + Alpha\ *:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # Should we change UNAME_MACHINE based on the output of uname instead + # of the specific Alpha model? + echo alpha-pc-interix + exit ;; + 21064:Windows_NT:50:3) + echo alpha-dec-winnt3.5 + exit ;; + Amiga*:UNIX_System_V:4.0:*) + echo m68k-unknown-sysv4 + exit ;; + *:[Aa]miga[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-amigaos + exit ;; + *:[Mm]orph[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-morphos + exit ;; + *:OS/390:*:*) + echo i370-ibm-openedition + exit ;; + *:z/VM:*:*) + echo s390-ibm-zvmoe + exit ;; + *:OS400:*:*) + echo powerpc-ibm-os400 + exit ;; + arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) + echo arm-acorn-riscix${UNAME_RELEASE} + exit ;; + arm:riscos:*:*|arm:RISCOS:*:*) + echo arm-unknown-riscos + exit ;; + SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) + echo hppa1.1-hitachi-hiuxmpp + exit ;; + Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) + # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. + if test "`(/bin/universe) 2>/dev/null`" = att ; then + echo pyramid-pyramid-sysv3 + else + echo pyramid-pyramid-bsd + fi + exit ;; + NILE*:*:*:dcosx) + echo pyramid-pyramid-svr4 + exit ;; + DRS?6000:unix:4.0:6*) + echo sparc-icl-nx6 + exit ;; + DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) + case `/usr/bin/uname -p` in + sparc) echo sparc-icl-nx7; exit ;; + esac ;; + sun4H:SunOS:5.*:*) + echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) + echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) + echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:6*:*) + # According to config.sub, this is the proper way to canonicalize + # SunOS6. Hard to guess exactly what SunOS6 will be like, but + # it's likely to be more like Solaris than SunOS4. + echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:*:*) + case "`/usr/bin/arch -k`" in + Series*|S4*) + UNAME_RELEASE=`uname -v` + ;; + esac + # Japanese Language versions have a version number like `4.1.3-JL'. + echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + exit ;; + sun3*:SunOS:*:*) + echo m68k-sun-sunos${UNAME_RELEASE} + exit ;; + sun*:*:4.2BSD:*) + UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` + test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + case "`/bin/arch`" in + sun3) + echo m68k-sun-sunos${UNAME_RELEASE} + ;; + sun4) + echo sparc-sun-sunos${UNAME_RELEASE} + ;; + esac + exit ;; + aushp:SunOS:*:*) + echo sparc-auspex-sunos${UNAME_RELEASE} + exit ;; + # The situation for MiNT is a little confusing. The machine name + # can be virtually everything (everything which is not + # "atarist" or "atariste" at least should have a processor + # > m68000). The system name ranges from "MiNT" over "FreeMiNT" + # to the lowercase version "mint" (or "freemint"). Finally + # the system name "TOS" denotes a system which is actually not + # MiNT. But MiNT is downward compatible to TOS, so this should + # be no problem. + atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) + echo m68k-milan-mint${UNAME_RELEASE} + exit ;; + hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) + echo m68k-hades-mint${UNAME_RELEASE} + exit ;; + *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) + echo m68k-unknown-mint${UNAME_RELEASE} + exit ;; + m68k:machten:*:*) + echo m68k-apple-machten${UNAME_RELEASE} + exit ;; + powerpc:machten:*:*) + echo powerpc-apple-machten${UNAME_RELEASE} + exit ;; + RISC*:Mach:*:*) + echo mips-dec-mach_bsd4.3 + exit ;; + RISC*:ULTRIX:*:*) + echo mips-dec-ultrix${UNAME_RELEASE} + exit ;; + VAX*:ULTRIX*:*:*) + echo vax-dec-ultrix${UNAME_RELEASE} + exit ;; + 2020:CLIX:*:* | 2430:CLIX:*:*) + echo clipper-intergraph-clix${UNAME_RELEASE} + exit ;; + mips:*:*:UMIPS | mips:*:*:RISCos) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c +#ifdef __cplusplus +#include <stdio.h> /* for printf() prototype */ + int main (int argc, char *argv[]) { +#else + int main (argc, argv) int argc; char *argv[]; { +#endif + #if defined (host_mips) && defined (MIPSEB) + #if defined (SYSTYPE_SYSV) + printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_SVR4) + printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) + printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + #endif + #endif + exit (-1); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && + dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`$dummy $dummyarg` && + { echo "$SYSTEM_NAME"; exit; } + echo mips-mips-riscos${UNAME_RELEASE} + exit ;; + Motorola:PowerMAX_OS:*:*) + echo powerpc-motorola-powermax + exit ;; + Motorola:*:4.3:PL8-*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:Power_UNIX:*:*) + echo powerpc-harris-powerunix + exit ;; + m88k:CX/UX:7*:*) + echo m88k-harris-cxux7 + exit ;; + m88k:*:4*:R4*) + echo m88k-motorola-sysv4 + exit ;; + m88k:*:3*:R3*) + echo m88k-motorola-sysv3 + exit ;; + AViiON:dgux:*:*) + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` + if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + then + if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ + [ ${TARGET_BINARY_INTERFACE}x = x ] + then + echo m88k-dg-dgux${UNAME_RELEASE} + else + echo m88k-dg-dguxbcs${UNAME_RELEASE} + fi + else + echo i586-dg-dgux${UNAME_RELEASE} + fi + exit ;; + M88*:DolphinOS:*:*) # DolphinOS (SVR3) + echo m88k-dolphin-sysv3 + exit ;; + M88*:*:R3*:*) + # Delta 88k system running SVR3 + echo m88k-motorola-sysv3 + exit ;; + XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) + echo m88k-tektronix-sysv3 + exit ;; + Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) + echo m68k-tektronix-bsd + exit ;; + *:IRIX*:*:*) + echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + exit ;; + ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + i*86:AIX:*:*) + echo i386-ibm-aix + exit ;; + ia64:AIX:*:*) + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + exit ;; + *:AIX:2:3) + if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include <sys/systemcfg.h> + + main() + { + if (!__power_pc()) + exit(1); + puts("powerpc-ibm-aix3.2.5"); + exit(0); + } +EOF + if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + then + echo "$SYSTEM_NAME" + else + echo rs6000-ibm-aix3.2.5 + fi + elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then + echo rs6000-ibm-aix3.2.4 + else + echo rs6000-ibm-aix3.2 + fi + exit ;; + *:AIX:*:[456]) + IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` + if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + IBM_ARCH=rs6000 + else + IBM_ARCH=powerpc + fi + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${IBM_ARCH}-ibm-aix${IBM_REV} + exit ;; + *:AIX:*:*) + echo rs6000-ibm-aix + exit ;; + ibmrt:4.4BSD:*|romp-ibm:BSD:*) + echo romp-ibm-bsd4.4 + exit ;; + ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and + echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + exit ;; # report: romp-ibm BSD 4.3 + *:BOSX:*:*) + echo rs6000-bull-bosx + exit ;; + DPX/2?00:B.O.S.:*:*) + echo m68k-bull-sysv3 + exit ;; + 9000/[34]??:4.3bsd:1.*:*) + echo m68k-hp-bsd + exit ;; + hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) + echo m68k-hp-bsd4.4 + exit ;; + 9000/[34678]??:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + case "${UNAME_MACHINE}" in + 9000/31? ) HP_ARCH=m68000 ;; + 9000/[34]?? ) HP_ARCH=m68k ;; + 9000/[678][0-9][0-9]) + if [ -x /usr/bin/getconf ]; then + sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 + 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH="hppa2.0n" ;; + 64) HP_ARCH="hppa2.0w" ;; + '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + esac ;; + esac + fi + if [ "${HP_ARCH}" = "" ]; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + + #define _HPUX_SOURCE + #include <stdlib.h> + #include <unistd.h> + + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); + + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } +EOF + (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + test -z "$HP_ARCH" && HP_ARCH=hppa + fi ;; + esac + if [ ${HP_ARCH} = "hppa2.0w" ] + then + eval $set_cc_for_build + + # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating + # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler + # generating 64-bit code. GNU and HP use different nomenclature: + # + # $ CC_FOR_BUILD=cc ./config.guess + # => hppa2.0w-hp-hpux11.23 + # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess + # => hppa64-hp-hpux11.23 + + if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + grep __LP64__ >/dev/null + then + HP_ARCH="hppa2.0w" + else + HP_ARCH="hppa64" + fi + fi + echo ${HP_ARCH}-hp-hpux${HPUX_REV} + exit ;; + ia64:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux${HPUX_REV} + exit ;; + 3050*:HI-UX:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include <unistd.h> + int + main () + { + long cpu = sysconf (_SC_CPU_VERSION); + /* The order matters, because CPU_IS_HP_MC68K erroneously returns + true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct + results, however. */ + if (CPU_IS_PA_RISC (cpu)) + { + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; + case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; + default: puts ("hppa-hitachi-hiuxwe2"); break; + } + } + else if (CPU_IS_HP_MC68K (cpu)) + puts ("m68k-hitachi-hiuxwe2"); + else puts ("unknown-hitachi-hiuxwe2"); + exit (0); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } + echo unknown-hitachi-hiuxwe2 + exit ;; + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + echo hppa1.1-hp-bsd + exit ;; + 9000/8??:4.3bsd:*:*) + echo hppa1.0-hp-bsd + exit ;; + *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) + echo hppa1.0-hp-mpeix + exit ;; + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + echo hppa1.1-hp-osf + exit ;; + hp8??:OSF1:*:*) + echo hppa1.0-hp-osf + exit ;; + i*86:OSF1:*:*) + if [ -x /usr/sbin/sysversion ] ; then + echo ${UNAME_MACHINE}-unknown-osf1mk + else + echo ${UNAME_MACHINE}-unknown-osf1 + fi + exit ;; + parisc*:Lites*:*:*) + echo hppa1.1-hp-lites + exit ;; + C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) + echo c1-convex-bsd + exit ;; + C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit ;; + C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) + echo c34-convex-bsd + exit ;; + C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) + echo c38-convex-bsd + exit ;; + C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) + echo c4-convex-bsd + exit ;; + CRAY*Y-MP:*:*:*) + echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*[A-Z]90:*:*:*) + echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ + -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ + -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*TS:*:*:*) + echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*T3E:*:*:*) + echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*SV1:*:*:*) + echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + *:UNICOS/mp:*:*) + echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) + FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + 5000:UNIX_System_V:4.*:*) + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) + echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + exit ;; + sparc*:BSD/OS:*:*) + echo sparc-unknown-bsdi${UNAME_RELEASE} + exit ;; + *:BSD/OS:*:*) + echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + exit ;; + *:FreeBSD:*:*) + case ${UNAME_MACHINE} in + pc98) + echo i386-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + amd64) + echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + *) + echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + esac + exit ;; + i*:CYGWIN*:*) + echo ${UNAME_MACHINE}-pc-cygwin + exit ;; + *:MINGW*:*) + echo ${UNAME_MACHINE}-pc-mingw32 + exit ;; + i*:windows32*:*) + # uname -m includes "-pc" on this system. + echo ${UNAME_MACHINE}-mingw32 + exit ;; + i*:PW*:*) + echo ${UNAME_MACHINE}-pc-pw32 + exit ;; + *:Interix*:[3456]*) + case ${UNAME_MACHINE} in + x86) + echo i586-pc-interix${UNAME_RELEASE} + exit ;; + EM64T | authenticamd) + echo x86_64-unknown-interix${UNAME_RELEASE} + exit ;; + IA64) + echo ia64-unknown-interix${UNAME_RELEASE} + exit ;; + esac ;; + [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) + echo i${UNAME_MACHINE}-pc-mks + exit ;; + i*:Windows_NT*:* | Pentium*:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we + # UNAME_MACHINE based on the output of uname instead of i386? + echo i586-pc-interix + exit ;; + i*:UWIN*:*) + echo ${UNAME_MACHINE}-pc-uwin + exit ;; + amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) + echo x86_64-unknown-cygwin + exit ;; + p*:CYGWIN*:*) + echo powerpcle-unknown-cygwin + exit ;; + prep*:SunOS:5.*:*) + echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + *:GNU:*:*) + # the GNU system + echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + exit ;; + *:GNU/*:*:*) + # other systems with GNU libc and userland + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu + exit ;; + i*86:Minix:*:*) + echo ${UNAME_MACHINE}-pc-minix + exit ;; + arm*:Linux:*:*) + eval $set_cc_for_build + if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_EABI__ + then + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + else + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi + fi + exit ;; + avr32*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + cris:Linux:*:*) + echo cris-axis-linux-${LIBC} + exit ;; + crisv32:Linux:*:*) + echo crisv32-axis-linux-${LIBC} + exit ;; + frv:Linux:*:*) + echo frv-unknown-linux-${LIBC} + exit ;; + ia64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + m32r*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + m68*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + mips:Linux:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #undef CPU + #undef mips + #undef mipsel + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) + CPU=mipsel + #else + #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) + CPU=mips + #else + CPU= + #endif + #endif +EOF + eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n ' + /^CPU/{ + s: ::g + p + }'`" + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } + ;; + mips64:Linux:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #undef CPU + #undef mips64 + #undef mips64el + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) + CPU=mips64el + #else + #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) + CPU=mips64 + #else + CPU= + #endif + #endif +EOF + eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n ' + /^CPU/{ + s: ::g + p + }'`" + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } + ;; + or32:Linux:*:*) + echo or32-unknown-linux-${LIBC} + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-${LIBC} + exit ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-${LIBC} + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null + if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + parisc:Linux:*:* | hppa:Linux:*:*) + # Look for CPU level + case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in + PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; + PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; + *) echo hppa-unknown-linux-${LIBC} ;; + esac + exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-${LIBC} + exit ;; + s390:Linux:*:* | s390x:Linux:*:*) + echo ${UNAME_MACHINE}-ibm-linux + exit ;; + sh64*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + sh*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + sparc:Linux:*:* | sparc64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + vax:Linux:*:*) + echo ${UNAME_MACHINE}-dec-linux-${LIBC} + exit ;; + x86_64:Linux:*:*) + echo x86_64-unknown-linux-${LIBC} + exit ;; + xtensa*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + i*86:Linux:*:*) + # The BFD linker knows what the default object file format is, so + # first see if it will tell us. cd to the root directory to prevent + # problems with other programs or directories called `ld' in the path. + # Set LC_ALL=C to ensure ld outputs messages in English. + ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \ + | sed -ne '/supported targets:/!d + s/[ ][ ]*/ /g + s/.*supported targets: *// + s/ .*// + p'` + case "$ld_supported_targets" in + elf32-i386) + TENTATIVE="${UNAME_MACHINE}-pc-linux-${LIBC}" + ;; + a.out-i386-linux) + echo "${UNAME_MACHINE}-pc-linux-${LIBC}aout" + exit ;; + coff-i386) + echo "${UNAME_MACHINE}-pc-linux-${LIBC}coff" + exit ;; + "") + # Either a pre-BFD a.out linker (linux-gnuoldld) or + # one that does not give us useful --help. + echo "${UNAME_MACHINE}-pc-linux-${LIBC}oldld" + exit ;; + esac + # This should get integrated into the C code below, but now we hack + if [ "$LIBC" != "gnu" ] ; then echo "$TENTATIVE" && exit 0 ; fi + # Determine whether the default compiler is a.out or elf + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include <features.h> + #ifdef __ELF__ + # ifdef __GLIBC__ + # if __GLIBC__ >= 2 + LIBC=gnu + # else + LIBC=gnulibc1 + # endif + # else + LIBC=gnulibc1 + # endif + #else + #if defined(__INTEL_COMPILER) || defined(__PGI) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) + LIBC=gnu + #else + LIBC=gnuaout + #endif + #endif + #ifdef __dietlibc__ + LIBC=dietlibc + #endif +EOF + eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n ' + /^LIBC/{ + s: ::g + p + }'`" + test x"${LIBC}" != x && { + echo "${UNAME_MACHINE}-pc-linux-${LIBC}" + exit + } + test x"${TENTATIVE}" != x && { echo "${TENTATIVE}"; exit; } + ;; + i*86:DYNIX/ptx:4*:*) + # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. + # earlier versions are messed up and put the nodename in both + # sysname and nodename. + echo i386-sequent-sysv4 + exit ;; + i*86:UNIX_SV:4.2MP:2.*) + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, + # I just have to hope. -- rms. + # Use sysv4.2uw... so that sysv4* matches it. + echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + exit ;; + i*86:OS/2:*:*) + # If we were able to find `uname', then EMX Unix compatibility + # is probably installed. + echo ${UNAME_MACHINE}-pc-os2-emx + exit ;; + i*86:XTS-300:*:STOP) + echo ${UNAME_MACHINE}-unknown-stop + exit ;; + i*86:atheos:*:*) + echo ${UNAME_MACHINE}-unknown-atheos + exit ;; + i*86:syllable:*:*) + echo ${UNAME_MACHINE}-pc-syllable + exit ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*) + echo i386-unknown-lynxos${UNAME_RELEASE} + exit ;; + i*86:*DOS:*:*) + echo ${UNAME_MACHINE}-pc-msdosdjgpp + exit ;; + i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) + UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then + echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + else + echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + fi + exit ;; + i*86:*:5:[678]*) + # UnixWare 7.x, OpenUNIX and OpenServer 6. + case `/bin/uname -X | grep "^Machine"` in + *486*) UNAME_MACHINE=i486 ;; + *Pentium) UNAME_MACHINE=i586 ;; + *Pent*|*Celeron) UNAME_MACHINE=i686 ;; + esac + echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + exit ;; + i*86:*:3.2:*) + if test -f /usr/options/cb.name; then + UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name` + echo ${UNAME_MACHINE}-pc-isc$UNAME_REL + elif /bin/uname -X 2>/dev/null >/dev/null ; then + UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 + (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ + && UNAME_MACHINE=i586 + (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ + && UNAME_MACHINE=i686 + (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ + && UNAME_MACHINE=i686 + echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + else + echo ${UNAME_MACHINE}-pc-sysv32 + fi + exit ;; + pc:*:*:*) + # Left here for compatibility: + # uname -m prints for DJGPP always 'pc', but it prints nothing about + # the processor, so we play safe by assuming i386. + echo i386-pc-msdosdjgpp + exit ;; + Intel:Mach:3*:*) + echo i386-pc-mach3 + exit ;; + paragon:*:*:*) + echo i860-intel-osf1 + exit ;; + i860:*:4.*:*) # i860-SVR4 + if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then + echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + else # Add other i860-SVR4 vendors below as they are discovered. + echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + fi + exit ;; + mini*:CTIX:SYS*5:*) + # "miniframe" + echo m68010-convergent-sysv + exit ;; + mc68k:UNIX:SYSTEM5:3.51m) + echo m68k-convergent-sysv + exit ;; + M680?0:D-NIX:5.3:*) + echo m68k-diab-dnix + exit ;; + M68*:*:R3V[5678]*:*) + test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) + OS_REL='' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; + m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) + echo m68k-unknown-lynxos${UNAME_RELEASE} + exit ;; + mc68030:UNIX_System_V:4.*:*) + echo m68k-atari-sysv4 + exit ;; + TSUNAMI:LynxOS:2.*:*) + echo sparc-unknown-lynxos${UNAME_RELEASE} + exit ;; + rs6000:LynxOS:2.*:*) + echo rs6000-unknown-lynxos${UNAME_RELEASE} + exit ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*) + echo powerpc-unknown-lynxos${UNAME_RELEASE} + exit ;; + SM[BE]S:UNIX_SV:*:*) + echo mips-dde-sysv${UNAME_RELEASE} + exit ;; + RM*:ReliantUNIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + RM*:SINIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + *:SINIX-*:*:*) + if uname -p 2>/dev/null >/dev/null ; then + UNAME_MACHINE=`(uname -p) 2>/dev/null` + echo ${UNAME_MACHINE}-sni-sysv4 + else + echo ns32k-sni-sysv + fi + exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says <Richard.M.Bartel@ccMail.Census.GOV> + echo i586-unisys-sysv4 + exit ;; + *:UNIX_System_V:4*:FTX*) + # From Gerald Hewes <hewes@openmarket.com>. + # How about differentiating between stratus architectures? -djm + echo hppa1.1-stratus-sysv4 + exit ;; + *:*:*:FTX*) + # From seanf@swdc.stratus.com. + echo i860-stratus-sysv4 + exit ;; + i*86:VOS:*:*) + # From Paul.Green@stratus.com. + echo ${UNAME_MACHINE}-stratus-vos + exit ;; + *:VOS:*:*) + # From Paul.Green@stratus.com. + echo hppa1.1-stratus-vos + exit ;; + mc68*:A/UX:*:*) + echo m68k-apple-aux${UNAME_RELEASE} + exit ;; + news*:NEWS-OS:6*:*) + echo mips-sony-newsos6 + exit ;; + R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) + if [ -d /usr/nec ]; then + echo mips-nec-sysv${UNAME_RELEASE} + else + echo mips-unknown-sysv${UNAME_RELEASE} + fi + exit ;; + BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. + echo powerpc-be-beos + exit ;; + BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. + echo powerpc-apple-beos + exit ;; + BePC:BeOS:*:*) # BeOS running on Intel PC compatible. + echo i586-pc-beos + exit ;; + SX-4:SUPER-UX:*:*) + echo sx4-nec-superux${UNAME_RELEASE} + exit ;; + SX-5:SUPER-UX:*:*) + echo sx5-nec-superux${UNAME_RELEASE} + exit ;; + SX-6:SUPER-UX:*:*) + echo sx6-nec-superux${UNAME_RELEASE} + exit ;; + SX-7:SUPER-UX:*:*) + echo sx7-nec-superux${UNAME_RELEASE} + exit ;; + SX-8:SUPER-UX:*:*) + echo sx8-nec-superux${UNAME_RELEASE} + exit ;; + SX-8R:SUPER-UX:*:*) + echo sx8r-nec-superux${UNAME_RELEASE} + exit ;; + Power*:Rhapsody:*:*) + echo powerpc-apple-rhapsody${UNAME_RELEASE} + exit ;; + *:Rhapsody:*:*) + echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + exit ;; + *:Darwin:*:*) + UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown + case $UNAME_PROCESSOR in + unknown) UNAME_PROCESSOR=powerpc ;; + esac + echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + exit ;; + *:procnto*:*:* | *:QNX:[0123456789]*:*) + UNAME_PROCESSOR=`uname -p` + if test "$UNAME_PROCESSOR" = "x86"; then + UNAME_PROCESSOR=i386 + UNAME_MACHINE=pc + fi + echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + exit ;; + *:QNX:*:4*) + echo i386-pc-qnx + exit ;; + NSE-?:NONSTOP_KERNEL:*:*) + echo nse-tandem-nsk${UNAME_RELEASE} + exit ;; + NSR-?:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk${UNAME_RELEASE} + exit ;; + *:NonStop-UX:*:*) + echo mips-compaq-nonstopux + exit ;; + BS2000:POSIX*:*:*) + echo bs2000-siemens-sysv + exit ;; + DS/*:UNIX_System_V:*:*) + echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + exit ;; + *:Plan9:*:*) + # "uname -m" is not consistent, so use $cputype instead. 386 + # is converted to i386 for consistency with other x86 + # operating systems. + if test "$cputype" = "386"; then + UNAME_MACHINE=i386 + else + UNAME_MACHINE="$cputype" + fi + echo ${UNAME_MACHINE}-unknown-plan9 + exit ;; + *:TOPS-10:*:*) + echo pdp10-unknown-tops10 + exit ;; + *:TENEX:*:*) + echo pdp10-unknown-tenex + exit ;; + KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) + echo pdp10-dec-tops20 + exit ;; + XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) + echo pdp10-xkl-tops20 + exit ;; + *:TOPS-20:*:*) + echo pdp10-unknown-tops20 + exit ;; + *:ITS:*:*) + echo pdp10-unknown-its + exit ;; + SEI:*:*:SEIUX) + echo mips-sei-seiux${UNAME_RELEASE} + exit ;; + *:DragonFly:*:*) + echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + exit ;; + *:*VMS:*:*) + UNAME_MACHINE=`(uname -p) 2>/dev/null` + case "${UNAME_MACHINE}" in + A*) echo alpha-dec-vms ; exit ;; + I*) echo ia64-dec-vms ; exit ;; + V*) echo vax-dec-vms ; exit ;; + esac ;; + *:XENIX:*:SysV) + echo i386-pc-xenix + exit ;; + i*86:skyos:*:*) + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + exit ;; + i*86:rdos:*:*) + echo ${UNAME_MACHINE}-pc-rdos + exit ;; +esac + +#echo '(No uname command or uname output not recognized.)' 1>&2 +#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2 + +eval $set_cc_for_build +cat >$dummy.c <<EOF +#ifdef _SEQUENT_ +# include <sys/types.h> +# include <sys/utsname.h> +#endif +main () +{ +#if defined (sony) +#if defined (MIPSEB) + /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, + I don't know.... */ + printf ("mips-sony-bsd\n"); exit (0); +#else +#include <sys/param.h> + printf ("m68k-sony-newsos%s\n", +#ifdef NEWSOS4 + "4" +#else + "" +#endif + ); exit (0); +#endif +#endif + +#if defined (__arm) && defined (__acorn) && defined (__unix) + printf ("arm-acorn-riscix\n"); exit (0); +#endif + +#if defined (hp300) && !defined (hpux) + printf ("m68k-hp-bsd\n"); exit (0); +#endif + +#if defined (NeXT) +#if !defined (__ARCHITECTURE__) +#define __ARCHITECTURE__ "m68k" +#endif + int version; + version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; + if (version < 4) + printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); + else + printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); + exit (0); +#endif + +#if defined (MULTIMAX) || defined (n16) +#if defined (UMAXV) + printf ("ns32k-encore-sysv\n"); exit (0); +#else +#if defined (CMU) + printf ("ns32k-encore-mach\n"); exit (0); +#else + printf ("ns32k-encore-bsd\n"); exit (0); +#endif +#endif +#endif + +#if defined (__386BSD__) + printf ("i386-pc-bsd\n"); exit (0); +#endif + +#if defined (sequent) +#if defined (i386) + printf ("i386-sequent-dynix\n"); exit (0); +#endif +#if defined (ns32000) + printf ("ns32k-sequent-dynix\n"); exit (0); +#endif +#endif + +#if defined (_SEQUENT_) + struct utsname un; + + uname(&un); + + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); + +#endif + +#if defined (vax) +# if !defined (ultrix) +# include <sys/param.h> +# if defined (BSD) +# if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +# else +# if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +# else + printf ("vax-dec-bsd\n"); exit (0); +# endif +# endif +# else + printf ("vax-dec-bsd\n"); exit (0); +# endif +# else + printf ("vax-dec-ultrix\n"); exit (0); +# endif +#endif + +#if defined (alliant) && defined (i860) + printf ("i860-alliant-bsd\n"); exit (0); +#endif + + exit (1); +} +EOF + +$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } + +# Apollos put the system type in the environment. + +test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } + +# Convex versions that predate uname can use getsysinfo(1) + +if [ -x /usr/convex/getsysinfo ] +then + case `getsysinfo -f cpu_type` in + c1*) + echo c1-convex-bsd + exit ;; + c2*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit ;; + c34*) + echo c34-convex-bsd + exit ;; + c38*) + echo c38-convex-bsd + exit ;; + c4*) + echo c4-convex-bsd + exit ;; + esac +fi + +cat >&2 <<EOF +$0: unable to guess system type + +This script, last modified $timestamp, has failed to recognize +the operating system you are using. It is advised that you +download the most up to date version of the config scripts from + + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +and + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + +If the version you run ($0) is already up to date, please +send the following data and any information you think might be +pertinent to <config-patches@gnu.org> in order to provide the needed +information to handle your system. + +config.guess timestamp = $timestamp + +uname -m = `(uname -m) 2>/dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null` + +hostinfo = `(hostinfo) 2>/dev/null` +/bin/universe = `(/bin/universe) 2>/dev/null` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` +/bin/arch = `(/bin/arch) 2>/dev/null` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` + +UNAME_MACHINE = ${UNAME_MACHINE} +UNAME_RELEASE = ${UNAME_RELEASE} +UNAME_SYSTEM = ${UNAME_SYSTEM} +UNAME_VERSION = ${UNAME_VERSION} +EOF + +exit 1 + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/contrib/lmserver/config.h.in b/contrib/lmserver/config.h.in new file mode 100644 index 000000000..afdac98b6 --- /dev/null +++ b/contrib/lmserver/config.h.in @@ -0,0 +1,97 @@ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* machine is bigendian */ +#undef ENDIAN_BIG + +/* machine is littleendian */ +#undef ENDIAN_LITTLE + +/* Define this if you have daemon() */ +#undef HAVE_DAEMON + +/* Define to 1 if you have the `getpagesizes' function. */ +#undef HAVE_GETPAGESIZES + +/* Define to 1 if you have the <inttypes.h> header file. */ +#undef HAVE_INTTYPES_H + +/* do we have malloc.h? */ +#undef HAVE_MALLOC_H + +/* Define to 1 if you have the `memcntl' function. */ +#undef HAVE_MEMCNTL + +/* Define to 1 if you have the <memory.h> header file. */ +#undef HAVE_MEMORY_H + +/* Define to 1 if you have the `mlockall' function. */ +#undef HAVE_MLOCKALL + +/* flag for SRILM */ +#undef HAVE_SRILM + +/* Define to 1 if stdbool.h conforms to C99. */ +#undef HAVE_STDBOOL_H + +/* Define to 1 if you have the <stdint.h> header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if you have the <stdlib.h> header file. */ +#undef HAVE_STDLIB_H + +/* Define to 1 if you have the <strings.h> header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the <string.h> header file. */ +#undef HAVE_STRING_H + +/* do we have stuct mallinfo? */ +#undef HAVE_STRUCT_MALLINFO + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#undef HAVE_SYS_STAT_H + +/* Define to 1 if you have the <sys/types.h> header file. */ +#undef HAVE_SYS_TYPES_H + +/* Define to 1 if you have the <unistd.h> header file. */ +#undef HAVE_UNISTD_H + +/* Define to 1 if the system has the type `_Bool'. */ +#undef HAVE__BOOL + +/* Define to 1 if your C compiler doesn't accept -c and -o together. */ +#undef NO_MINUS_C_MINUS_O + +/* Name of package */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* Define to 1 if you have the ANSI C header files. */ +#undef STDC_HEADERS + +/* Define this if you want to use pthreads */ +#undef USE_THREADS + +/* Version number of package */ +#undef VERSION + +/* Define to empty if `const' does not conform to ANSI C. */ +#undef const + +/* define to int if socklen_t not available */ +#undef socklen_t diff --git a/contrib/lmserver/config.status b/contrib/lmserver/config.status new file mode 100755 index 000000000..490bcaf91 --- /dev/null +++ b/contrib/lmserver/config.status @@ -0,0 +1,1207 @@ +#! /bin/sh +# Generated by configure. +# Run this file to recreate the current configuration. +# Compiler output produced by configure, useful for debugging +# configure, is in config.log if it exists. + +debug=false +ac_cs_recheck=false +ac_cs_silent=false +SHELL=${CONFIG_SHELL-/bin/sh} +## --------------------- ## +## M4sh Initialization. ## +## --------------------- ## + +# Be Bourne compatible +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' +elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then + set -o posix +fi +DUALCASE=1; export DUALCASE # for MKS sh + +# Support unset when possible. +if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then + as_unset=unset +else + as_unset=false +fi + + +# Work around bugs in pre-3.0 UWIN ksh. +$as_unset ENV MAIL MAILPATH +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +for as_var in \ + LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \ + LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \ + LC_TELEPHONE LC_TIME +do + if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then + eval $as_var=C; export $as_var + else + $as_unset $as_var + fi +done + +# Required to use basename. +if expr a : '\(a\)' >/dev/null 2>&1; then + as_expr=expr +else + as_expr=false +fi + +if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + + +# Name of the executable. +as_me=`$as_basename "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)$' \| \ + . : '\(.\)' 2>/dev/null || +echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; } + /^X\/\(\/\/\)$/{ s//\1/; q; } + /^X\/\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + + +# PATH needs CR, and LINENO needs CR and PATH. +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + echo "#! /bin/sh" >conf$$.sh + echo "exit 0" >>conf$$.sh + chmod +x conf$$.sh + if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then + PATH_SEPARATOR=';' + else + PATH_SEPARATOR=: + fi + rm -f conf$$.sh +fi + + + as_lineno_1=$LINENO + as_lineno_2=$LINENO + as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x$as_lineno_3" = "x$as_lineno_2" || { + # Find who we are. Look in the path if we contain no path at all + # relative or not. + case $0 in + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break +done + + ;; + esac + # We did not find ourselves, most probably we were run as `sh COMMAND' + # in which case we are not to be found in the path. + if test "x$as_myself" = x; then + as_myself=$0 + fi + if test ! -f "$as_myself"; then + { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5 +echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;} + { (exit 1); exit 1; }; } + fi + case $CONFIG_SHELL in + '') + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for as_base in sh bash ksh sh5; do + case $as_dir in + /*) + if ("$as_dir/$as_base" -c ' + as_lineno_1=$LINENO + as_lineno_2=$LINENO + as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x$as_lineno_3" = "x$as_lineno_2" ') 2>/dev/null; then + $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; } + $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; } + CONFIG_SHELL=$as_dir/$as_base + export CONFIG_SHELL + exec "$CONFIG_SHELL" "$0" ${1+"$@"} + fi;; + esac + done +done +;; + esac + + # Create $as_me.lineno as a copy of $as_myself, but with $LINENO + # uniformly replaced by the line number. The first 'sed' inserts a + # line-number line before each line; the second 'sed' does the real + # work. The second script uses 'N' to pair each line-number line + # with the numbered line, and appends trailing '-' during + # substitution so that $LINENO is not a special case at line end. + # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the + # second 'sed' script. Blame Lee E. McMahon for sed's syntax. :-) + sed '=' <$as_myself | + sed ' + N + s,$,-, + : loop + s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3, + t loop + s,-$,, + s,^['$as_cr_digits']*\n,, + ' >$as_me.lineno && + chmod +x $as_me.lineno || + { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5 +echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;} + { (exit 1); exit 1; }; } + + # Don't try to exec as it changes $[0], causing all sort of problems + # (the dirname of $[0] is not the place where we might find the + # original and so on. Autoconf is especially sensible to this). + . ./$as_me.lineno + # Exit status is that of the last command. + exit +} + + +case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in + *c*,-n*) ECHO_N= ECHO_C=' +' ECHO_T=' ' ;; + *c*,* ) ECHO_N=-n ECHO_C= ECHO_T= ;; + *) ECHO_N= ECHO_C='\c' ECHO_T= ;; +esac + +if expr a : '\(a\)' >/dev/null 2>&1; then + as_expr=expr +else + as_expr=false +fi + +rm -f conf$$ conf$$.exe conf$$.file +echo >conf$$.file +if ln -s conf$$.file conf$$ 2>/dev/null; then + # We could just check for DJGPP; but this test a) works b) is more generic + # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04). + if test -f conf$$.exe; then + # Don't use ln at all; we don't have any links + as_ln_s='cp -p' + else + as_ln_s='ln -s' + fi +elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.file + +if mkdir -p . 2>/dev/null; then + as_mkdir_p=: +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +as_executable_p="test -f" + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + +# IFS +# We need space, tab and new line, in precisely that order. +as_nl=' +' +IFS=" $as_nl" + +# CDPATH. +$as_unset CDPATH + +exec 6>&1 + +# Open the log real soon, to keep \$[0] and so on meaningful, and to +# report actual input values of CONFIG_FILES etc. instead of their +# values after options handling. Logging --version etc. is OK. +exec 5>>config.log +{ + echo + sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX +## Running $as_me. ## +_ASBOX +} >&5 +cat >&5 <<_CSEOF + +This file was extended by lmserver $as_me 1.0, which was +generated by GNU Autoconf 2.59. Invocation command line was + + CONFIG_FILES = $CONFIG_FILES + CONFIG_HEADERS = $CONFIG_HEADERS + CONFIG_LINKS = $CONFIG_LINKS + CONFIG_COMMANDS = $CONFIG_COMMANDS + $ $0 $@ + +_CSEOF +echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5 +echo >&5 +config_files=" Makefile" +config_headers=" config.h" +config_commands=" depfiles" + +ac_cs_usage="\ +\`$as_me' instantiates files from templates according to the +current configuration. + +Usage: $0 [OPTIONS] [FILE]... + + -h, --help print this help, then exit + -V, --version print version number, then exit + -q, --quiet do not print progress messages + -d, --debug don't remove temporary files + --recheck update $as_me by reconfiguring in the same conditions + --file=FILE[:TEMPLATE] + instantiate the configuration file FILE + --header=FILE[:TEMPLATE] + instantiate the configuration header FILE + +Configuration files: +$config_files + +Configuration headers: +$config_headers + +Configuration commands: +$config_commands + +Report bugs to <bug-autoconf@gnu.org>." +ac_cs_version="\ +lmserver config.status 1.0 +configured by ./configure, generated by GNU Autoconf 2.59, + with options \"'--with-libevent=/fs/clip-software/libevent-1.4.8-stable' '--with-srilm=/fs/clip-software/srilm-1.5.6-PIC'\" + +Copyright (C) 2003 Free Software Foundation, Inc. +This config.status script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it." +srcdir=. +INSTALL="/usr/bin/install -c" +# If no file are specified by the user, then we need to provide default +# value. By we need to know if files were specified by the user. +ac_need_defaults=: +while test $# != 0 +do + case $1 in + --*=*) + ac_option=`expr "x$1" : 'x\([^=]*\)='` + ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'` + ac_shift=: + ;; + -*) + ac_option=$1 + ac_optarg=$2 + ac_shift=shift + ;; + *) # This is not an option, so the user has probably given explicit + # arguments. + ac_option=$1 + ac_need_defaults=false;; + esac + + case $ac_option in + # Handling of the options. + -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) + ac_cs_recheck=: ;; + --version | --vers* | -V ) + echo "$ac_cs_version"; exit 0 ;; + --he | --h) + # Conflict between --help and --header + { { echo "$as_me:$LINENO: error: ambiguous option: $1 +Try \`$0 --help' for more information." >&5 +echo "$as_me: error: ambiguous option: $1 +Try \`$0 --help' for more information." >&2;} + { (exit 1); exit 1; }; };; + --help | --hel | -h ) + echo "$ac_cs_usage"; exit 0 ;; + --debug | --d* | -d ) + debug=: ;; + --file | --fil | --fi | --f ) + $ac_shift + CONFIG_FILES="$CONFIG_FILES $ac_optarg" + ac_need_defaults=false;; + --header | --heade | --head | --hea ) + $ac_shift + CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg" + ac_need_defaults=false;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil | --si | --s) + ac_cs_silent=: ;; + + # This is an error. + -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1 +Try \`$0 --help' for more information." >&5 +echo "$as_me: error: unrecognized option: $1 +Try \`$0 --help' for more information." >&2;} + { (exit 1); exit 1; }; } ;; + + *) ac_config_targets="$ac_config_targets $1" ;; + + esac + shift +done + +ac_configure_extra_args= + +if $ac_cs_silent; then + exec 6>/dev/null + ac_configure_extra_args="$ac_configure_extra_args --silent" +fi + +if $ac_cs_recheck; then + echo "running /bin/sh ./configure " '--with-libevent=/fs/clip-software/libevent-1.4.8-stable' '--with-srilm=/fs/clip-software/srilm-1.5.6-PIC' $ac_configure_extra_args " --no-create --no-recursion" >&6 + exec /bin/sh ./configure '--with-libevent=/fs/clip-software/libevent-1.4.8-stable' '--with-srilm=/fs/clip-software/srilm-1.5.6-PIC' $ac_configure_extra_args --no-create --no-recursion +fi + +# +# INIT-COMMANDS section. +# + +AMDEP_TRUE="" ac_aux_dir="." + +for ac_config_target in $ac_config_targets +do + case "$ac_config_target" in + # Handling of arguments. + "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;; + "depfiles" ) CONFIG_COMMANDS="$CONFIG_COMMANDS depfiles" ;; + "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; + *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5 +echo "$as_me: error: invalid argument: $ac_config_target" >&2;} + { (exit 1); exit 1; }; };; + esac +done + +# If the user did not use the arguments to specify the items to instantiate, +# then the envvar interface is used. Set only those that are not. +# We use the long form for the default assignment because of an extremely +# bizarre bug on SunOS 4.1.3. +if $ac_need_defaults; then + test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files + test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers + test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands +fi + +# Have a temporary directory for convenience. Make it in the build tree +# simply because there is no reason to put it here, and in addition, +# creating and moving files from /tmp can sometimes cause problems. +# Create a temporary directory, and hook for its removal unless debugging. +$debug || +{ + trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0 + trap '{ (exit 1); exit 1; }' 1 2 13 15 +} + +# Create a (secure) tmp directory for tmp files. + +{ + tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` && + test -n "$tmp" && test -d "$tmp" +} || +{ + tmp=./confstat$$-$RANDOM + (umask 077 && mkdir $tmp) +} || +{ + echo "$me: cannot create a temporary directory in ." >&2 + { (exit 1); exit 1; } +} + + +# +# CONFIG_FILES section. +# + +# No need to generate the scripts if there are no CONFIG_FILES. +# This happens for instance when ./config.status config.h +if test -n "$CONFIG_FILES"; then + # Protect against being on the right side of a sed subst in config.status. + sed 's/,@/@@/; s/@,/@@/; s/,;t t$/@;t t/; /@;t t$/s/[\\&,]/\\&/g; + s/@@/,@/; s/@@/@,/; s/@;t t$/,;t t/' >$tmp/subs.sed <<\CEOF +s,@SHELL@,/bin/sh,;t t +s,@PATH_SEPARATOR@,:,;t t +s,@PACKAGE_NAME@,lmserver,;t t +s,@PACKAGE_TARNAME@,lmserver,;t t +s,@PACKAGE_VERSION@,1.0,;t t +s,@PACKAGE_STRING@,lmserver 1.0,;t t +s,@PACKAGE_BUGREPORT@,,;t t +s,@exec_prefix@,${prefix},;t t +s,@prefix@,/usr/local,;t t +s,@program_transform_name@,s,x,x,,;t t +s,@bindir@,${exec_prefix}/bin,;t t +s,@sbindir@,${exec_prefix}/sbin,;t t +s,@libexecdir@,${exec_prefix}/libexec,;t t +s,@datadir@,${prefix}/share,;t t +s,@sysconfdir@,${prefix}/etc,;t t +s,@sharedstatedir@,${prefix}/com,;t t +s,@localstatedir@,${prefix}/var,;t t +s,@libdir@,${exec_prefix}/lib,;t t +s,@includedir@,${prefix}/include,;t t +s,@oldincludedir@,/usr/include,;t t +s,@infodir@,${prefix}/info,;t t +s,@mandir@,${prefix}/man,;t t +s,@build_alias@,,;t t +s,@host_alias@,,;t t +s,@target_alias@,,;t t +s,@DEFS@,-DHAVE_CONFIG_H,;t t +s,@ECHO_C@,,;t t +s,@ECHO_N@,-n,;t t +s,@ECHO_T@,,;t t +s,@LIBS@, -loolm -ldstruct -lmisc -levent,;t t +s,@build@,x86_64-unknown-linux-gnu,;t t +s,@build_cpu@,x86_64,;t t +s,@build_vendor@,unknown,;t t +s,@build_os@,linux-gnu,;t t +s,@host@,x86_64-unknown-linux-gnu,;t t +s,@host_cpu@,x86_64,;t t +s,@host_vendor@,unknown,;t t +s,@host_os@,linux-gnu,;t t +s,@target@,x86_64-unknown-linux-gnu,;t t +s,@target_cpu@,x86_64,;t t +s,@target_vendor@,unknown,;t t +s,@target_os@,linux-gnu,;t t +s,@INSTALL_PROGRAM@,${INSTALL},;t t +s,@INSTALL_SCRIPT@,${INSTALL},;t t +s,@INSTALL_DATA@,${INSTALL} -m 644,;t t +s,@CYGPATH_W@,echo,;t t +s,@PACKAGE@,lmserver,;t t +s,@VERSION@,1.0,;t t +s,@ACLOCAL@,${SHELL} /chomes/redpony/moses/moses-arabic/lmserver/missing --run aclocal-1.9,;t t +s,@AUTOCONF@,${SHELL} /chomes/redpony/moses/moses-arabic/lmserver/missing --run autoconf,;t t +s,@AUTOMAKE@,${SHELL} /chomes/redpony/moses/moses-arabic/lmserver/missing --run automake-1.9,;t t +s,@AUTOHEADER@,${SHELL} /chomes/redpony/moses/moses-arabic/lmserver/missing --run autoheader,;t t +s,@MAKEINFO@,${SHELL} /chomes/redpony/moses/moses-arabic/lmserver/missing --run makeinfo,;t t +s,@install_sh@,/chomes/redpony/moses/moses-arabic/lmserver/install-sh,;t t +s,@STRIP@,,;t t +s,@ac_ct_STRIP@,,;t t +s,@INSTALL_STRIP_PROGRAM@,${SHELL} $(install_sh) -c -s,;t t +s,@mkdir_p@,mkdir -p --,;t t +s,@AWK@,gawk,;t t +s,@SET_MAKE@,,;t t +s,@am__leading_dot@,.,;t t +s,@AMTAR@,${SHELL} /chomes/redpony/moses/moses-arabic/lmserver/missing --run tar,;t t +s,@am__tar@,${AMTAR} chof - "$$tardir",;t t +s,@am__untar@,${AMTAR} xf -,;t t +s,@CC@,gcc,;t t +s,@CFLAGS@,-g -O2,;t t +s,@LDFLAGS@,-L/fs/clip-software/libevent-1.4.8-stable/lib -L/fs/clip-software/srilm-1.5.6-PIC/lib/i686,;t t +s,@CPPFLAGS@,-I/fs/clip-software/libevent-1.4.8-stable/include -I/fs/clip-software/srilm-1.5.6-PIC/include,;t t +s,@ac_ct_CC@,gcc,;t t +s,@EXEEXT@,,;t t +s,@OBJEXT@,o,;t t +s,@DEPDIR@,.deps,;t t +s,@am__include@,include,;t t +s,@am__quote@,,;t t +s,@AMDEP_TRUE@,,;t t +s,@AMDEP_FALSE@,#,;t t +s,@AMDEPBACKSLASH@,\,;t t +s,@CCDEPMODE@,depmode=gcc3,;t t +s,@am__fastdepCC_TRUE@,,;t t +s,@am__fastdepCC_FALSE@,#,;t t +s,@CXX@,g++,;t t +s,@CXXFLAGS@,-g -O2,;t t +s,@ac_ct_CXX@,g++,;t t +s,@CXXDEPMODE@,depmode=gcc3,;t t +s,@am__fastdepCXX_TRUE@,,;t t +s,@am__fastdepCXX_FALSE@,#,;t t +s,@CPP@,gcc -E,;t t +s,@EGREP@,grep -E,;t t +s,@SRI_LM_TRUE@,,;t t +s,@SRI_LM_FALSE@,#,;t t +s,@DAEMON_OBJ@,,;t t +s,@LIBOBJS@,,;t t +s,@LTLIBOBJS@,,;t t +CEOF + + # Split the substitutions into bite-sized pieces for seds with + # small command number limits, like on Digital OSF/1 and HP-UX. + ac_max_sed_lines=48 + ac_sed_frag=1 # Number of current file. + ac_beg=1 # First line for current file. + ac_end=$ac_max_sed_lines # Line after last line for current file. + ac_more_lines=: + ac_sed_cmds= + while $ac_more_lines; do + if test $ac_beg -gt 1; then + sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag + else + sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag + fi + if test ! -s $tmp/subs.frag; then + ac_more_lines=false + else + # The purpose of the label and of the branching condition is to + # speed up the sed processing (if there are no `@' at all, there + # is no need to browse any of the substitutions). + # These are the two extra sed commands mentioned above. + (echo ':t + /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed + if test -z "$ac_sed_cmds"; then + ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed" + else + ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed" + fi + ac_sed_frag=`expr $ac_sed_frag + 1` + ac_beg=$ac_end + ac_end=`expr $ac_end + $ac_max_sed_lines` + fi + done + if test -z "$ac_sed_cmds"; then + ac_sed_cmds=cat + fi +fi # test -n "$CONFIG_FILES" + +for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue + # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in". + case $ac_file in + - | *:- | *:-:* ) # input from stdin + cat >$tmp/stdin + ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + * ) ac_file_in=$ac_file.in ;; + esac + + # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories. + ac_dir=`(dirname "$ac_file") 2>/dev/null || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p "$ac_dir" + else + as_dir="$ac_dir" + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5 +echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;} + { (exit 1); exit 1; }; }; } + + ac_builddir=. + +if test "$ac_dir" != .; then + ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'` + # A "../" for each directory in $ac_dir_suffix. + ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'` +else + ac_dir_suffix= ac_top_builddir= +fi + +case $srcdir in + .) # No --srcdir option. We are building in place. + ac_srcdir=. + if test -z "$ac_top_builddir"; then + ac_top_srcdir=. + else + ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'` + fi ;; + [\\/]* | ?:[\\/]* ) # Absolute path. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir ;; + *) # Relative path. + ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_builddir$srcdir ;; +esac + +# Do not use `cd foo && pwd` to compute absolute paths, because +# the directories may not exist. +case `pwd` in +.) ac_abs_builddir="$ac_dir";; +*) + case "$ac_dir" in + .) ac_abs_builddir=`pwd`;; + [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";; + *) ac_abs_builddir=`pwd`/"$ac_dir";; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_builddir=${ac_top_builddir}.;; +*) + case ${ac_top_builddir}. in + .) ac_abs_top_builddir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;; + *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_srcdir=$ac_srcdir;; +*) + case $ac_srcdir in + .) ac_abs_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;; + *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_srcdir=$ac_top_srcdir;; +*) + case $ac_top_srcdir in + .) ac_abs_top_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;; + *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;; + esac;; +esac + + + case $INSTALL in + [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;; + *) ac_INSTALL=$ac_top_builddir$INSTALL ;; + esac + + if test x"$ac_file" != x-; then + { echo "$as_me:$LINENO: creating $ac_file" >&5 +echo "$as_me: creating $ac_file" >&6;} + rm -f "$ac_file" + fi + # Let's still pretend it is `configure' which instantiates (i.e., don't + # use $as_me), people would be surprised to read: + # /* config.h. Generated by config.status. */ + if test x"$ac_file" = x-; then + configure_input= + else + configure_input="$ac_file. " + fi + configure_input=$configure_input"Generated from `echo $ac_file_in | + sed 's,.*/,,'` by configure." + + # First look for the input files in the build tree, otherwise in the + # src tree. + ac_file_inputs=`IFS=: + for f in $ac_file_in; do + case $f in + -) echo $tmp/stdin ;; + [\\/$]*) + # Absolute (can't be DOS-style, as IFS=:) + test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + echo "$f";; + *) # Relative + if test -f "$f"; then + # Build tree + echo "$f" + elif test -f "$srcdir/$f"; then + # Source tree + echo "$srcdir/$f" + else + # /dev/null tree + { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + fi;; + esac + done` || { (exit 1); exit 1; } + sed "/^[ ]*VPATH[ ]*=/{ +s/:*\$(srcdir):*/:/; +s/:*\${srcdir}:*/:/; +s/:*@srcdir@:*/:/; +s/^\([^=]*=[ ]*\):*/\1/; +s/:*$//; +s/^[^=]*=[ ]*$//; +} + +:t +/@[a-zA-Z_][a-zA-Z_0-9]*@/!b +s,@configure_input@,$configure_input,;t t +s,@srcdir@,$ac_srcdir,;t t +s,@abs_srcdir@,$ac_abs_srcdir,;t t +s,@top_srcdir@,$ac_top_srcdir,;t t +s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t +s,@builddir@,$ac_builddir,;t t +s,@abs_builddir@,$ac_abs_builddir,;t t +s,@top_builddir@,$ac_top_builddir,;t t +s,@abs_top_builddir@,$ac_abs_top_builddir,;t t +s,@INSTALL@,$ac_INSTALL,;t t +" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out + rm -f $tmp/stdin + if test x"$ac_file" != x-; then + mv $tmp/out $ac_file + else + cat $tmp/out + rm -f $tmp/out + fi + +done + +# +# CONFIG_HEADER section. +# + +# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where +# NAME is the cpp macro being defined and VALUE is the value it is being given. +# +# ac_d sets the value in "#define NAME VALUE" lines. +ac_dA='s,^\([ ]*\)#\([ ]*define[ ][ ]*\)' +ac_dB='[ ].*$,\1#\2' +ac_dC=' ' +ac_dD=',;t' +# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE". +ac_uA='s,^\([ ]*\)#\([ ]*\)undef\([ ][ ]*\)' +ac_uB='$,\1#\2define\3' +ac_uC=' ' +ac_uD=',;t' + +for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue + # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in". + case $ac_file in + - | *:- | *:-:* ) # input from stdin + cat >$tmp/stdin + ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + * ) ac_file_in=$ac_file.in ;; + esac + + test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5 +echo "$as_me: creating $ac_file" >&6;} + + # First look for the input files in the build tree, otherwise in the + # src tree. + ac_file_inputs=`IFS=: + for f in $ac_file_in; do + case $f in + -) echo $tmp/stdin ;; + [\\/$]*) + # Absolute (can't be DOS-style, as IFS=:) + test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + # Do quote $f, to prevent DOS paths from being IFS'd. + echo "$f";; + *) # Relative + if test -f "$f"; then + # Build tree + echo "$f" + elif test -f "$srcdir/$f"; then + # Source tree + echo "$srcdir/$f" + else + # /dev/null tree + { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + fi;; + esac + done` || { (exit 1); exit 1; } + # Remove the trailing spaces. + sed 's/[ ]*$//' $ac_file_inputs >$tmp/in + + # Handle all the #define templates only if necessary. + if grep "^[ ]*#[ ]*define" $tmp/in >/dev/null; then + # If there are no defines, we may have an empty if/fi + : + cat >$tmp/defines.sed <<CEOF +/^[ ]*#[ ]*define/!b +t clr +: clr +${ac_dA}PACKAGE_NAME${ac_dB}PACKAGE_NAME${ac_dC}"lmserver"${ac_dD} +${ac_dA}PACKAGE_TARNAME${ac_dB}PACKAGE_TARNAME${ac_dC}"lmserver"${ac_dD} +${ac_dA}PACKAGE_VERSION${ac_dB}PACKAGE_VERSION${ac_dC}"1.0"${ac_dD} +${ac_dA}PACKAGE_STRING${ac_dB}PACKAGE_STRING${ac_dC}"lmserver 1.0"${ac_dD} +${ac_dA}PACKAGE_BUGREPORT${ac_dB}PACKAGE_BUGREPORT${ac_dC}""${ac_dD} +${ac_dA}PACKAGE${ac_dB}PACKAGE${ac_dC}"lmserver"${ac_dD} +${ac_dA}VERSION${ac_dB}VERSION${ac_dC}"1.0"${ac_dD} +${ac_dA}STDC_HEADERS${ac_dB}STDC_HEADERS${ac_dC}1${ac_dD} +${ac_dA}HAVE_SYS_TYPES_H${ac_dB}HAVE_SYS_TYPES_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_SYS_STAT_H${ac_dB}HAVE_SYS_STAT_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_STDLIB_H${ac_dB}HAVE_STDLIB_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_STRING_H${ac_dB}HAVE_STRING_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_MEMORY_H${ac_dB}HAVE_MEMORY_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_STRINGS_H${ac_dB}HAVE_STRINGS_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_INTTYPES_H${ac_dB}HAVE_INTTYPES_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_STDINT_H${ac_dB}HAVE_STDINT_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_UNISTD_H${ac_dB}HAVE_UNISTD_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_SRILM${ac_dB}HAVE_SRILM${ac_dC}${ac_dD} +${ac_dA}HAVE_DAEMON${ac_dB}HAVE_DAEMON${ac_dC}${ac_dD} +${ac_dA}HAVE__BOOL${ac_dB}HAVE__BOOL${ac_dC}1${ac_dD} +${ac_dA}HAVE_STDBOOL_H${ac_dB}HAVE_STDBOOL_H${ac_dC}1${ac_dD} +${ac_dA}HAVE_MALLOC_H${ac_dB}HAVE_MALLOC_H${ac_dC}${ac_dD} +${ac_dA}HAVE_STRUCT_MALLINFO${ac_dB}HAVE_STRUCT_MALLINFO${ac_dC}${ac_dD} +${ac_dA}ENDIAN_LITTLE${ac_dB}ENDIAN_LITTLE${ac_dC}1${ac_dD} +${ac_dA}HAVE_MLOCKALL${ac_dB}HAVE_MLOCKALL${ac_dC}1${ac_dD} +CEOF + sed -f $tmp/defines.sed $tmp/in >$tmp/out + rm -f $tmp/in + mv $tmp/out $tmp/in + + fi # grep + + # Handle all the #undef templates + cat >$tmp/undefs.sed <<CEOF +/^[ ]*#[ ]*undef/!b +t clr +: clr +${ac_uA}PACKAGE_NAME${ac_uB}PACKAGE_NAME${ac_uC}"lmserver"${ac_uD} +${ac_uA}PACKAGE_TARNAME${ac_uB}PACKAGE_TARNAME${ac_uC}"lmserver"${ac_uD} +${ac_uA}PACKAGE_VERSION${ac_uB}PACKAGE_VERSION${ac_uC}"1.0"${ac_uD} +${ac_uA}PACKAGE_STRING${ac_uB}PACKAGE_STRING${ac_uC}"lmserver 1.0"${ac_uD} +${ac_uA}PACKAGE_BUGREPORT${ac_uB}PACKAGE_BUGREPORT${ac_uC}""${ac_uD} +${ac_uA}PACKAGE${ac_uB}PACKAGE${ac_uC}"lmserver"${ac_uD} +${ac_uA}VERSION${ac_uB}VERSION${ac_uC}"1.0"${ac_uD} +${ac_uA}STDC_HEADERS${ac_uB}STDC_HEADERS${ac_uC}1${ac_uD} +${ac_uA}HAVE_SYS_TYPES_H${ac_uB}HAVE_SYS_TYPES_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_SYS_STAT_H${ac_uB}HAVE_SYS_STAT_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_STDLIB_H${ac_uB}HAVE_STDLIB_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_STRING_H${ac_uB}HAVE_STRING_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_MEMORY_H${ac_uB}HAVE_MEMORY_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_STRINGS_H${ac_uB}HAVE_STRINGS_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_INTTYPES_H${ac_uB}HAVE_INTTYPES_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_STDINT_H${ac_uB}HAVE_STDINT_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_UNISTD_H${ac_uB}HAVE_UNISTD_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_SRILM${ac_uB}HAVE_SRILM${ac_uC}${ac_uD} +${ac_uA}HAVE_DAEMON${ac_uB}HAVE_DAEMON${ac_uC}${ac_uD} +${ac_uA}HAVE__BOOL${ac_uB}HAVE__BOOL${ac_uC}1${ac_uD} +${ac_uA}HAVE_STDBOOL_H${ac_uB}HAVE_STDBOOL_H${ac_uC}1${ac_uD} +${ac_uA}HAVE_MALLOC_H${ac_uB}HAVE_MALLOC_H${ac_uC}${ac_uD} +${ac_uA}HAVE_STRUCT_MALLINFO${ac_uB}HAVE_STRUCT_MALLINFO${ac_uC}${ac_uD} +${ac_uA}ENDIAN_LITTLE${ac_uB}ENDIAN_LITTLE${ac_uC}1${ac_uD} +${ac_uA}HAVE_MLOCKALL${ac_uB}HAVE_MLOCKALL${ac_uC}1${ac_uD} +s,^[ ]*#[ ]*undef[ ][ ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */, +CEOF + sed -f $tmp/undefs.sed $tmp/in >$tmp/out + rm -f $tmp/in + mv $tmp/out $tmp/in + + # Let's still pretend it is `configure' which instantiates (i.e., don't + # use $as_me), people would be surprised to read: + # /* config.h. Generated by config.status. */ + if test x"$ac_file" = x-; then + echo "/* Generated by configure. */" >$tmp/config.h + else + echo "/* $ac_file. Generated by configure. */" >$tmp/config.h + fi + cat $tmp/in >>$tmp/config.h + rm -f $tmp/in + if test x"$ac_file" != x-; then + if diff $ac_file $tmp/config.h >/dev/null 2>&1; then + { echo "$as_me:$LINENO: $ac_file is unchanged" >&5 +echo "$as_me: $ac_file is unchanged" >&6;} + else + ac_dir=`(dirname "$ac_file") 2>/dev/null || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p "$ac_dir" + else + as_dir="$ac_dir" + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5 +echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;} + { (exit 1); exit 1; }; }; } + + rm -f $ac_file + mv $tmp/config.h $ac_file + fi + else + cat $tmp/config.h + rm -f $tmp/config.h + fi +# Compute $ac_file's index in $config_headers. +_am_stamp_count=1 +for _am_header in $config_headers :; do + case $_am_header in + $ac_file | $ac_file:* ) + break ;; + * ) + _am_stamp_count=`expr $_am_stamp_count + 1` ;; + esac +done +echo "timestamp for $ac_file" >`(dirname $ac_file) 2>/dev/null || +$as_expr X$ac_file : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X$ac_file : 'X\(//\)[^/]' \| \ + X$ac_file : 'X\(//\)$' \| \ + X$ac_file : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X$ac_file | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'`/stamp-h$_am_stamp_count +done + +# +# CONFIG_COMMANDS section. +# +for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue + ac_dest=`echo "$ac_file" | sed 's,:.*,,'` + ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_dir=`(dirname "$ac_dest") 2>/dev/null || +$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_dest" : 'X\(//\)[^/]' \| \ + X"$ac_dest" : 'X\(//\)$' \| \ + X"$ac_dest" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$ac_dest" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p "$ac_dir" + else + as_dir="$ac_dir" + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5 +echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;} + { (exit 1); exit 1; }; }; } + + ac_builddir=. + +if test "$ac_dir" != .; then + ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'` + # A "../" for each directory in $ac_dir_suffix. + ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'` +else + ac_dir_suffix= ac_top_builddir= +fi + +case $srcdir in + .) # No --srcdir option. We are building in place. + ac_srcdir=. + if test -z "$ac_top_builddir"; then + ac_top_srcdir=. + else + ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'` + fi ;; + [\\/]* | ?:[\\/]* ) # Absolute path. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir ;; + *) # Relative path. + ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_builddir$srcdir ;; +esac + +# Do not use `cd foo && pwd` to compute absolute paths, because +# the directories may not exist. +case `pwd` in +.) ac_abs_builddir="$ac_dir";; +*) + case "$ac_dir" in + .) ac_abs_builddir=`pwd`;; + [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";; + *) ac_abs_builddir=`pwd`/"$ac_dir";; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_builddir=${ac_top_builddir}.;; +*) + case ${ac_top_builddir}. in + .) ac_abs_top_builddir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;; + *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_srcdir=$ac_srcdir;; +*) + case $ac_srcdir in + .) ac_abs_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;; + *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_srcdir=$ac_top_srcdir;; +*) + case $ac_top_srcdir in + .) ac_abs_top_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;; + *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;; + esac;; +esac + + + { echo "$as_me:$LINENO: executing $ac_dest commands" >&5 +echo "$as_me: executing $ac_dest commands" >&6;} + case $ac_dest in + depfiles ) test x"$AMDEP_TRUE" != x"" || for mf in $CONFIG_FILES; do + # Strip MF so we end up with the name of the file. + mf=`echo "$mf" | sed -e 's/:.*$//'` + # Check whether this is an Automake generated Makefile or not. + # We used to match only the files named `Makefile.in', but + # some people rename them; so instead we look at the file content. + # Grep'ing the first line is not enough: some people post-process + # each Makefile.in and add a new line on top of each file to say so. + # So let's grep whole file. + if grep '^#.*generated by automake' $mf > /dev/null 2>&1; then + dirpart=`(dirname "$mf") 2>/dev/null || +$as_expr X"$mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$mf" : 'X\(//\)[^/]' \| \ + X"$mf" : 'X\(//\)$' \| \ + X"$mf" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$mf" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + else + continue + fi + # Extract the definition of DEPDIR, am__include, and am__quote + # from the Makefile without running `make'. + DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` + test -z "$DEPDIR" && continue + am__include=`sed -n 's/^am__include = //p' < "$mf"` + test -z "am__include" && continue + am__quote=`sed -n 's/^am__quote = //p' < "$mf"` + # When using ansi2knr, U may be empty or an underscore; expand it + U=`sed -n 's/^U = //p' < "$mf"` + # Find all dependency output files, they are included files with + # $(DEPDIR) in their names. We invoke sed twice because it is the + # simplest approach to changing $(DEPDIR) to its actual value in the + # expansion. + for file in `sed -n " + s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ + sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do + # Make sure the directory exists. + test -f "$dirpart/$file" && continue + fdir=`(dirname "$file") 2>/dev/null || +$as_expr X"$file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$file" : 'X\(//\)[^/]' \| \ + X"$file" : 'X\(//\)$' \| \ + X"$file" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p $dirpart/$fdir + else + as_dir=$dirpart/$fdir + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory $dirpart/$fdir" >&5 +echo "$as_me: error: cannot create directory $dirpart/$fdir" >&2;} + { (exit 1); exit 1; }; }; } + + # echo "creating $dirpart/$file" + echo '# dummy' > "$dirpart/$file" + done +done + ;; + esac +done + +{ (exit 0); exit 0; } diff --git a/contrib/lmserver/config.sub b/contrib/lmserver/config.sub new file mode 100755 index 000000000..ba16ebf55 --- /dev/null +++ b/contrib/lmserver/config.sub @@ -0,0 +1,1676 @@ +#! /bin/sh +# Configuration validation subroutine script. +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, +# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 +# Free Software Foundation, Inc. + +timestamp='2008-01-16' + +# This file is (in principle) common to ALL GNU software. +# The presence of a machine in this file suggests that SOME GNU software +# can handle that machine. It does not imply ALL GNU software can. +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA +# 02110-1301, USA. +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + + +# Please send patches to <config-patches@gnu.org>. Submit a context +# diff and a properly formatted ChangeLog entry. +# +# Configuration subroutine to validate and canonicalize a configuration type. +# Supply the specified configuration type as an argument. +# If it is invalid, we print an error message on stderr and exit with code 1. +# Otherwise, we print the canonical config type on stdout and succeed. + +# This file is supposed to be the same for all GNU packages +# and recognize all the CPU types, system types and aliases +# that are meaningful with *any* GNU software. +# Each package is responsible for reporting which valid configurations +# it does not support. The user should be able to distinguish +# a failure to support a valid configuration from a meaningless +# configuration. + +# The goal of this file is to map all the various variations of a given +# machine specification into a single specification in the form: +# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM +# or in some cases, the newer four-part form: +# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM +# It is wrong to echo any other type of specification. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] CPU-MFR-OPSYS + $0 [OPTION] ALIAS + +Canonicalize a configuration name. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to <config-patches@gnu.org>." + +version="\ +GNU config.sub ($timestamp) + +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, +2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" + exit 1 ;; + + *local*) + # First pass through any local machine types. + echo $1 + exit ;; + + * ) + break ;; + esac +done + +case $# in + 0) echo "$me: missing argument$help" >&2 + exit 1;; + 1) ;; + *) echo "$me: too many arguments$help" >&2 + exit 1;; +esac + +# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). +# Here we must recognize all the valid KERNEL-OS combinations. +maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` +case $maybe_os in + nto-qnx* | linux-gnu* | linux-dietlibc | linux-newlib* | linux-uclibc* | \ + uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | \ + storm-chaos* | os2-emx* | rtmk-nova*) + os=-$maybe_os + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` + ;; + *) + basic_machine=`echo $1 | sed 's/-[^-]*$//'` + if [ $basic_machine != $1 ] + then os=`echo $1 | sed 's/.*-/-/'` + else os=; fi + ;; +esac + +### Let's recognize common machines as not being operating systems so +### that things like config.sub decstation-3100 work. We also +### recognize some manufacturers as not being operating systems, so we +### can provide default operating systems below. +case $os in + -sun*os*) + # Prevent following clause from handling this invalid input. + ;; + -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ + -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ + -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ + -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ + -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ + -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ + -apple | -axis | -knuth | -cray) + os= + basic_machine=$1 + ;; + -sim | -cisco | -oki | -wec | -winbond) + os= + basic_machine=$1 + ;; + -scout) + ;; + -wrs) + os=-vxworks + basic_machine=$1 + ;; + -chorusos*) + os=-chorusos + basic_machine=$1 + ;; + -chorusrdb) + os=-chorusrdb + basic_machine=$1 + ;; + -hiux*) + os=-hiuxwe2 + ;; + -sco6) + os=-sco5v6 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco5) + os=-sco3.2v5 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco4) + os=-sco3.2v4 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2v[4-9]*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco*) + os=-sco3.2v2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -udk*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -isc) + os=-isc2.2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -clix*) + basic_machine=clipper-intergraph + ;; + -isc*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -lynx*) + os=-lynxos + ;; + -ptx*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + ;; + -windowsnt*) + os=`echo $os | sed -e 's/windowsnt/winnt/'` + ;; + -psos*) + os=-psos + ;; + -mint | -mint[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Recognize the basic CPU types without company name. + # Some are omitted here because they have special meanings below. + 1750a | 580 \ + | a29k \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ + | am33_2.0 \ + | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \ + | bfin \ + | c4x | clipper \ + | d10v | d30v | dlx | dsp16xx | dvp \ + | fido | fr30 | frv \ + | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | i370 | i860 | i960 | ia64 \ + | ip2k | iq2000 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | maxq | mb | microblaze | mcore | mep \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64el \ + | mips64vr | mips64vrel \ + | mips64orion | mips64orionel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipstx39 | mipstx39el \ + | mn10200 | mn10300 \ + | mt \ + | msp430 \ + | nios | nios2 \ + | ns16k | ns32k \ + | or32 \ + | pdp10 | pdp11 | pj | pjl \ + | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \ + | pyramid \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]a*eb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu | strongarm \ + | tahoe | thumb | tic4x | tic80 | tron \ + | v850 | v850e \ + | we32k \ + | x86 | xc16x | xscale | xscalee[bl] | xstormy16 | xtensa \ + | z8k) + basic_machine=$basic_machine-unknown + ;; + m6811 | m68hc11 | m6812 | m68hc12) + # Motorola 68HC11/12. + basic_machine=$basic_machine-unknown + os=-none + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + ;; + ms1) + basic_machine=mt-unknown + ;; + + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + basic_machine=$basic_machine-pc + ;; + # Object if more than one company name word. + *-*-*) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; + # Recognize the basic CPU types with company name. + 580-* \ + | a29k-* \ + | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ + | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ + | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ + | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ + | avr-* | avr32-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \ + | clipper-* | craynv-* | cydra-* \ + | d10v-* | d30v-* | dlx-* \ + | elxsi-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ + | h8300-* | h8500-* \ + | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | i*86-* | i860-* | i960-* | ia64-* \ + | ip2k-* | iq2000-* \ + | m32c-* | m32r-* | m32rle-* \ + | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ + | m88110-* | m88k-* | maxq-* | mcore-* \ + | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ + | mips16-* \ + | mips64-* | mips64el-* \ + | mips64vr-* | mips64vrel-* \ + | mips64orion-* | mips64orionel-* \ + | mips64vr4100-* | mips64vr4100el-* \ + | mips64vr4300-* | mips64vr4300el-* \ + | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ + | mipsisa32-* | mipsisa32el-* \ + | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa64-* | mipsisa64el-* \ + | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64sb1-* | mipsisa64sb1el-* \ + | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mt-* \ + | msp430-* \ + | nios-* | nios2-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | orion-* \ + | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \ + | pyramid-* \ + | romp-* | rs6000-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]a*eb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ + | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | strongarm-* | sv1-* | sx?-* \ + | tahoe-* | thumb-* \ + | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tron-* \ + | v850-* | v850e-* | vax-* \ + | we32k-* \ + | x86-* | x86_64-* | xc16x-* | xps100-* | xscale-* | xscalee[bl]-* \ + | xstormy16-* | xtensa*-* \ + | ymp-* \ + | z8k-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown + ;; + # Recognize the various machine names and aliases which stand + # for a CPU type and a company and sometimes even an OS. + 386bsd) + basic_machine=i386-unknown + os=-bsd + ;; + 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) + basic_machine=m68000-att + ;; + 3b*) + basic_machine=we32k-att + ;; + a29khif) + basic_machine=a29k-amd + os=-udi + ;; + abacus) + basic_machine=abacus-unknown + ;; + adobe68k) + basic_machine=m68010-adobe + os=-scout + ;; + alliant | fx80) + basic_machine=fx80-alliant + ;; + altos | altos3068) + basic_machine=m68k-altos + ;; + am29k) + basic_machine=a29k-none + os=-bsd + ;; + amd64) + basic_machine=x86_64-pc + ;; + amd64-*) + basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + amdahl) + basic_machine=580-amdahl + os=-sysv + ;; + amiga | amiga-*) + basic_machine=m68k-unknown + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=-amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=-sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=-sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=-bsd + ;; + aux) + basic_machine=m68k-apple + os=-aux + ;; + balance) + basic_machine=ns32k-sequent + os=-dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=-linux + ;; + blackfin-*) + basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + c90) + basic_machine=c90-cray + os=-unicos + ;; + convex-c1) + basic_machine=c1-convex + os=-bsd + ;; + convex-c2) + basic_machine=c2-convex + os=-bsd + ;; + convex-c32) + basic_machine=c32-convex + os=-bsd + ;; + convex-c34) + basic_machine=c34-convex + os=-bsd + ;; + convex-c38) + basic_machine=c38-convex + os=-bsd + ;; + cray | j90) + basic_machine=j90-cray + os=-unicos + ;; + craynv) + basic_machine=craynv-cray + os=-unicosmp + ;; + cr16) + basic_machine=cr16-unknown + os=-elf + ;; + crds | unos) + basic_machine=m68k-crds + ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; + cris | cris-* | etrax*) + basic_machine=cris-axis + ;; + crx) + basic_machine=crx-unknown + os=-elf + ;; + da30 | da30-*) + basic_machine=m68k-da30 + ;; + decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) + basic_machine=mips-dec + ;; + decsystem10* | dec10*) + basic_machine=pdp10-dec + os=-tops10 + ;; + decsystem20* | dec20*) + basic_machine=pdp10-dec + os=-tops20 + ;; + delta | 3300 | motorola-3300 | motorola-delta \ + | 3300-motorola | delta-motorola) + basic_machine=m68k-motorola + ;; + delta88) + basic_machine=m88k-motorola + os=-sysv3 + ;; + djgpp) + basic_machine=i586-pc + os=-msdosdjgpp + ;; + dpx20 | dpx20-*) + basic_machine=rs6000-bull + os=-bosx + ;; + dpx2* | dpx2*-bull) + basic_machine=m68k-bull + os=-sysv3 + ;; + ebmon29k) + basic_machine=a29k-amd + os=-ebmon + ;; + elxsi) + basic_machine=elxsi-elxsi + os=-bsd + ;; + encore | umax | mmax) + basic_machine=ns32k-encore + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=-ose + ;; + fx2800) + basic_machine=i860-alliant + ;; + genix) + basic_machine=ns32k-ns + ;; + gmicro) + basic_machine=tron-gmicro + os=-sysv + ;; + go32) + basic_machine=i386-pc + os=-go32 + ;; + h3050r* | hiux*) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=-hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=-xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=-hms + ;; + harris) + basic_machine=m88k-harris + os=-sysv3 + ;; + hp300-*) + basic_machine=m68k-hp + ;; + hp300bsd) + basic_machine=m68k-hp + os=-bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=-hpux + ;; + hp3k9[0-9][0-9] | hp9[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k2[0-9][0-9] | hp9k31[0-9]) + basic_machine=m68000-hp + ;; + hp9k3[2-9][0-9]) + basic_machine=m68k-hp + ;; + hp9k6[0-9][0-9] | hp6[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k7[0-79][0-9] | hp7[0-79][0-9]) + basic_machine=hppa1.1-hp + ;; + hp9k78[0-9] | hp78[0-9]) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][13679] | hp8[0-9][13679]) + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][0-9] | hp8[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hppa-next) + os=-nextstep3 + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=-osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=-proelf + ;; + i370-ibm* | ibm*) + basic_machine=i370-ibm + ;; +# I'm not sure what "Sysv32" means. Should this be sysv3.2? + i*86v32) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv32 + ;; + i*86v4*) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv4 + ;; + i*86v) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv + ;; + i*86sol2) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-solaris2 + ;; + i386mach) + basic_machine=i386-mach + os=-mach + ;; + i386-vsta | vsta) + basic_machine=i386-unknown + os=-vsta + ;; + iris | iris4d) + basic_machine=mips-sgi + case $os in + -irix*) + ;; + *) + os=-irix4 + ;; + esac + ;; + isi68 | isi) + basic_machine=m68k-isi + os=-sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=-linux + ;; + m68knommu-*) + basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + m88k-omron*) + basic_machine=m88k-omron + ;; + magnum | m3230) + basic_machine=mips-mips + os=-sysv + ;; + merlin) + basic_machine=ns32k-utek + os=-sysv + ;; + mingw32) + basic_machine=i386-pc + os=-mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=-mingw32ce + ;; + miniframe) + basic_machine=m68000-convergent + ;; + *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; + mipsEE* | ee | ps2) + basic_machine=mips64r5900el-scei + case $os in + -linux*) + ;; + *) + os=-elf + ;; + esac + ;; + iop) + basic_machine=mipsel-scei + os=-irx + ;; + dvp) + basic_machine=dvp-scei + os=-elf + ;; + mips3*-*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` + ;; + mips3*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown + ;; + monitor) + basic_machine=m68k-rom68k + os=-coff + ;; + morphos) + basic_machine=powerpc-unknown + os=-morphos + ;; + msdos) + basic_machine=i386-pc + os=-msdos + ;; + ms1-*) + basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` + ;; + mvs) + basic_machine=i370-ibm + os=-mvs + ;; + ncr3000) + basic_machine=i486-ncr + os=-sysv4 + ;; + netbsd386) + basic_machine=i386-unknown + os=-netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=-linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=-newsos + ;; + news1000) + basic_machine=m68030-sony + os=-newsos + ;; + news-3600 | risc-news) + basic_machine=mips-sony + os=-newsos + ;; + necv70) + basic_machine=v70-nec + os=-sysv + ;; + next | m*-next ) + basic_machine=m68k-next + case $os in + -nextstep* ) + ;; + -ns2*) + os=-nextstep2 + ;; + *) + os=-nextstep3 + ;; + esac + ;; + nh3000) + basic_machine=m68k-harris + os=-cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=-cxux + ;; + nindy960) + basic_machine=i960-intel + os=-nindy + ;; + mon960) + basic_machine=i960-intel + os=-mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=-nonstopux + ;; + np1) + basic_machine=np1-gould + ;; + nsr-tandem) + basic_machine=nsr-tandem + ;; + op50n-* | op60c-*) + basic_machine=hppa1.1-oki + os=-proelf + ;; + openrisc | openrisc-*) + basic_machine=or32-unknown + ;; + os400) + basic_machine=powerpc-ibm + os=-os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=-ose + ;; + os68k) + basic_machine=m68k-none + os=-os68k + ;; + pa-hitachi) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + paragon) + basic_machine=i860-intel + os=-osf + ;; + parisc) + basic_machine=hppa-unknown + os=-linux + ;; + parisc-*) + basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + pbd) + basic_machine=sparc-tti + ;; + pbb) + basic_machine=m68k-tti + ;; + pc532 | pc532-*) + basic_machine=ns32k-pc532 + ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentium | p5 | k5 | k6 | nexgen | viac3) + basic_machine=i586-pc + ;; + pentiumpro | p6 | 6x86 | athlon | athlon_*) + basic_machine=i686-pc + ;; + pentiumii | pentium2 | pentiumiii | pentium3) + basic_machine=i686-pc + ;; + pentium4) + basic_machine=i786-pc + ;; + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumpro-* | p6-* | 6x86-* | athlon-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentium4-*) + basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pn) + basic_machine=pn-gould + ;; + power) basic_machine=power-ibm + ;; + ppc) basic_machine=powerpc-unknown + ;; + ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppcle | powerpclittle | ppc-le | powerpc-little) + basic_machine=powerpcle-unknown + ;; + ppcle-* | powerpclittle-*) + basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64) basic_machine=powerpc64-unknown + ;; + ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64le | powerpc64little | ppc64-le | powerpc64-little) + basic_machine=powerpc64le-unknown + ;; + ppc64le-* | powerpc64little-*) + basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ps2) + basic_machine=i386-ibm + ;; + pw32) + basic_machine=i586-unknown + os=-pw32 + ;; + rdos) + basic_machine=i386-pc + os=-rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=-coff + ;; + rm[46]00) + basic_machine=mips-siemens + ;; + rtpc | rtpc-*) + basic_machine=romp-ibm + ;; + s390 | s390-*) + basic_machine=s390-ibm + ;; + s390x | s390x-*) + basic_machine=s390x-ibm + ;; + sa29200) + basic_machine=a29k-amd + os=-udi + ;; + sb1) + basic_machine=mipsisa64sb1-unknown + ;; + sb1el) + basic_machine=mipsisa64sb1el-unknown + ;; + sde) + basic_machine=mipsisa32-sde + os=-elf + ;; + sei) + basic_machine=mips-sei + os=-seiux + ;; + sequent) + basic_machine=i386-sequent + ;; + sh) + basic_machine=sh-hitachi + os=-hms + ;; + sh5el) + basic_machine=sh5le-unknown + ;; + sh64) + basic_machine=sh64-unknown + ;; + sparclite-wrs | simso-wrs) + basic_machine=sparclite-wrs + os=-vxworks + ;; + sps7) + basic_machine=m68k-bull + os=-sysv2 + ;; + spur) + basic_machine=spur-unknown + ;; + st2000) + basic_machine=m68k-tandem + ;; + stratus) + basic_machine=i860-stratus + os=-sysv4 + ;; + sun2) + basic_machine=m68000-sun + ;; + sun2os3) + basic_machine=m68000-sun + os=-sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=-sunos4 + ;; + sun3os3) + basic_machine=m68k-sun + os=-sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=-sunos4 + ;; + sun4os3) + basic_machine=sparc-sun + os=-sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=-sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=-solaris2 + ;; + sun3 | sun3-*) + basic_machine=m68k-sun + ;; + sun4) + basic_machine=sparc-sun + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + ;; + sv1) + basic_machine=sv1-cray + os=-unicos + ;; + symmetry) + basic_machine=i386-sequent + os=-dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=-unicos + ;; + t90) + basic_machine=t90-cray + os=-unicos + ;; + tic54x | c54x*) + basic_machine=tic54x-unknown + os=-coff + ;; + tic55x | c55x*) + basic_machine=tic55x-unknown + os=-coff + ;; + tic6x | c6x*) + basic_machine=tic6x-unknown + os=-coff + ;; + tile*) + basic_machine=tile-unknown + os=-linux-gnu + ;; + tx39) + basic_machine=mipstx39-unknown + ;; + tx39el) + basic_machine=mipstx39el-unknown + ;; + toad1) + basic_machine=pdp10-xkl + os=-tops20 + ;; + tower | tower-32) + basic_machine=m68k-ncr + ;; + tpf) + basic_machine=s390x-ibm + os=-tpf + ;; + udi29k) + basic_machine=a29k-amd + os=-udi + ;; + ultra3) + basic_machine=a29k-nyu + os=-sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=-none + ;; + vaxv) + basic_machine=vax-dec + os=-sysv + ;; + vms) + basic_machine=vax-dec + os=-vms + ;; + vpp*|vx|vx-*) + basic_machine=f301-fujitsu + ;; + vxworks960) + basic_machine=i960-wrs + os=-vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=-vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=-vxworks + ;; + w65*) + basic_machine=w65-wdc + os=-none + ;; + w89k-*) + basic_machine=hppa1.1-winbond + os=-proelf + ;; + xbox) + basic_machine=i686-pc + os=-mingw32 + ;; + xps | xps100) + basic_machine=xps100-honeywell + ;; + ymp) + basic_machine=ymp-cray + os=-unicos + ;; + z8k-*-coff) + basic_machine=z8k-unknown + os=-sim + ;; + none) + basic_machine=none-none + os=-none + ;; + +# Here we handle the default manufacturer of certain CPU types. It is in +# some cases the only manufacturer, in others, it is the most popular. + w89k) + basic_machine=hppa1.1-winbond + ;; + op50n) + basic_machine=hppa1.1-oki + ;; + op60c) + basic_machine=hppa1.1-oki + ;; + romp) + basic_machine=romp-ibm + ;; + mmix) + basic_machine=mmix-knuth + ;; + rs6000) + basic_machine=rs6000-ibm + ;; + vax) + basic_machine=vax-dec + ;; + pdp10) + # there are many clones, so DEC is not a safe bet + basic_machine=pdp10-unknown + ;; + pdp11) + basic_machine=pdp11-dec + ;; + we32k) + basic_machine=we32k-att + ;; + sh[1234] | sh[24]a | sh[34]eb | sh[1234]le | sh[23]ele) + basic_machine=sh-unknown + ;; + sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) + basic_machine=sparc-sun + ;; + cydra) + basic_machine=cydra-cydrome + ;; + orion) + basic_machine=orion-highlevel + ;; + orion105) + basic_machine=clipper-highlevel + ;; + mac | mpw | mac-mpw) + basic_machine=m68k-apple + ;; + pmac | pmac-mpw) + basic_machine=powerpc-apple + ;; + *-unknown) + # Make sure to match an already-canonicalized machine name. + ;; + *) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; +esac + +# Here we canonicalize certain aliases for manufacturers. +case $basic_machine in + *-digital*) + basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` + ;; + *-commodore*) + basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + ;; + *) + ;; +esac + +# Decode manufacturer-specific aliases for certain operating systems. + +if [ x"$os" != x"" ] +then +case $os in + # First match some system type aliases + # that might get confused with valid system types. + # -solaris* is a basic system type, with this one exception. + -solaris1 | -solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` + ;; + -solaris) + os=-solaris2 + ;; + -svr4*) + os=-sysv4 + ;; + -unixware*) + os=-sysv4.2uw + ;; + -gnu/linux*) + os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` + ;; + # First accept the basic system types. + # The portable systems comes first. + # Each alternative MUST END IN A *, to match a version number. + # -sysv* is not here because it comes later, after sysvr4. + -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ + | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\ + | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \ + | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ + | -aos* \ + | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ + | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ + | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ + | -openbsd* | -solidbsd* \ + | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ + | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ + | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ + | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ + | -chorusos* | -chorusrdb* \ + | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ + | -mingw32* | -linux-gnu* | -linux-newlib* | -linux-uclibc* \ + | -uxpv* | -beos* | -mpeix* | -udk* \ + | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ + | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ + | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ + | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ + | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ + | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -irx*) + # Remember, each alternative MUST END IN *, to match a version number. + ;; + -qnx*) + case $basic_machine in + x86-* | i*86-*) + ;; + *) + os=-nto$os + ;; + esac + ;; + -nto-qnx*) + ;; + -nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` + ;; + -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ + | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ + | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + ;; + -mac*) + os=`echo $os | sed -e 's|mac|macos|'` + ;; + -linux-dietlibc) + os=-linux-dietlibc + ;; + -linux*) + os=`echo $os | sed -e 's|linux|linux-gnu|'` + ;; + -sunos5*) + os=`echo $os | sed -e 's|sunos5|solaris2|'` + ;; + -sunos6*) + os=`echo $os | sed -e 's|sunos6|solaris3|'` + ;; + -opened*) + os=-openedition + ;; + -os400*) + os=-os400 + ;; + -wince*) + os=-wince + ;; + -osfrose*) + os=-osfrose + ;; + -osf*) + os=-osf + ;; + -utek*) + os=-bsd + ;; + -dynix*) + os=-bsd + ;; + -acis*) + os=-aos + ;; + -atheos*) + os=-atheos + ;; + -syllable*) + os=-syllable + ;; + -386bsd) + os=-bsd + ;; + -ctix* | -uts*) + os=-sysv + ;; + -nova*) + os=-rtmk-nova + ;; + -ns2 ) + os=-nextstep2 + ;; + -nsk*) + os=-nsk + ;; + # Preserve the version number of sinix5. + -sinix5.*) + os=`echo $os | sed -e 's|sinix|sysv|'` + ;; + -sinix*) + os=-sysv4 + ;; + -tpf*) + os=-tpf + ;; + -triton*) + os=-sysv3 + ;; + -oss*) + os=-sysv3 + ;; + -svr4) + os=-sysv4 + ;; + -svr3) + os=-sysv3 + ;; + -sysvr4) + os=-sysv4 + ;; + # This must come after -sysvr4. + -sysv*) + ;; + -ose*) + os=-ose + ;; + -es1800*) + os=-ose + ;; + -xenix) + os=-xenix + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + os=-mint + ;; + -aros*) + os=-aros + ;; + -kaos*) + os=-kaos + ;; + -zvmoe) + os=-zvmoe + ;; + -none) + ;; + *) + # Get rid of the `-' at the beginning of $os. + os=`echo $os | sed 's/[^-]*-//'` + echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + exit 1 + ;; +esac +else + +# Here we handle the default operating systems that come with various machines. +# The value should be what the vendor currently ships out the door with their +# machine or put another way, the most popular os provided with the machine. + +# Note that if you're going to try to match "-MANUFACTURER" here (say, +# "-sun"), then you have to tell the case statement up towards the top +# that MANUFACTURER isn't an operating system. Otherwise, code above +# will signal an error saying that MANUFACTURER isn't an operating +# system, and we'll never get to this point. + +case $basic_machine in + score-*) + os=-elf + ;; + spu-*) + os=-elf + ;; + *-acorn) + os=-riscix1.2 + ;; + arm*-rebel) + os=-linux + ;; + arm*-semi) + os=-aout + ;; + c4x-* | tic4x-*) + os=-coff + ;; + # This must come before the *-dec entry. + pdp10-*) + os=-tops20 + ;; + pdp11-*) + os=-none + ;; + *-dec | vax-*) + os=-ultrix4.2 + ;; + m68*-apollo) + os=-domain + ;; + i386-sun) + os=-sunos4.0.2 + ;; + m68000-sun) + os=-sunos3 + # This also exists in the configure program, but was not the + # default. + # os=-sunos4 + ;; + m68*-cisco) + os=-aout + ;; + mep-*) + os=-elf + ;; + mips*-cisco) + os=-elf + ;; + mips*-*) + os=-elf + ;; + or32-*) + os=-coff + ;; + *-tti) # must be before sparc entry or we get the wrong os. + os=-sysv3 + ;; + sparc-* | *-sun) + os=-sunos4.1.1 + ;; + *-be) + os=-beos + ;; + *-haiku) + os=-haiku + ;; + *-ibm) + os=-aix + ;; + *-knuth) + os=-mmixware + ;; + *-wec) + os=-proelf + ;; + *-winbond) + os=-proelf + ;; + *-oki) + os=-proelf + ;; + *-hp) + os=-hpux + ;; + *-hitachi) + os=-hiux + ;; + i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) + os=-sysv + ;; + *-cbm) + os=-amigaos + ;; + *-dg) + os=-dgux + ;; + *-dolphin) + os=-sysv3 + ;; + m68k-ccur) + os=-rtu + ;; + m88k-omron*) + os=-luna + ;; + *-next ) + os=-nextstep + ;; + *-sequent) + os=-ptx + ;; + *-crds) + os=-unos + ;; + *-ns) + os=-genix + ;; + i370-*) + os=-mvs + ;; + *-next) + os=-nextstep3 + ;; + *-gould) + os=-sysv + ;; + *-highlevel) + os=-bsd + ;; + *-encore) + os=-bsd + ;; + *-sgi) + os=-irix + ;; + *-siemens) + os=-sysv4 + ;; + *-masscomp) + os=-rtu + ;; + f30[01]-fujitsu | f700-fujitsu) + os=-uxpv + ;; + *-rom68k) + os=-coff + ;; + *-*bug) + os=-coff + ;; + *-apple) + os=-macos + ;; + *-atari*) + os=-mint + ;; + *) + os=-none + ;; +esac +fi + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +vendor=unknown +case $basic_machine in + *-unknown) + case $os in + -riscix*) + vendor=acorn + ;; + -sunos*) + vendor=sun + ;; + -aix*) + vendor=ibm + ;; + -beos*) + vendor=be + ;; + -hpux*) + vendor=hp + ;; + -mpeix*) + vendor=hp + ;; + -hiux*) + vendor=hitachi + ;; + -unos*) + vendor=crds + ;; + -dgux*) + vendor=dg + ;; + -luna*) + vendor=omron + ;; + -genix*) + vendor=ns + ;; + -mvs* | -opened*) + vendor=ibm + ;; + -os400*) + vendor=ibm + ;; + -ptx*) + vendor=sequent + ;; + -tpf*) + vendor=ibm + ;; + -vxsim* | -vxworks* | -windiss*) + vendor=wrs + ;; + -aux*) + vendor=apple + ;; + -hms*) + vendor=hitachi + ;; + -mpw* | -macos*) + vendor=apple + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + vendor=atari + ;; + -vos*) + vendor=stratus + ;; + esac + basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` + ;; +esac + +echo $basic_machine$os +exit + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/contrib/lmserver/configure b/contrib/lmserver/configure new file mode 100755 index 000000000..69a1a6f02 --- /dev/null +++ b/contrib/lmserver/configure @@ -0,0 +1,7585 @@ +#! /bin/sh +# Guess values for system-dependent variables and create Makefiles. +# Generated by GNU Autoconf 2.59 for lmserver 1.0. +# +# Copyright (C) 2003 Free Software Foundation, Inc. +# This configure script is free software; the Free Software Foundation +# gives unlimited permission to copy, distribute and modify it. +## --------------------- ## +## M4sh Initialization. ## +## --------------------- ## + +# Be Bourne compatible +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' +elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then + set -o posix +fi +DUALCASE=1; export DUALCASE # for MKS sh + +# Support unset when possible. +if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then + as_unset=unset +else + as_unset=false +fi + + +# Work around bugs in pre-3.0 UWIN ksh. +$as_unset ENV MAIL MAILPATH +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +for as_var in \ + LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \ + LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \ + LC_TELEPHONE LC_TIME +do + if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then + eval $as_var=C; export $as_var + else + $as_unset $as_var + fi +done + +# Required to use basename. +if expr a : '\(a\)' >/dev/null 2>&1; then + as_expr=expr +else + as_expr=false +fi + +if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + + +# Name of the executable. +as_me=`$as_basename "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)$' \| \ + . : '\(.\)' 2>/dev/null || +echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; } + /^X\/\(\/\/\)$/{ s//\1/; q; } + /^X\/\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + + +# PATH needs CR, and LINENO needs CR and PATH. +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + echo "#! /bin/sh" >conf$$.sh + echo "exit 0" >>conf$$.sh + chmod +x conf$$.sh + if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then + PATH_SEPARATOR=';' + else + PATH_SEPARATOR=: + fi + rm -f conf$$.sh +fi + + + as_lineno_1=$LINENO + as_lineno_2=$LINENO + as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x$as_lineno_3" = "x$as_lineno_2" || { + # Find who we are. Look in the path if we contain no path at all + # relative or not. + case $0 in + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break +done + + ;; + esac + # We did not find ourselves, most probably we were run as `sh COMMAND' + # in which case we are not to be found in the path. + if test "x$as_myself" = x; then + as_myself=$0 + fi + if test ! -f "$as_myself"; then + { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2 + { (exit 1); exit 1; }; } + fi + case $CONFIG_SHELL in + '') + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for as_base in sh bash ksh sh5; do + case $as_dir in + /*) + if ("$as_dir/$as_base" -c ' + as_lineno_1=$LINENO + as_lineno_2=$LINENO + as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x$as_lineno_3" = "x$as_lineno_2" ') 2>/dev/null; then + $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; } + $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; } + CONFIG_SHELL=$as_dir/$as_base + export CONFIG_SHELL + exec "$CONFIG_SHELL" "$0" ${1+"$@"} + fi;; + esac + done +done +;; + esac + + # Create $as_me.lineno as a copy of $as_myself, but with $LINENO + # uniformly replaced by the line number. The first 'sed' inserts a + # line-number line before each line; the second 'sed' does the real + # work. The second script uses 'N' to pair each line-number line + # with the numbered line, and appends trailing '-' during + # substitution so that $LINENO is not a special case at line end. + # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the + # second 'sed' script. Blame Lee E. McMahon for sed's syntax. :-) + sed '=' <$as_myself | + sed ' + N + s,$,-, + : loop + s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3, + t loop + s,-$,, + s,^['$as_cr_digits']*\n,, + ' >$as_me.lineno && + chmod +x $as_me.lineno || + { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2 + { (exit 1); exit 1; }; } + + # Don't try to exec as it changes $[0], causing all sort of problems + # (the dirname of $[0] is not the place where we might find the + # original and so on. Autoconf is especially sensible to this). + . ./$as_me.lineno + # Exit status is that of the last command. + exit +} + + +case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in + *c*,-n*) ECHO_N= ECHO_C=' +' ECHO_T=' ' ;; + *c*,* ) ECHO_N=-n ECHO_C= ECHO_T= ;; + *) ECHO_N= ECHO_C='\c' ECHO_T= ;; +esac + +if expr a : '\(a\)' >/dev/null 2>&1; then + as_expr=expr +else + as_expr=false +fi + +rm -f conf$$ conf$$.exe conf$$.file +echo >conf$$.file +if ln -s conf$$.file conf$$ 2>/dev/null; then + # We could just check for DJGPP; but this test a) works b) is more generic + # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04). + if test -f conf$$.exe; then + # Don't use ln at all; we don't have any links + as_ln_s='cp -p' + else + as_ln_s='ln -s' + fi +elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.file + +if mkdir -p . 2>/dev/null; then + as_mkdir_p=: +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +as_executable_p="test -f" + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + +# IFS +# We need space, tab and new line, in precisely that order. +as_nl=' +' +IFS=" $as_nl" + +# CDPATH. +$as_unset CDPATH + + +# Name of the host. +# hostname on some systems (SVR3.2, Linux) returns a bogus exit status, +# so uname gets run too. +ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` + +exec 6>&1 + +# +# Initializations. +# +ac_default_prefix=/usr/local +ac_config_libobj_dir=. +cross_compiling=no +subdirs= +MFLAGS= +MAKEFLAGS= +SHELL=${CONFIG_SHELL-/bin/sh} + +# Maximum number of lines to put in a shell here document. +# This variable seems obsolete. It should probably be removed, and +# only ac_max_sed_lines should be used. +: ${ac_max_here_lines=38} + +# Identity of this package. +PACKAGE_NAME='lmserver' +PACKAGE_TARNAME='lmserver' +PACKAGE_VERSION='1.0' +PACKAGE_STRING='lmserver 1.0' +PACKAGE_BUGREPORT='' + +ac_unique_file="lmserver.c" +# Factoring default headers for most tests. +ac_includes_default="\ +#include <stdio.h> +#if HAVE_SYS_TYPES_H +# include <sys/types.h> +#endif +#if HAVE_SYS_STAT_H +# include <sys/stat.h> +#endif +#if STDC_HEADERS +# include <stdlib.h> +# include <stddef.h> +#else +# if HAVE_STDLIB_H +# include <stdlib.h> +# endif +#endif +#if HAVE_STRING_H +# if !STDC_HEADERS && HAVE_MEMORY_H +# include <memory.h> +# endif +# include <string.h> +#endif +#if HAVE_STRINGS_H +# include <strings.h> +#endif +#if HAVE_INTTYPES_H +# include <inttypes.h> +#else +# if HAVE_STDINT_H +# include <stdint.h> +# endif +#endif +#if HAVE_UNISTD_H +# include <unistd.h> +#endif" + +ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS build build_cpu build_vendor build_os host host_cpu host_vendor host_os target target_cpu target_vendor target_os INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA CYGPATH_W PACKAGE VERSION ACLOCAL AUTOCONF AUTOMAKE AUTOHEADER MAKEINFO install_sh STRIP ac_ct_STRIP INSTALL_STRIP_PROGRAM mkdir_p AWK SET_MAKE am__leading_dot AMTAR am__tar am__untar CC CFLAGS LDFLAGS CPPFLAGS ac_ct_CC EXEEXT OBJEXT DEPDIR am__include am__quote AMDEP_TRUE AMDEP_FALSE AMDEPBACKSLASH CCDEPMODE am__fastdepCC_TRUE am__fastdepCC_FALSE CXX CXXFLAGS ac_ct_CXX CXXDEPMODE am__fastdepCXX_TRUE am__fastdepCXX_FALSE CPP EGREP SRI_LM_TRUE SRI_LM_FALSE DAEMON_OBJ LIBOBJS LTLIBOBJS' +ac_subst_files='' + +# Initialize some variables set by options. +ac_init_help= +ac_init_version=false +# The variables have the same names as the options, with +# dashes changed to underlines. +cache_file=/dev/null +exec_prefix=NONE +no_create= +no_recursion= +prefix=NONE +program_prefix=NONE +program_suffix=NONE +program_transform_name=s,x,x, +silent= +site= +srcdir= +verbose= +x_includes=NONE +x_libraries=NONE + +# Installation directory options. +# These are left unexpanded so users can "make install exec_prefix=/foo" +# and all the variables that are supposed to be based on exec_prefix +# by default will actually change. +# Use braces instead of parens because sh, perl, etc. also accept them. +bindir='${exec_prefix}/bin' +sbindir='${exec_prefix}/sbin' +libexecdir='${exec_prefix}/libexec' +datadir='${prefix}/share' +sysconfdir='${prefix}/etc' +sharedstatedir='${prefix}/com' +localstatedir='${prefix}/var' +libdir='${exec_prefix}/lib' +includedir='${prefix}/include' +oldincludedir='/usr/include' +infodir='${prefix}/info' +mandir='${prefix}/man' + +ac_prev= +for ac_option +do + # If the previous option needs an argument, assign it. + if test -n "$ac_prev"; then + eval "$ac_prev=\$ac_option" + ac_prev= + continue + fi + + ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'` + + # Accept the important Cygnus configure options, so we can diagnose typos. + + case $ac_option in + + -bindir | --bindir | --bindi | --bind | --bin | --bi) + ac_prev=bindir ;; + -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) + bindir=$ac_optarg ;; + + -build | --build | --buil | --bui | --bu) + ac_prev=build_alias ;; + -build=* | --build=* | --buil=* | --bui=* | --bu=*) + build_alias=$ac_optarg ;; + + -cache-file | --cache-file | --cache-fil | --cache-fi \ + | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) + ac_prev=cache_file ;; + -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ + | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) + cache_file=$ac_optarg ;; + + --config-cache | -C) + cache_file=config.cache ;; + + -datadir | --datadir | --datadi | --datad | --data | --dat | --da) + ac_prev=datadir ;; + -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \ + | --da=*) + datadir=$ac_optarg ;; + + -disable-* | --disable-*) + ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null && + { echo "$as_me: error: invalid feature name: $ac_feature" >&2 + { (exit 1); exit 1; }; } + ac_feature=`echo $ac_feature | sed 's/-/_/g'` + eval "enable_$ac_feature=no" ;; + + -enable-* | --enable-*) + ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null && + { echo "$as_me: error: invalid feature name: $ac_feature" >&2 + { (exit 1); exit 1; }; } + ac_feature=`echo $ac_feature | sed 's/-/_/g'` + case $ac_option in + *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;; + *) ac_optarg=yes ;; + esac + eval "enable_$ac_feature='$ac_optarg'" ;; + + -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ + | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ + | --exec | --exe | --ex) + ac_prev=exec_prefix ;; + -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ + | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ + | --exec=* | --exe=* | --ex=*) + exec_prefix=$ac_optarg ;; + + -gas | --gas | --ga | --g) + # Obsolete; use --with-gas. + with_gas=yes ;; + + -help | --help | --hel | --he | -h) + ac_init_help=long ;; + -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) + ac_init_help=recursive ;; + -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) + ac_init_help=short ;; + + -host | --host | --hos | --ho) + ac_prev=host_alias ;; + -host=* | --host=* | --hos=* | --ho=*) + host_alias=$ac_optarg ;; + + -includedir | --includedir | --includedi | --included | --include \ + | --includ | --inclu | --incl | --inc) + ac_prev=includedir ;; + -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ + | --includ=* | --inclu=* | --incl=* | --inc=*) + includedir=$ac_optarg ;; + + -infodir | --infodir | --infodi | --infod | --info | --inf) + ac_prev=infodir ;; + -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) + infodir=$ac_optarg ;; + + -libdir | --libdir | --libdi | --libd) + ac_prev=libdir ;; + -libdir=* | --libdir=* | --libdi=* | --libd=*) + libdir=$ac_optarg ;; + + -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ + | --libexe | --libex | --libe) + ac_prev=libexecdir ;; + -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ + | --libexe=* | --libex=* | --libe=*) + libexecdir=$ac_optarg ;; + + -localstatedir | --localstatedir | --localstatedi | --localstated \ + | --localstate | --localstat | --localsta | --localst \ + | --locals | --local | --loca | --loc | --lo) + ac_prev=localstatedir ;; + -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ + | --localstate=* | --localstat=* | --localsta=* | --localst=* \ + | --locals=* | --local=* | --loca=* | --loc=* | --lo=*) + localstatedir=$ac_optarg ;; + + -mandir | --mandir | --mandi | --mand | --man | --ma | --m) + ac_prev=mandir ;; + -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) + mandir=$ac_optarg ;; + + -nfp | --nfp | --nf) + # Obsolete; use --without-fp. + with_fp=no ;; + + -no-create | --no-create | --no-creat | --no-crea | --no-cre \ + | --no-cr | --no-c | -n) + no_create=yes ;; + + -no-recursion | --no-recursion | --no-recursio | --no-recursi \ + | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) + no_recursion=yes ;; + + -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ + | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ + | --oldin | --oldi | --old | --ol | --o) + ac_prev=oldincludedir ;; + -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ + | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ + | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) + oldincludedir=$ac_optarg ;; + + -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) + ac_prev=prefix ;; + -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) + prefix=$ac_optarg ;; + + -program-prefix | --program-prefix | --program-prefi | --program-pref \ + | --program-pre | --program-pr | --program-p) + ac_prev=program_prefix ;; + -program-prefix=* | --program-prefix=* | --program-prefi=* \ + | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) + program_prefix=$ac_optarg ;; + + -program-suffix | --program-suffix | --program-suffi | --program-suff \ + | --program-suf | --program-su | --program-s) + ac_prev=program_suffix ;; + -program-suffix=* | --program-suffix=* | --program-suffi=* \ + | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) + program_suffix=$ac_optarg ;; + + -program-transform-name | --program-transform-name \ + | --program-transform-nam | --program-transform-na \ + | --program-transform-n | --program-transform- \ + | --program-transform | --program-transfor \ + | --program-transfo | --program-transf \ + | --program-trans | --program-tran \ + | --progr-tra | --program-tr | --program-t) + ac_prev=program_transform_name ;; + -program-transform-name=* | --program-transform-name=* \ + | --program-transform-nam=* | --program-transform-na=* \ + | --program-transform-n=* | --program-transform-=* \ + | --program-transform=* | --program-transfor=* \ + | --program-transfo=* | --program-transf=* \ + | --program-trans=* | --program-tran=* \ + | --progr-tra=* | --program-tr=* | --program-t=*) + program_transform_name=$ac_optarg ;; + + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + silent=yes ;; + + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) + ac_prev=sbindir ;; + -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ + | --sbi=* | --sb=*) + sbindir=$ac_optarg ;; + + -sharedstatedir | --sharedstatedir | --sharedstatedi \ + | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ + | --sharedst | --shareds | --shared | --share | --shar \ + | --sha | --sh) + ac_prev=sharedstatedir ;; + -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ + | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ + | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ + | --sha=* | --sh=*) + sharedstatedir=$ac_optarg ;; + + -site | --site | --sit) + ac_prev=site ;; + -site=* | --site=* | --sit=*) + site=$ac_optarg ;; + + -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) + ac_prev=srcdir ;; + -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) + srcdir=$ac_optarg ;; + + -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ + | --syscon | --sysco | --sysc | --sys | --sy) + ac_prev=sysconfdir ;; + -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ + | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) + sysconfdir=$ac_optarg ;; + + -target | --target | --targe | --targ | --tar | --ta | --t) + ac_prev=target_alias ;; + -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) + target_alias=$ac_optarg ;; + + -v | -verbose | --verbose | --verbos | --verbo | --verb) + verbose=yes ;; + + -version | --version | --versio | --versi | --vers | -V) + ac_init_version=: ;; + + -with-* | --with-*) + ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null && + { echo "$as_me: error: invalid package name: $ac_package" >&2 + { (exit 1); exit 1; }; } + ac_package=`echo $ac_package| sed 's/-/_/g'` + case $ac_option in + *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;; + *) ac_optarg=yes ;; + esac + eval "with_$ac_package='$ac_optarg'" ;; + + -without-* | --without-*) + ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null && + { echo "$as_me: error: invalid package name: $ac_package" >&2 + { (exit 1); exit 1; }; } + ac_package=`echo $ac_package | sed 's/-/_/g'` + eval "with_$ac_package=no" ;; + + --x) + # Obsolete; use --with-x. + with_x=yes ;; + + -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ + | --x-incl | --x-inc | --x-in | --x-i) + ac_prev=x_includes ;; + -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ + | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) + x_includes=$ac_optarg ;; + + -x-libraries | --x-libraries | --x-librarie | --x-librari \ + | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) + ac_prev=x_libraries ;; + -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ + | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) + x_libraries=$ac_optarg ;; + + -*) { echo "$as_me: error: unrecognized option: $ac_option +Try \`$0 --help' for more information." >&2 + { (exit 1); exit 1; }; } + ;; + + *=*) + ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` + # Reject names that are not valid shell variable names. + expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null && + { echo "$as_me: error: invalid variable name: $ac_envvar" >&2 + { (exit 1); exit 1; }; } + ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` + eval "$ac_envvar='$ac_optarg'" + export $ac_envvar ;; + + *) + # FIXME: should be removed in autoconf 3.0. + echo "$as_me: WARNING: you should use --build, --host, --target" >&2 + expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && + echo "$as_me: WARNING: invalid host type: $ac_option" >&2 + : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option} + ;; + + esac +done + +if test -n "$ac_prev"; then + ac_option=--`echo $ac_prev | sed 's/_/-/g'` + { echo "$as_me: error: missing argument to $ac_option" >&2 + { (exit 1); exit 1; }; } +fi + +# Be sure to have absolute paths. +for ac_var in exec_prefix prefix +do + eval ac_val=$`echo $ac_var` + case $ac_val in + [\\/$]* | ?:[\\/]* | NONE | '' ) ;; + *) { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2 + { (exit 1); exit 1; }; };; + esac +done + +# Be sure to have absolute paths. +for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \ + localstatedir libdir includedir oldincludedir infodir mandir +do + eval ac_val=$`echo $ac_var` + case $ac_val in + [\\/$]* | ?:[\\/]* ) ;; + *) { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2 + { (exit 1); exit 1; }; };; + esac +done + +# There might be people who depend on the old broken behavior: `$host' +# used to hold the argument of --host etc. +# FIXME: To remove some day. +build=$build_alias +host=$host_alias +target=$target_alias + +# FIXME: To remove some day. +if test "x$host_alias" != x; then + if test "x$build_alias" = x; then + cross_compiling=maybe + echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host. + If a cross compiler is detected then cross compile mode will be used." >&2 + elif test "x$build_alias" != "x$host_alias"; then + cross_compiling=yes + fi +fi + +ac_tool_prefix= +test -n "$host_alias" && ac_tool_prefix=$host_alias- + +test "$silent" = yes && exec 6>/dev/null + + +# Find the source files, if location was not specified. +if test -z "$srcdir"; then + ac_srcdir_defaulted=yes + # Try the directory containing this script, then its parent. + ac_confdir=`(dirname "$0") 2>/dev/null || +$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$0" : 'X\(//\)[^/]' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$0" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + srcdir=$ac_confdir + if test ! -r $srcdir/$ac_unique_file; then + srcdir=.. + fi +else + ac_srcdir_defaulted=no +fi +if test ! -r $srcdir/$ac_unique_file; then + if test "$ac_srcdir_defaulted" = yes; then + { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2 + { (exit 1); exit 1; }; } + else + { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2 + { (exit 1); exit 1; }; } + fi +fi +(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null || + { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2 + { (exit 1); exit 1; }; } +srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'` +ac_env_build_alias_set=${build_alias+set} +ac_env_build_alias_value=$build_alias +ac_cv_env_build_alias_set=${build_alias+set} +ac_cv_env_build_alias_value=$build_alias +ac_env_host_alias_set=${host_alias+set} +ac_env_host_alias_value=$host_alias +ac_cv_env_host_alias_set=${host_alias+set} +ac_cv_env_host_alias_value=$host_alias +ac_env_target_alias_set=${target_alias+set} +ac_env_target_alias_value=$target_alias +ac_cv_env_target_alias_set=${target_alias+set} +ac_cv_env_target_alias_value=$target_alias +ac_env_CC_set=${CC+set} +ac_env_CC_value=$CC +ac_cv_env_CC_set=${CC+set} +ac_cv_env_CC_value=$CC +ac_env_CFLAGS_set=${CFLAGS+set} +ac_env_CFLAGS_value=$CFLAGS +ac_cv_env_CFLAGS_set=${CFLAGS+set} +ac_cv_env_CFLAGS_value=$CFLAGS +ac_env_LDFLAGS_set=${LDFLAGS+set} +ac_env_LDFLAGS_value=$LDFLAGS +ac_cv_env_LDFLAGS_set=${LDFLAGS+set} +ac_cv_env_LDFLAGS_value=$LDFLAGS +ac_env_CPPFLAGS_set=${CPPFLAGS+set} +ac_env_CPPFLAGS_value=$CPPFLAGS +ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set} +ac_cv_env_CPPFLAGS_value=$CPPFLAGS +ac_env_CXX_set=${CXX+set} +ac_env_CXX_value=$CXX +ac_cv_env_CXX_set=${CXX+set} +ac_cv_env_CXX_value=$CXX +ac_env_CXXFLAGS_set=${CXXFLAGS+set} +ac_env_CXXFLAGS_value=$CXXFLAGS +ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set} +ac_cv_env_CXXFLAGS_value=$CXXFLAGS +ac_env_CPP_set=${CPP+set} +ac_env_CPP_value=$CPP +ac_cv_env_CPP_set=${CPP+set} +ac_cv_env_CPP_value=$CPP + +# +# Report the --help message. +# +if test "$ac_init_help" = "long"; then + # Omit some internal or obsolete options to make the list less imposing. + # This message is too long to be a string in the A/UX 3.1 sh. + cat <<_ACEOF +\`configure' configures lmserver 1.0 to adapt to many kinds of systems. + +Usage: $0 [OPTION]... [VAR=VALUE]... + +To assign environment variables (e.g., CC, CFLAGS...), specify them as +VAR=VALUE. See below for descriptions of some of the useful variables. + +Defaults for the options are specified in brackets. + +Configuration: + -h, --help display this help and exit + --help=short display options specific to this package + --help=recursive display the short help of all the included packages + -V, --version display version information and exit + -q, --quiet, --silent do not print \`checking...' messages + --cache-file=FILE cache test results in FILE [disabled] + -C, --config-cache alias for \`--cache-file=config.cache' + -n, --no-create do not create output files + --srcdir=DIR find the sources in DIR [configure dir or \`..'] + +_ACEOF + + cat <<_ACEOF +Installation directories: + --prefix=PREFIX install architecture-independent files in PREFIX + [$ac_default_prefix] + --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX + [PREFIX] + +By default, \`make install' will install all the files in +\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify +an installation prefix other than \`$ac_default_prefix' using \`--prefix', +for instance \`--prefix=\$HOME'. + +For better control, use the options below. + +Fine tuning of the installation directories: + --bindir=DIR user executables [EPREFIX/bin] + --sbindir=DIR system admin executables [EPREFIX/sbin] + --libexecdir=DIR program executables [EPREFIX/libexec] + --datadir=DIR read-only architecture-independent data [PREFIX/share] + --sysconfdir=DIR read-only single-machine data [PREFIX/etc] + --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] + --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --libdir=DIR object code libraries [EPREFIX/lib] + --includedir=DIR C header files [PREFIX/include] + --oldincludedir=DIR C header files for non-gcc [/usr/include] + --infodir=DIR info documentation [PREFIX/info] + --mandir=DIR man documentation [PREFIX/man] +_ACEOF + + cat <<\_ACEOF + +Program names: + --program-prefix=PREFIX prepend PREFIX to installed program names + --program-suffix=SUFFIX append SUFFIX to installed program names + --program-transform-name=PROGRAM run sed PROGRAM on installed program names + +System types: + --build=BUILD configure for building on BUILD [guessed] + --host=HOST cross-compile to build programs to run on HOST [BUILD] + --target=TARGET configure for building compilers for TARGET [HOST] +_ACEOF +fi + +if test -n "$ac_init_help"; then + case $ac_init_help in + short | recursive ) echo "Configuration of lmserver 1.0:";; + esac + cat <<\_ACEOF + +Optional Features: + --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) + --enable-FEATURE[=ARG] include FEATURE [ARG=yes] + --disable-dependency-tracking speeds up one-time build + --enable-dependency-tracking do not reject slow dependency extractors + --enable-64bit build 64bit verison + --enable-threads support multithreaded execution + +Optional Packages: + --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] + --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) + --with-srilm=PATH (optional) path to SRI's LM toolkit + --with-libevent=PATH Specify path to libevent installation + +Some influential environment variables: + CC C compiler command + CFLAGS C compiler flags + LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries in a + nonstandard directory <lib dir> + CPPFLAGS C/C++ preprocessor flags, e.g. -I<include dir> if you have + headers in a nonstandard directory <include dir> + CXX C++ compiler command + CXXFLAGS C++ compiler flags + CPP C preprocessor + +Use these variables to override the choices made by `configure' or to help +it to find libraries and programs with nonstandard names/locations. + +_ACEOF +fi + +if test "$ac_init_help" = "recursive"; then + # If there are subdirs, report their specific --help. + ac_popdir=`pwd` + for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue + test -d $ac_dir || continue + ac_builddir=. + +if test "$ac_dir" != .; then + ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'` + # A "../" for each directory in $ac_dir_suffix. + ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'` +else + ac_dir_suffix= ac_top_builddir= +fi + +case $srcdir in + .) # No --srcdir option. We are building in place. + ac_srcdir=. + if test -z "$ac_top_builddir"; then + ac_top_srcdir=. + else + ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'` + fi ;; + [\\/]* | ?:[\\/]* ) # Absolute path. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir ;; + *) # Relative path. + ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_builddir$srcdir ;; +esac + +# Do not use `cd foo && pwd` to compute absolute paths, because +# the directories may not exist. +case `pwd` in +.) ac_abs_builddir="$ac_dir";; +*) + case "$ac_dir" in + .) ac_abs_builddir=`pwd`;; + [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";; + *) ac_abs_builddir=`pwd`/"$ac_dir";; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_builddir=${ac_top_builddir}.;; +*) + case ${ac_top_builddir}. in + .) ac_abs_top_builddir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;; + *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_srcdir=$ac_srcdir;; +*) + case $ac_srcdir in + .) ac_abs_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;; + *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_srcdir=$ac_top_srcdir;; +*) + case $ac_top_srcdir in + .) ac_abs_top_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;; + *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;; + esac;; +esac + + cd $ac_dir + # Check for guested configure; otherwise get Cygnus style configure. + if test -f $ac_srcdir/configure.gnu; then + echo + $SHELL $ac_srcdir/configure.gnu --help=recursive + elif test -f $ac_srcdir/configure; then + echo + $SHELL $ac_srcdir/configure --help=recursive + elif test -f $ac_srcdir/configure.ac || + test -f $ac_srcdir/configure.in; then + echo + $ac_configure --help + else + echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 + fi + cd $ac_popdir + done +fi + +test -n "$ac_init_help" && exit 0 +if $ac_init_version; then + cat <<\_ACEOF +lmserver configure 1.0 +generated by GNU Autoconf 2.59 + +Copyright (C) 2003 Free Software Foundation, Inc. +This configure script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it. +_ACEOF + exit 0 +fi +exec 5>config.log +cat >&5 <<_ACEOF +This file contains any messages produced by compilers while +running configure, to aid debugging if configure makes a mistake. + +It was created by lmserver $as_me 1.0, which was +generated by GNU Autoconf 2.59. Invocation command line was + + $ $0 $@ + +_ACEOF +{ +cat <<_ASUNAME +## --------- ## +## Platform. ## +## --------- ## + +hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` +uname -m = `(uname -m) 2>/dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` + +/bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` +hostinfo = `(hostinfo) 2>/dev/null || echo unknown` +/bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` +/bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` + +_ASUNAME + +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + echo "PATH: $as_dir" +done + +} >&5 + +cat >&5 <<_ACEOF + + +## ----------- ## +## Core tests. ## +## ----------- ## + +_ACEOF + + +# Keep a trace of the command line. +# Strip out --no-create and --no-recursion so they do not pile up. +# Strip out --silent because we don't want to record it for future runs. +# Also quote any args containing shell meta-characters. +# Make two passes to allow for proper duplicate-argument suppression. +ac_configure_args= +ac_configure_args0= +ac_configure_args1= +ac_sep= +ac_must_keep_next=false +for ac_pass in 1 2 +do + for ac_arg + do + case $ac_arg in + -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + continue ;; + *" "*|*" "*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*) + ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + case $ac_pass in + 1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;; + 2) + ac_configure_args1="$ac_configure_args1 '$ac_arg'" + if test $ac_must_keep_next = true; then + ac_must_keep_next=false # Got value, back to normal. + else + case $ac_arg in + *=* | --config-cache | -C | -disable-* | --disable-* \ + | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ + | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ + | -with-* | --with-* | -without-* | --without-* | --x) + case "$ac_configure_args0 " in + "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; + esac + ;; + -* ) ac_must_keep_next=true ;; + esac + fi + ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'" + # Get rid of the leading space. + ac_sep=" " + ;; + esac + done +done +$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; } +$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; } + +# When interrupted or exit'd, cleanup temporary files, and complete +# config.log. We remove comments because anyway the quotes in there +# would cause problems or look ugly. +# WARNING: Be sure not to use single quotes in there, as some shells, +# such as our DU 5.0 friend, will then `close' the trap. +trap 'exit_status=$? + # Save into config.log some information that might help in debugging. + { + echo + + cat <<\_ASBOX +## ---------------- ## +## Cache variables. ## +## ---------------- ## +_ASBOX + echo + # The following way of writing the cache mishandles newlines in values, +{ + (set) 2>&1 | + case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in + *ac_space=\ *) + sed -n \ + "s/'"'"'/'"'"'\\\\'"'"''"'"'/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p" + ;; + *) + sed -n \ + "s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p" + ;; + esac; +} + echo + + cat <<\_ASBOX +## ----------------- ## +## Output variables. ## +## ----------------- ## +_ASBOX + echo + for ac_var in $ac_subst_vars + do + eval ac_val=$`echo $ac_var` + echo "$ac_var='"'"'$ac_val'"'"'" + done | sort + echo + + if test -n "$ac_subst_files"; then + cat <<\_ASBOX +## ------------- ## +## Output files. ## +## ------------- ## +_ASBOX + echo + for ac_var in $ac_subst_files + do + eval ac_val=$`echo $ac_var` + echo "$ac_var='"'"'$ac_val'"'"'" + done | sort + echo + fi + + if test -s confdefs.h; then + cat <<\_ASBOX +## ----------- ## +## confdefs.h. ## +## ----------- ## +_ASBOX + echo + sed "/^$/d" confdefs.h | sort + echo + fi + test "$ac_signal" != 0 && + echo "$as_me: caught signal $ac_signal" + echo "$as_me: exit $exit_status" + } >&5 + rm -f core *.core && + rm -rf conftest* confdefs* conf$$* $ac_clean_files && + exit $exit_status + ' 0 +for ac_signal in 1 2 13 15; do + trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal +done +ac_signal=0 + +# confdefs.h avoids OS command line length limits that DEFS can exceed. +rm -rf conftest* confdefs.h +# AIX cpp loses on an empty file, so make sure it contains at least a newline. +echo >confdefs.h + +# Predefined preprocessor variables. + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_NAME "$PACKAGE_NAME" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_TARNAME "$PACKAGE_TARNAME" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_VERSION "$PACKAGE_VERSION" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_STRING "$PACKAGE_STRING" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" +_ACEOF + + +# Let the site file select an alternate cache file if it wants to. +# Prefer explicitly selected file to automatically selected ones. +if test -z "$CONFIG_SITE"; then + if test "x$prefix" != xNONE; then + CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site" + else + CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site" + fi +fi +for ac_site_file in $CONFIG_SITE; do + if test -r "$ac_site_file"; then + { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5 +echo "$as_me: loading site script $ac_site_file" >&6;} + sed 's/^/| /' "$ac_site_file" >&5 + . "$ac_site_file" + fi +done + +if test -r "$cache_file"; then + # Some versions of bash will fail to source /dev/null (special + # files actually), so we avoid doing that. + if test -f "$cache_file"; then + { echo "$as_me:$LINENO: loading cache $cache_file" >&5 +echo "$as_me: loading cache $cache_file" >&6;} + case $cache_file in + [\\/]* | ?:[\\/]* ) . $cache_file;; + *) . ./$cache_file;; + esac + fi +else + { echo "$as_me:$LINENO: creating cache $cache_file" >&5 +echo "$as_me: creating cache $cache_file" >&6;} + >$cache_file +fi + +# Check that the precious variables saved in the cache have kept the same +# value. +ac_cache_corrupted=false +for ac_var in `(set) 2>&1 | + sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do + eval ac_old_set=\$ac_cv_env_${ac_var}_set + eval ac_new_set=\$ac_env_${ac_var}_set + eval ac_old_val="\$ac_cv_env_${ac_var}_value" + eval ac_new_val="\$ac_env_${ac_var}_value" + case $ac_old_set,$ac_new_set in + set,) + { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 +echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,set) + { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5 +echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,);; + *) + if test "x$ac_old_val" != "x$ac_new_val"; then + { echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5 +echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} + { echo "$as_me:$LINENO: former value: $ac_old_val" >&5 +echo "$as_me: former value: $ac_old_val" >&2;} + { echo "$as_me:$LINENO: current value: $ac_new_val" >&5 +echo "$as_me: current value: $ac_new_val" >&2;} + ac_cache_corrupted=: + fi;; + esac + # Pass precious variables to config.status. + if test "$ac_new_set" = set; then + case $ac_new_val in + *" "*|*" "*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*) + ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; + *) ac_arg=$ac_var=$ac_new_val ;; + esac + case " $ac_configure_args " in + *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. + *) ac_configure_args="$ac_configure_args '$ac_arg'" ;; + esac + fi +done +if $ac_cache_corrupted; then + { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5 +echo "$as_me: error: changes in the environment can compromise the build" >&2;} + { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5 +echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;} + { (exit 1); exit 1; }; } +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + + + + + + + + + + + + + + + + + + + + + + + + + + +ac_aux_dir= +for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do + if test -f $ac_dir/install-sh; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install-sh -c" + break + elif test -f $ac_dir/install.sh; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install.sh -c" + break + elif test -f $ac_dir/shtool; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/shtool install -c" + break + fi +done +if test -z "$ac_aux_dir"; then + { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5 +echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;} + { (exit 1); exit 1; }; } +fi +ac_config_guess="$SHELL $ac_aux_dir/config.guess" +ac_config_sub="$SHELL $ac_aux_dir/config.sub" +ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure. + +# Make sure we can run config.sub. +$ac_config_sub sun4 >/dev/null 2>&1 || + { { echo "$as_me:$LINENO: error: cannot run $ac_config_sub" >&5 +echo "$as_me: error: cannot run $ac_config_sub" >&2;} + { (exit 1); exit 1; }; } + +echo "$as_me:$LINENO: checking build system type" >&5 +echo $ECHO_N "checking build system type... $ECHO_C" >&6 +if test "${ac_cv_build+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_cv_build_alias=$build_alias +test -z "$ac_cv_build_alias" && + ac_cv_build_alias=`$ac_config_guess` +test -z "$ac_cv_build_alias" && + { { echo "$as_me:$LINENO: error: cannot guess build type; you must specify one" >&5 +echo "$as_me: error: cannot guess build type; you must specify one" >&2;} + { (exit 1); exit 1; }; } +ac_cv_build=`$ac_config_sub $ac_cv_build_alias` || + { { echo "$as_me:$LINENO: error: $ac_config_sub $ac_cv_build_alias failed" >&5 +echo "$as_me: error: $ac_config_sub $ac_cv_build_alias failed" >&2;} + { (exit 1); exit 1; }; } + +fi +echo "$as_me:$LINENO: result: $ac_cv_build" >&5 +echo "${ECHO_T}$ac_cv_build" >&6 +build=$ac_cv_build +build_cpu=`echo $ac_cv_build | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\1/'` +build_vendor=`echo $ac_cv_build | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\2/'` +build_os=`echo $ac_cv_build | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\3/'` + + +echo "$as_me:$LINENO: checking host system type" >&5 +echo $ECHO_N "checking host system type... $ECHO_C" >&6 +if test "${ac_cv_host+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_cv_host_alias=$host_alias +test -z "$ac_cv_host_alias" && + ac_cv_host_alias=$ac_cv_build_alias +ac_cv_host=`$ac_config_sub $ac_cv_host_alias` || + { { echo "$as_me:$LINENO: error: $ac_config_sub $ac_cv_host_alias failed" >&5 +echo "$as_me: error: $ac_config_sub $ac_cv_host_alias failed" >&2;} + { (exit 1); exit 1; }; } + +fi +echo "$as_me:$LINENO: result: $ac_cv_host" >&5 +echo "${ECHO_T}$ac_cv_host" >&6 +host=$ac_cv_host +host_cpu=`echo $ac_cv_host | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\1/'` +host_vendor=`echo $ac_cv_host | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\2/'` +host_os=`echo $ac_cv_host | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\3/'` + + +echo "$as_me:$LINENO: checking target system type" >&5 +echo $ECHO_N "checking target system type... $ECHO_C" >&6 +if test "${ac_cv_target+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_cv_target_alias=$target_alias +test "x$ac_cv_target_alias" = "x" && + ac_cv_target_alias=$ac_cv_host_alias +ac_cv_target=`$ac_config_sub $ac_cv_target_alias` || + { { echo "$as_me:$LINENO: error: $ac_config_sub $ac_cv_target_alias failed" >&5 +echo "$as_me: error: $ac_config_sub $ac_cv_target_alias failed" >&2;} + { (exit 1); exit 1; }; } + +fi +echo "$as_me:$LINENO: result: $ac_cv_target" >&5 +echo "${ECHO_T}$ac_cv_target" >&6 +target=$ac_cv_target +target_cpu=`echo $ac_cv_target | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\1/'` +target_vendor=`echo $ac_cv_target | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\2/'` +target_os=`echo $ac_cv_target | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\3/'` + + +# The aliases save the names the user supplied, while $host etc. +# will get canonicalized. +test -n "$target_alias" && + test "$program_prefix$program_suffix$program_transform_name" = \ + NONENONEs,x,x, && + program_prefix=${target_alias}- + +am__api_version="1.9" +# Find a good install program. We prefer a C program (faster), +# so one script is as good as another. But avoid the broken or +# incompatible versions: +# SysV /etc/install, /usr/sbin/install +# SunOS /usr/etc/install +# IRIX /sbin/install +# AIX /bin/install +# AmigaOS /C/install, which installs bootblocks on floppy discs +# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag +# AFS /usr/afsws/bin/install, which mishandles nonexistent args +# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" +# OS/2's system install, which has a completely different semantic +# ./install, which can be erroneously created by make from ./install.sh. +echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5 +echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6 +if test -z "$INSTALL"; then +if test "${ac_cv_path_install+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + # Account for people who put trailing slashes in PATH elements. +case $as_dir/ in + ./ | .// | /cC/* | \ + /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ + ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \ + /usr/ucb/* ) ;; + *) + # OSF1 and SCO ODT 3.0 have their own names for install. + # Don't use installbsd from OSF since it installs stuff as root + # by default. + for ac_prog in ginstall scoinst install; do + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then + if test $ac_prog = install && + grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # AIX install. It has an incompatible calling convention. + : + elif test $ac_prog = install && + grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # program-specific install script used by HP pwplus--don't use. + : + else + ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c" + break 3 + fi + fi + done + done + ;; +esac +done + + +fi + if test "${ac_cv_path_install+set}" = set; then + INSTALL=$ac_cv_path_install + else + # As a last resort, use the slow shell script. We don't cache a + # path for INSTALL within a source directory, because that will + # break other packages using the cache if that directory is + # removed, or if the path is relative. + INSTALL=$ac_install_sh + fi +fi +echo "$as_me:$LINENO: result: $INSTALL" >&5 +echo "${ECHO_T}$INSTALL" >&6 + +# Use test -z because SunOS4 sh mishandles braces in ${var-val}. +# It thinks the first close brace ends the variable substitution. +test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' + +test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' + +test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' + +echo "$as_me:$LINENO: checking whether build environment is sane" >&5 +echo $ECHO_N "checking whether build environment is sane... $ECHO_C" >&6 +# Just in case +sleep 1 +echo timestamp > conftest.file +# Do `set' in a subshell so we don't clobber the current shell's +# arguments. Must try -L first in case configure is actually a +# symlink; some systems play weird games with the mod time of symlinks +# (eg FreeBSD returns the mod time of the symlink's containing +# directory). +if ( + set X `ls -Lt $srcdir/configure conftest.file 2> /dev/null` + if test "$*" = "X"; then + # -L didn't work. + set X `ls -t $srcdir/configure conftest.file` + fi + rm -f conftest.file + if test "$*" != "X $srcdir/configure conftest.file" \ + && test "$*" != "X conftest.file $srcdir/configure"; then + + # If neither matched, then we have a broken ls. This can happen + # if, for instance, CONFIG_SHELL is bash and it inherits a + # broken ls alias from the environment. This has actually + # happened. Such a system could not be considered "sane". + { { echo "$as_me:$LINENO: error: ls -t appears to fail. Make sure there is not a broken +alias in your environment" >&5 +echo "$as_me: error: ls -t appears to fail. Make sure there is not a broken +alias in your environment" >&2;} + { (exit 1); exit 1; }; } + fi + + test "$2" = conftest.file + ) +then + # Ok. + : +else + { { echo "$as_me:$LINENO: error: newly created file is older than distributed files! +Check your system clock" >&5 +echo "$as_me: error: newly created file is older than distributed files! +Check your system clock" >&2;} + { (exit 1); exit 1; }; } +fi +echo "$as_me:$LINENO: result: yes" >&5 +echo "${ECHO_T}yes" >&6 +test "$program_prefix" != NONE && + program_transform_name="s,^,$program_prefix,;$program_transform_name" +# Use a double $ so make ignores it. +test "$program_suffix" != NONE && + program_transform_name="s,\$,$program_suffix,;$program_transform_name" +# Double any \ or $. echo might interpret backslashes. +# By default was `s,x,x', remove it if useless. +cat <<\_ACEOF >conftest.sed +s/[\\$]/&&/g;s/;s,x,x,$// +_ACEOF +program_transform_name=`echo $program_transform_name | sed -f conftest.sed` +rm conftest.sed + +# expand $ac_aux_dir to an absolute path +am_aux_dir=`cd $ac_aux_dir && pwd` + +test x"${MISSING+set}" = xset || MISSING="\${SHELL} $am_aux_dir/missing" +# Use eval to expand $SHELL +if eval "$MISSING --run true"; then + am_missing_run="$MISSING --run " +else + am_missing_run= + { echo "$as_me:$LINENO: WARNING: \`missing' script is too old or missing" >&5 +echo "$as_me: WARNING: \`missing' script is too old or missing" >&2;} +fi + +if mkdir -p --version . >/dev/null 2>&1 && test ! -d ./--version; then + # We used to keeping the `.' as first argument, in order to + # allow $(mkdir_p) to be used without argument. As in + # $(mkdir_p) $(somedir) + # where $(somedir) is conditionally defined. However this is wrong + # for two reasons: + # 1. if the package is installed by a user who cannot write `.' + # make install will fail, + # 2. the above comment should most certainly read + # $(mkdir_p) $(DESTDIR)$(somedir) + # so it does not work when $(somedir) is undefined and + # $(DESTDIR) is not. + # To support the latter case, we have to write + # test -z "$(somedir)" || $(mkdir_p) $(DESTDIR)$(somedir), + # so the `.' trick is pointless. + mkdir_p='mkdir -p --' +else + # On NextStep and OpenStep, the `mkdir' command does not + # recognize any option. It will interpret all options as + # directories to create, and then abort because `.' already + # exists. + for d in ./-p ./--version; + do + test -d $d && rmdir $d + done + # $(mkinstalldirs) is defined by Automake if mkinstalldirs exists. + if test -f "$ac_aux_dir/mkinstalldirs"; then + mkdir_p='$(mkinstalldirs)' + else + mkdir_p='$(install_sh) -d' + fi +fi + +for ac_prog in gawk mawk nawk awk +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_AWK+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$AWK"; then + ac_cv_prog_AWK="$AWK" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_AWK="$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +AWK=$ac_cv_prog_AWK +if test -n "$AWK"; then + echo "$as_me:$LINENO: result: $AWK" >&5 +echo "${ECHO_T}$AWK" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + test -n "$AWK" && break +done + +echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5 +echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6 +set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'` +if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.make <<\_ACEOF +all: + @echo 'ac_maketemp="$(MAKE)"' +_ACEOF +# GNU make sometimes prints "make[1]: Entering...", which would confuse us. +eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=` +if test -n "$ac_maketemp"; then + eval ac_cv_prog_make_${ac_make}_set=yes +else + eval ac_cv_prog_make_${ac_make}_set=no +fi +rm -f conftest.make +fi +if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then + echo "$as_me:$LINENO: result: yes" >&5 +echo "${ECHO_T}yes" >&6 + SET_MAKE= +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 + SET_MAKE="MAKE=${MAKE-make}" +fi + +rm -rf .tst 2>/dev/null +mkdir .tst 2>/dev/null +if test -d .tst; then + am__leading_dot=. +else + am__leading_dot=_ +fi +rmdir .tst 2>/dev/null + +# test to see if srcdir already configured +if test "`cd $srcdir && pwd`" != "`pwd`" && + test -f $srcdir/config.status; then + { { echo "$as_me:$LINENO: error: source directory already configured; run \"make distclean\" there first" >&5 +echo "$as_me: error: source directory already configured; run \"make distclean\" there first" >&2;} + { (exit 1); exit 1; }; } +fi + +# test whether we have cygpath +if test -z "$CYGPATH_W"; then + if (cygpath --version) >/dev/null 2>/dev/null; then + CYGPATH_W='cygpath -w' + else + CYGPATH_W=echo + fi +fi + + +# Define the identity of the package. + PACKAGE=lmserver + VERSION=1.0 + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE "$PACKAGE" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define VERSION "$VERSION" +_ACEOF + +# Some tools Automake needs. + +ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"} + + +AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"} + + +AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"} + + +AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"} + + +MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"} + +install_sh=${install_sh-"$am_aux_dir/install-sh"} + +# Installed binaries are usually stripped using `strip' when the user +# run `make install-strip'. However `strip' might not be the right +# tool to use in cross-compilation environments, therefore Automake +# will honor the `STRIP' environment variable to overrule this program. +if test "$cross_compiling" != no; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. +set dummy ${ac_tool_prefix}strip; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_STRIP+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$STRIP"; then + ac_cv_prog_STRIP="$STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_STRIP="${ac_tool_prefix}strip" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +STRIP=$ac_cv_prog_STRIP +if test -n "$STRIP"; then + echo "$as_me:$LINENO: result: $STRIP" >&5 +echo "${ECHO_T}$STRIP" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + +fi +if test -z "$ac_cv_prog_STRIP"; then + ac_ct_STRIP=$STRIP + # Extract the first word of "strip", so it can be a program name with args. +set dummy strip; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_ac_ct_STRIP+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$ac_ct_STRIP"; then + ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_STRIP="strip" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + + test -z "$ac_cv_prog_ac_ct_STRIP" && ac_cv_prog_ac_ct_STRIP=":" +fi +fi +ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP +if test -n "$ac_ct_STRIP"; then + echo "$as_me:$LINENO: result: $ac_ct_STRIP" >&5 +echo "${ECHO_T}$ac_ct_STRIP" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + STRIP=$ac_ct_STRIP +else + STRIP="$ac_cv_prog_STRIP" +fi + +fi +INSTALL_STRIP_PROGRAM="\${SHELL} \$(install_sh) -c -s" + +# We need awk for the "check" target. The system "awk" is bad on +# some platforms. +# Always define AMTAR for backward compatibility. + +AMTAR=${AMTAR-"${am_missing_run}tar"} + +am__tar='${AMTAR} chof - "$$tardir"'; am__untar='${AMTAR} xf -' + + + + + + ac_config_headers="$ac_config_headers config.h" + + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. +set dummy ${ac_tool_prefix}gcc; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_CC+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_CC="${ac_tool_prefix}gcc" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + echo "$as_me:$LINENO: result: $CC" >&5 +echo "${ECHO_T}$CC" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + +fi +if test -z "$ac_cv_prog_CC"; then + ac_ct_CC=$CC + # Extract the first word of "gcc", so it can be a program name with args. +set dummy gcc; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_ac_ct_CC+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_CC="gcc" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 +echo "${ECHO_T}$ac_ct_CC" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + CC=$ac_ct_CC +else + CC="$ac_cv_prog_CC" +fi + +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. +set dummy ${ac_tool_prefix}cc; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_CC+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_CC="${ac_tool_prefix}cc" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + echo "$as_me:$LINENO: result: $CC" >&5 +echo "${ECHO_T}$CC" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + +fi +if test -z "$ac_cv_prog_CC"; then + ac_ct_CC=$CC + # Extract the first word of "cc", so it can be a program name with args. +set dummy cc; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_ac_ct_CC+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_CC="cc" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 +echo "${ECHO_T}$ac_ct_CC" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + CC=$ac_ct_CC +else + CC="$ac_cv_prog_CC" +fi + +fi +if test -z "$CC"; then + # Extract the first word of "cc", so it can be a program name with args. +set dummy cc; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_CC+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else + ac_prog_rejected=no +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then + ac_prog_rejected=yes + continue + fi + ac_cv_prog_CC="cc" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +if test $ac_prog_rejected = yes; then + # We found a bogon in the path, so make sure we never use it. + set dummy $ac_cv_prog_CC + shift + if test $# != 0; then + # We chose a different compiler from the bogus one. + # However, it has the same basename, so the bogon will be chosen + # first if we set CC to just the basename; use the full file name. + shift + ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" + fi +fi +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + echo "$as_me:$LINENO: result: $CC" >&5 +echo "${ECHO_T}$CC" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + +fi +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + for ac_prog in cl + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_CC+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_CC="$ac_tool_prefix$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + echo "$as_me:$LINENO: result: $CC" >&5 +echo "${ECHO_T}$CC" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + test -n "$CC" && break + done +fi +if test -z "$CC"; then + ac_ct_CC=$CC + for ac_prog in cl +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_ac_ct_CC+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_CC="$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 +echo "${ECHO_T}$ac_ct_CC" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + test -n "$ac_ct_CC" && break +done + + CC=$ac_ct_CC +fi + +fi + + +test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH +See \`config.log' for more details." >&5 +echo "$as_me: error: no acceptable C compiler found in \$PATH +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } + +# Provide some information about the compiler. +echo "$as_me:$LINENO:" \ + "checking for C compiler version" >&5 +ac_compiler=`set X $ac_compile; echo $2` +{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5 + (eval $ac_compiler --version </dev/null >&5) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5 + (eval $ac_compiler -v </dev/null >&5) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5 + (eval $ac_compiler -V </dev/null >&5) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } + +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files a.out a.exe b.out" +# Try to create an executable without -o first, disregard a.out. +# It will help us diagnose broken compilers, and finding out an intuition +# of exeext. +echo "$as_me:$LINENO: checking for C compiler default output file name" >&5 +echo $ECHO_N "checking for C compiler default output file name... $ECHO_C" >&6 +ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` +if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5 + (eval $ac_link_default) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then + # Find the output, starting from the most likely. This scheme is +# not robust to junk in `.', hence go to wildcards (a.*) only as a last +# resort. + +# Be careful to initialize this variable, since it used to be cached. +# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile. +ac_cv_exeext= +# b.out is created by i960 compilers. +for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out +do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) + ;; + conftest.$ac_ext ) + # This is the source file. + ;; + [ab].out ) + # We found the default executable, but exeext='' is most + # certainly right. + break;; + *.* ) + ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + # FIXME: I believe we export ac_cv_exeext for Libtool, + # but it would be cool to find out if it's true. Does anybody + # maintain Libtool? --akim. + export ac_cv_exeext + break;; + * ) + break;; + esac +done +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { echo "$as_me:$LINENO: error: C compiler cannot create executables +See \`config.log' for more details." >&5 +echo "$as_me: error: C compiler cannot create executables +See \`config.log' for more details." >&2;} + { (exit 77); exit 77; }; } +fi + +ac_exeext=$ac_cv_exeext +echo "$as_me:$LINENO: result: $ac_file" >&5 +echo "${ECHO_T}$ac_file" >&6 + +# Check the compiler produces executables we can run. If not, either +# the compiler is broken, or we cross compile. +echo "$as_me:$LINENO: checking whether the C compiler works" >&5 +echo $ECHO_N "checking whether the C compiler works... $ECHO_C" >&6 +# FIXME: These cross compiler hacks should be removed for Autoconf 3.0 +# If not cross compiling, check that we can run a simple program. +if test "$cross_compiling" != yes; then + if { ac_try='./$ac_file' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + cross_compiling=no + else + if test "$cross_compiling" = maybe; then + cross_compiling=yes + else + { { echo "$as_me:$LINENO: error: cannot run C compiled programs. +If you meant to cross compile, use \`--host'. +See \`config.log' for more details." >&5 +echo "$as_me: error: cannot run C compiled programs. +If you meant to cross compile, use \`--host'. +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } + fi + fi +fi +echo "$as_me:$LINENO: result: yes" >&5 +echo "${ECHO_T}yes" >&6 + +rm -f a.out a.exe conftest$ac_cv_exeext b.out +ac_clean_files=$ac_clean_files_save +# Check the compiler produces executables we can run. If not, either +# the compiler is broken, or we cross compile. +echo "$as_me:$LINENO: checking whether we are cross compiling" >&5 +echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6 +echo "$as_me:$LINENO: result: $cross_compiling" >&5 +echo "${ECHO_T}$cross_compiling" >&6 + +echo "$as_me:$LINENO: checking for suffix of executables" >&5 +echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6 +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then + # If both `conftest.exe' and `conftest' are `present' (well, observable) +# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will +# work properly (i.e., refer to `conftest.exe'), while it won't with +# `rm'. +for ac_file in conftest.exe conftest conftest.*; do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;; + *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + export ac_cv_exeext + break;; + * ) break;; + esac +done +else + { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details." >&5 +echo "$as_me: error: cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } +fi + +rm -f conftest$ac_cv_exeext +echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5 +echo "${ECHO_T}$ac_cv_exeext" >&6 + +rm -f conftest.$ac_ext +EXEEXT=$ac_cv_exeext +ac_exeext=$EXEEXT +echo "$as_me:$LINENO: checking for suffix of object files" >&5 +echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6 +if test "${ac_cv_objext+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.o conftest.obj +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then + for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;; + *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` + break;; + esac +done +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile +See \`config.log' for more details." >&5 +echo "$as_me: error: cannot compute suffix of object files: cannot compile +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } +fi + +rm -f conftest.$ac_cv_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_objext" >&5 +echo "${ECHO_T}$ac_cv_objext" >&6 +OBJEXT=$ac_cv_objext +ac_objext=$OBJEXT +echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5 +echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6 +if test "${ac_cv_c_compiler_gnu+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_compiler_gnu=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_compiler_gnu=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_c_compiler_gnu=$ac_compiler_gnu + +fi +echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5 +echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6 +GCC=`test $ac_compiler_gnu = yes && echo yes` +ac_test_CFLAGS=${CFLAGS+set} +ac_save_CFLAGS=$CFLAGS +CFLAGS="-g" +echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5 +echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6 +if test "${ac_cv_prog_cc_g+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_prog_cc_g=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_prog_cc_g=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5 +echo "${ECHO_T}$ac_cv_prog_cc_g" >&6 +if test "$ac_test_CFLAGS" = set; then + CFLAGS=$ac_save_CFLAGS +elif test $ac_cv_prog_cc_g = yes; then + if test "$GCC" = yes; then + CFLAGS="-g -O2" + else + CFLAGS="-g" + fi +else + if test "$GCC" = yes; then + CFLAGS="-O2" + else + CFLAGS= + fi +fi +echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5 +echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6 +if test "${ac_cv_prog_cc_stdc+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_cv_prog_cc_stdc=no +ac_save_CC=$CC +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <stdarg.h> +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ +struct buf { int x; }; +FILE * (*rcsopen) (struct buf *, struct stat *, int); +static char *e (p, i) + char **p; + int i; +{ + return p[i]; +} +static char *f (char * (*g) (char **, int), char **p, ...) +{ + char *s; + va_list v; + va_start (v,p); + s = g (p, va_arg (v,int)); + va_end (v); + return s; +} + +/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has + function prototypes and stuff, but not '\xHH' hex character constants. + These don't provoke an error unfortunately, instead are silently treated + as 'x'. The following induces an error, until -std1 is added to get + proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an + array size at least. It's necessary to write '\x00'==0 to get something + that's true only with -std1. */ +int osf4_cc_array ['\x00' == 0 ? 1 : -1]; + +int test (int i, double x); +struct s1 {int (*f) (int a);}; +struct s2 {int (*f) (double a);}; +int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); +int argc; +char **argv; +int +main () +{ +return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; + ; + return 0; +} +_ACEOF +# Don't try gcc -ansi; that turns off useful extensions and +# breaks some systems' header files. +# AIX -qlanglvl=ansi +# Ultrix and OSF/1 -std1 +# HP-UX 10.20 and later -Ae +# HP-UX older versions -Aa -D_HPUX_SOURCE +# SVR4 -Xc -D__EXTENSIONS__ +for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" +do + CC="$ac_save_CC $ac_arg" + rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_prog_cc_stdc=$ac_arg +break +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext +done +rm -f conftest.$ac_ext conftest.$ac_objext +CC=$ac_save_CC + +fi + +case "x$ac_cv_prog_cc_stdc" in + x|xno) + echo "$as_me:$LINENO: result: none needed" >&5 +echo "${ECHO_T}none needed" >&6 ;; + *) + echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5 +echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6 + CC="$CC $ac_cv_prog_cc_stdc" ;; +esac + +# Some people use a C++ compiler to compile C. Since we use `exit', +# in C++ we need to declare it. In case someone uses the same compiler +# for both compiling C and C++ we need to have the C++ compiler decide +# the declaration of exit, since it's the most demanding environment. +cat >conftest.$ac_ext <<_ACEOF +#ifndef __cplusplus + choke me +#endif +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + for ac_declaration in \ + '' \ + 'extern "C" void std::exit (int) throw (); using std::exit;' \ + 'extern "C" void std::exit (int); using std::exit;' \ + 'extern "C" void exit (int) throw ();' \ + 'extern "C" void exit (int);' \ + 'void exit (int);' +do + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_declaration +#include <stdlib.h> +int +main () +{ +exit (42); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + : +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +continue +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_declaration +int +main () +{ +exit (42); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + break +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +done +rm -f conftest* +if test -n "$ac_declaration"; then + echo '#ifdef __cplusplus' >>confdefs.h + echo $ac_declaration >>confdefs.h + echo '#endif' >>confdefs.h +fi + +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +DEPDIR="${am__leading_dot}deps" + + ac_config_commands="$ac_config_commands depfiles" + + +am_make=${MAKE-make} +cat > confinc << 'END' +am__doit: + @echo done +.PHONY: am__doit +END +# If we don't find an include directive, just comment out the code. +echo "$as_me:$LINENO: checking for style of include used by $am_make" >&5 +echo $ECHO_N "checking for style of include used by $am_make... $ECHO_C" >&6 +am__include="#" +am__quote= +_am_result=none +# First try GNU make style include. +echo "include confinc" > confmf +# We grep out `Entering directory' and `Leaving directory' +# messages which can occur if `w' ends up in MAKEFLAGS. +# In particular we don't look at `^make:' because GNU make might +# be invoked under some other name (usually "gmake"), in which +# case it prints its new name instead of `make'. +if test "`$am_make -s -f confmf 2> /dev/null | grep -v 'ing directory'`" = "done"; then + am__include=include + am__quote= + _am_result=GNU +fi +# Now try BSD make style include. +if test "$am__include" = "#"; then + echo '.include "confinc"' > confmf + if test "`$am_make -s -f confmf 2> /dev/null`" = "done"; then + am__include=.include + am__quote="\"" + _am_result=BSD + fi +fi + + +echo "$as_me:$LINENO: result: $_am_result" >&5 +echo "${ECHO_T}$_am_result" >&6 +rm -f confinc confmf + +# Check whether --enable-dependency-tracking or --disable-dependency-tracking was given. +if test "${enable_dependency_tracking+set}" = set; then + enableval="$enable_dependency_tracking" + +fi; +if test "x$enable_dependency_tracking" != xno; then + am_depcomp="$ac_aux_dir/depcomp" + AMDEPBACKSLASH='\' +fi + + +if test "x$enable_dependency_tracking" != xno; then + AMDEP_TRUE= + AMDEP_FALSE='#' +else + AMDEP_TRUE='#' + AMDEP_FALSE= +fi + + + + +depcc="$CC" am_compiler_list= + +echo "$as_me:$LINENO: checking dependency style of $depcc" >&5 +echo $ECHO_N "checking dependency style of $depcc... $ECHO_C" >&6 +if test "${am_cv_CC_dependencies_compiler_type+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CC_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + case $depmode in + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + none) break ;; + esac + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. + if depmode=$depmode \ + source=sub/conftest.c object=sub/conftest.${OBJEXT-o} \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c -o sub/conftest.${OBJEXT-o} sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftest.${OBJEXT-o} sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CC_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CC_dependencies_compiler_type=none +fi + +fi +echo "$as_me:$LINENO: result: $am_cv_CC_dependencies_compiler_type" >&5 +echo "${ECHO_T}$am_cv_CC_dependencies_compiler_type" >&6 +CCDEPMODE=depmode=$am_cv_CC_dependencies_compiler_type + + + +if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CC_dependencies_compiler_type" = gcc3; then + am__fastdepCC_TRUE= + am__fastdepCC_FALSE='#' +else + am__fastdepCC_TRUE='#' + am__fastdepCC_FALSE= +fi + + +ac_ext=cc +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +if test -n "$ac_tool_prefix"; then + for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_CXX+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$CXX"; then + ac_cv_prog_CXX="$CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_CXX="$ac_tool_prefix$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +CXX=$ac_cv_prog_CXX +if test -n "$CXX"; then + echo "$as_me:$LINENO: result: $CXX" >&5 +echo "${ECHO_T}$CXX" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + test -n "$CXX" && break + done +fi +if test -z "$CXX"; then + ac_ct_CXX=$CXX + for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 +if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$ac_ct_CXX"; then + ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_CXX="$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done + +fi +fi +ac_ct_CXX=$ac_cv_prog_ac_ct_CXX +if test -n "$ac_ct_CXX"; then + echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5 +echo "${ECHO_T}$ac_ct_CXX" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 +fi + + test -n "$ac_ct_CXX" && break +done +test -n "$ac_ct_CXX" || ac_ct_CXX="g++" + + CXX=$ac_ct_CXX +fi + + +# Provide some information about the compiler. +echo "$as_me:$LINENO:" \ + "checking for C++ compiler version" >&5 +ac_compiler=`set X $ac_compile; echo $2` +{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5 + (eval $ac_compiler --version </dev/null >&5) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5 + (eval $ac_compiler -v </dev/null >&5) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5 + (eval $ac_compiler -V </dev/null >&5) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } + +echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5 +echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6 +if test "${ac_cv_cxx_compiler_gnu+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_cxx_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_compiler_gnu=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_compiler_gnu=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_cxx_compiler_gnu=$ac_compiler_gnu + +fi +echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5 +echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6 +GXX=`test $ac_compiler_gnu = yes && echo yes` +ac_test_CXXFLAGS=${CXXFLAGS+set} +ac_save_CXXFLAGS=$CXXFLAGS +CXXFLAGS="-g" +echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5 +echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6 +if test "${ac_cv_prog_cxx_g+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_cxx_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_prog_cxx_g=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_prog_cxx_g=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5 +echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6 +if test "$ac_test_CXXFLAGS" = set; then + CXXFLAGS=$ac_save_CXXFLAGS +elif test $ac_cv_prog_cxx_g = yes; then + if test "$GXX" = yes; then + CXXFLAGS="-g -O2" + else + CXXFLAGS="-g" + fi +else + if test "$GXX" = yes; then + CXXFLAGS="-O2" + else + CXXFLAGS= + fi +fi +for ac_declaration in \ + '' \ + 'extern "C" void std::exit (int) throw (); using std::exit;' \ + 'extern "C" void std::exit (int); using std::exit;' \ + 'extern "C" void exit (int) throw ();' \ + 'extern "C" void exit (int);' \ + 'void exit (int);' +do + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_declaration +#include <stdlib.h> +int +main () +{ +exit (42); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_cxx_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + : +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +continue +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_declaration +int +main () +{ +exit (42); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_cxx_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + break +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +done +rm -f conftest* +if test -n "$ac_declaration"; then + echo '#ifdef __cplusplus' >>confdefs.h + echo $ac_declaration >>confdefs.h + echo '#endif' >>confdefs.h +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +depcc="$CXX" am_compiler_list= + +echo "$as_me:$LINENO: checking dependency style of $depcc" >&5 +echo $ECHO_N "checking dependency style of $depcc... $ECHO_C" >&6 +if test "${am_cv_CXX_dependencies_compiler_type+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CXX_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + case $depmode in + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + none) break ;; + esac + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. + if depmode=$depmode \ + source=sub/conftest.c object=sub/conftest.${OBJEXT-o} \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c -o sub/conftest.${OBJEXT-o} sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftest.${OBJEXT-o} sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CXX_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CXX_dependencies_compiler_type=none +fi + +fi +echo "$as_me:$LINENO: result: $am_cv_CXX_dependencies_compiler_type" >&5 +echo "${ECHO_T}$am_cv_CXX_dependencies_compiler_type" >&6 +CXXDEPMODE=depmode=$am_cv_CXX_dependencies_compiler_type + + + +if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CXX_dependencies_compiler_type" = gcc3; then + am__fastdepCXX_TRUE= + am__fastdepCXX_FALSE='#' +else + am__fastdepCXX_TRUE='#' + am__fastdepCXX_FALSE= +fi + + +if test "x$CC" != xcc; then + echo "$as_me:$LINENO: checking whether $CC and cc understand -c and -o together" >&5 +echo $ECHO_N "checking whether $CC and cc understand -c and -o together... $ECHO_C" >&6 +else + echo "$as_me:$LINENO: checking whether cc understands -c and -o together" >&5 +echo $ECHO_N "checking whether cc understands -c and -o together... $ECHO_C" >&6 +fi +set dummy $CC; ac_cc=`echo $2 | + sed 's/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/'` +if eval "test \"\${ac_cv_prog_cc_${ac_cc}_c_o+set}\" = set"; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +# Make sure it works both with $CC and with simple cc. +# We do the test twice because some compilers refuse to overwrite an +# existing .o file with -o, though they will create one. +ac_try='$CC -c conftest.$ac_ext -o conftest.$ac_objext >&5' +if { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + test -f conftest.$ac_objext && { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; +then + eval ac_cv_prog_cc_${ac_cc}_c_o=yes + if test "x$CC" != xcc; then + # Test first that cc exists at all. + if { ac_try='cc -c conftest.$ac_ext >&5' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_try='cc -c conftest.$ac_ext -o conftest.$ac_objext >&5' + if { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + test -f conftest.$ac_objext && { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; + then + # cc works too. + : + else + # cc exists but doesn't like -o. + eval ac_cv_prog_cc_${ac_cc}_c_o=no + fi + fi + fi +else + eval ac_cv_prog_cc_${ac_cc}_c_o=no +fi +rm -f conftest* + +fi +if eval "test \"`echo '$ac_cv_prog_cc_'${ac_cc}_c_o`\" = yes"; then + echo "$as_me:$LINENO: result: yes" >&5 +echo "${ECHO_T}yes" >&6 +else + echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6 + +cat >>confdefs.h <<\_ACEOF +#define NO_MINUS_C_MINUS_O 1 +_ACEOF + +fi + +# FIXME: we rely on the cache variable name because +# there is no other way. +set dummy $CC +ac_cc=`echo $2 | sed 's/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/'` +if eval "test \"`echo '$ac_cv_prog_cc_'${ac_cc}_c_o`\" != yes"; then + # Losing compiler, so override with the script. + # FIXME: It is wrong to rewrite CC. + # But if we don't then we get into trouble of one sort or another. + # A longer-term fix would be to have automake use am__CC in this case, + # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" + CC="$am_aux_dir/compile $CC" +fi + +# Find a good install program. We prefer a C program (faster), +# so one script is as good as another. But avoid the broken or +# incompatible versions: +# SysV /etc/install, /usr/sbin/install +# SunOS /usr/etc/install +# IRIX /sbin/install +# AIX /bin/install +# AmigaOS /C/install, which installs bootblocks on floppy discs +# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag +# AFS /usr/afsws/bin/install, which mishandles nonexistent args +# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" +# OS/2's system install, which has a completely different semantic +# ./install, which can be erroneously created by make from ./install.sh. +echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5 +echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6 +if test -z "$INSTALL"; then +if test "${ac_cv_path_install+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + # Account for people who put trailing slashes in PATH elements. +case $as_dir/ in + ./ | .// | /cC/* | \ + /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ + ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \ + /usr/ucb/* ) ;; + *) + # OSF1 and SCO ODT 3.0 have their own names for install. + # Don't use installbsd from OSF since it installs stuff as root + # by default. + for ac_prog in ginstall scoinst install; do + for ac_exec_ext in '' $ac_executable_extensions; do + if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then + if test $ac_prog = install && + grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # AIX install. It has an incompatible calling convention. + : + elif test $ac_prog = install && + grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # program-specific install script used by HP pwplus--don't use. + : + else + ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c" + break 3 + fi + fi + done + done + ;; +esac +done + + +fi + if test "${ac_cv_path_install+set}" = set; then + INSTALL=$ac_cv_path_install + else + # As a last resort, use the slow shell script. We don't cache a + # path for INSTALL within a source directory, because that will + # break other packages using the cache if that directory is + # removed, or if the path is relative. + INSTALL=$ac_install_sh + fi +fi +echo "$as_me:$LINENO: result: $INSTALL" >&5 +echo "${ECHO_T}$INSTALL" >&6 + +# Use test -z because SunOS4 sh mishandles braces in ${var-val}. +# It thinks the first close brace ends the variable substitution. +test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' + +test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' + +test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' + + + +# Check whether --with-srilm or --without-srilm was given. +if test "${with_srilm+set}" = set; then + withval="$with_srilm" + with_srilm=$withval +else + with_srilm=no + +fi; + +# Check whether --enable-64bit or --disable-64bit was given. +if test "${enable_64bit+set}" = set; then + enableval="$enable_64bit" + +fi; +if test "x$enable_64bit" == "xyes" +then + org_cflags=$CFLAGS + CFLAGS=-m64 + +if test "$cross_compiling" = yes; then + { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling +See \`config.log' for more details." >&5 +echo "$as_me: error: cannot run test program while cross compiling +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ +return sizeof(void*) == 8 ? 0 : 1; + + ; + return 0; +} + +_ACEOF +rm -f conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + + CFLAGS="-m64 $org_cflags" + +else + echo "$as_me: program exited with status $ac_status" >&5 +echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +( exit $ac_status ) + + { { echo "$as_me:$LINENO: error: Don't know how to build a 64-bit object." >&5 +echo "$as_me: error: Don't know how to build a 64-bit object." >&2;} + { (exit 1); exit 1; }; } + +fi +rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi +fi + +trylibeventdir="" + +# Check whether --with-libevent or --without-libevent was given. +if test "${with_libevent+set}" = set; then + withval="$with_libevent" + + if test "x$withval" != "xno" ; then + trylibeventdir=$withval + fi + + +fi; + +if test "x$with_srilm" != 'xno' +then + SAVE_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS -I${with_srilm}/include" + + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5 +echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6 +# On Suns, sometimes $CPP names a directory. +if test -n "$CPP" && test -d "$CPP"; then + CPP= +fi +if test -z "$CPP"; then + if test "${ac_cv_prog_CPP+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + # Double quotes because CPP needs to be expanded + for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" + do + ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + # <limits.h> exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + Syntax error +_ACEOF +if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5 + (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null; then + if test -s conftest.err; then + ac_cpp_err=$ac_c_preproc_warn_flag + ac_cpp_err=$ac_cpp_err$ac_c_werror_flag + else + ac_cpp_err= + fi +else + ac_cpp_err=yes +fi +if test -z "$ac_cpp_err"; then + : +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.$ac_ext + + # OK, works on sane cases. Now check whether non-existent headers + # can be detected and how. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <ac_nonexistent.h> +_ACEOF +if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5 + (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null; then + if test -s conftest.err; then + ac_cpp_err=$ac_c_preproc_warn_flag + ac_cpp_err=$ac_cpp_err$ac_c_werror_flag + else + ac_cpp_err= + fi +else + ac_cpp_err=yes +fi +if test -z "$ac_cpp_err"; then + # Broken: success on invalid input. +continue +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.err conftest.$ac_ext +if $ac_preproc_ok; then + break +fi + + done + ac_cv_prog_CPP=$CPP + +fi + CPP=$ac_cv_prog_CPP +else + ac_cv_prog_CPP=$CPP +fi +echo "$as_me:$LINENO: result: $CPP" >&5 +echo "${ECHO_T}$CPP" >&6 +ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + # <limits.h> exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + Syntax error +_ACEOF +if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5 + (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null; then + if test -s conftest.err; then + ac_cpp_err=$ac_c_preproc_warn_flag + ac_cpp_err=$ac_cpp_err$ac_c_werror_flag + else + ac_cpp_err= + fi +else + ac_cpp_err=yes +fi +if test -z "$ac_cpp_err"; then + : +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.$ac_ext + + # OK, works on sane cases. Now check whether non-existent headers + # can be detected and how. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <ac_nonexistent.h> +_ACEOF +if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5 + (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null; then + if test -s conftest.err; then + ac_cpp_err=$ac_c_preproc_warn_flag + ac_cpp_err=$ac_cpp_err$ac_c_werror_flag + else + ac_cpp_err= + fi +else + ac_cpp_err=yes +fi +if test -z "$ac_cpp_err"; then + # Broken: success on invalid input. +continue +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.err conftest.$ac_ext +if $ac_preproc_ok; then + : +else + { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details." >&5 +echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +echo "$as_me:$LINENO: checking for egrep" >&5 +echo $ECHO_N "checking for egrep... $ECHO_C" >&6 +if test "${ac_cv_prog_egrep+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if echo a | (grep -E '(a|b)') >/dev/null 2>&1 + then ac_cv_prog_egrep='grep -E' + else ac_cv_prog_egrep='egrep' + fi +fi +echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5 +echo "${ECHO_T}$ac_cv_prog_egrep" >&6 + EGREP=$ac_cv_prog_egrep + + +echo "$as_me:$LINENO: checking for ANSI C header files" >&5 +echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6 +if test "${ac_cv_header_stdc+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <stdlib.h> +#include <stdarg.h> +#include <string.h> +#include <float.h> + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_header_stdc=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_header_stdc=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext + +if test $ac_cv_header_stdc = yes; then + # SunOS 4.x string.h does not declare mem*, contrary to ANSI. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <string.h> + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "memchr" >/dev/null 2>&1; then + : +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <stdlib.h> + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "free" >/dev/null 2>&1; then + : +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. + if test "$cross_compiling" = yes; then + : +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <ctype.h> +#if ((' ' & 0x0FF) == 0x020) +# define ISLOWER(c) ('a' <= (c) && (c) <= 'z') +# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) +#else +# define ISLOWER(c) \ + (('a' <= (c) && (c) <= 'i') \ + || ('j' <= (c) && (c) <= 'r') \ + || ('s' <= (c) && (c) <= 'z')) +# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) +#endif + +#define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) +int +main () +{ + int i; + for (i = 0; i < 256; i++) + if (XOR (islower (i), ISLOWER (i)) + || toupper (i) != TOUPPER (i)) + exit(2); + exit (0); +} +_ACEOF +rm -f conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + : +else + echo "$as_me: program exited with status $ac_status" >&5 +echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +( exit $ac_status ) +ac_cv_header_stdc=no +fi +rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi +fi +fi +echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5 +echo "${ECHO_T}$ac_cv_header_stdc" >&6 +if test $ac_cv_header_stdc = yes; then + +cat >>confdefs.h <<\_ACEOF +#define STDC_HEADERS 1 +_ACEOF + +fi + +# On IRIX 5.3, sys/types and inttypes.h are conflicting. + + + + + + + + + +for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ + inttypes.h stdint.h unistd.h +do +as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh` +echo "$as_me:$LINENO: checking for $ac_header" >&5 +echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6 +if eval "test \"\${$as_ac_Header+set}\" = set"; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default + +#include <$ac_header> +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + eval "$as_ac_Header=yes" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +eval "$as_ac_Header=no" +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5 +echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6 +if test `eval echo '${'$as_ac_Header'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + +fi + +done + + +if test "${ac_cv_header_Prob_h+set}" = set; then + echo "$as_me:$LINENO: checking for Prob.h" >&5 +echo $ECHO_N "checking for Prob.h... $ECHO_C" >&6 +if test "${ac_cv_header_Prob_h+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +fi +echo "$as_me:$LINENO: result: $ac_cv_header_Prob_h" >&5 +echo "${ECHO_T}$ac_cv_header_Prob_h" >&6 +else + # Is the header compilable? +echo "$as_me:$LINENO: checking Prob.h usability" >&5 +echo $ECHO_N "checking Prob.h usability... $ECHO_C" >&6 +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +#include <Prob.h> +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_header_compiler=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_header_compiler=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +echo "${ECHO_T}$ac_header_compiler" >&6 + +# Is the header present? +echo "$as_me:$LINENO: checking Prob.h presence" >&5 +echo $ECHO_N "checking Prob.h presence... $ECHO_C" >&6 +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <Prob.h> +_ACEOF +if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5 + (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null; then + if test -s conftest.err; then + ac_cpp_err=$ac_c_preproc_warn_flag + ac_cpp_err=$ac_cpp_err$ac_c_werror_flag + else + ac_cpp_err= + fi +else + ac_cpp_err=yes +fi +if test -z "$ac_cpp_err"; then + ac_header_preproc=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi +rm -f conftest.err conftest.$ac_ext +echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +echo "${ECHO_T}$ac_header_preproc" >&6 + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in + yes:no: ) + { echo "$as_me:$LINENO: WARNING: Prob.h: accepted by the compiler, rejected by the preprocessor!" >&5 +echo "$as_me: WARNING: Prob.h: accepted by the compiler, rejected by the preprocessor!" >&2;} + { echo "$as_me:$LINENO: WARNING: Prob.h: proceeding with the compiler's result" >&5 +echo "$as_me: WARNING: Prob.h: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { echo "$as_me:$LINENO: WARNING: Prob.h: present but cannot be compiled" >&5 +echo "$as_me: WARNING: Prob.h: present but cannot be compiled" >&2;} + { echo "$as_me:$LINENO: WARNING: Prob.h: check for missing prerequisite headers?" >&5 +echo "$as_me: WARNING: Prob.h: check for missing prerequisite headers?" >&2;} + { echo "$as_me:$LINENO: WARNING: Prob.h: see the Autoconf documentation" >&5 +echo "$as_me: WARNING: Prob.h: see the Autoconf documentation" >&2;} + { echo "$as_me:$LINENO: WARNING: Prob.h: section \"Present But Cannot Be Compiled\"" >&5 +echo "$as_me: WARNING: Prob.h: section \"Present But Cannot Be Compiled\"" >&2;} + { echo "$as_me:$LINENO: WARNING: Prob.h: proceeding with the preprocessor's result" >&5 +echo "$as_me: WARNING: Prob.h: proceeding with the preprocessor's result" >&2;} + { echo "$as_me:$LINENO: WARNING: Prob.h: in the future, the compiler will take precedence" >&5 +echo "$as_me: WARNING: Prob.h: in the future, the compiler will take precedence" >&2;} + ( + cat <<\_ASBOX +## ----------------------------------- ## +## Report this to the lmserver lists. ## +## ----------------------------------- ## +_ASBOX + ) | + sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +echo "$as_me:$LINENO: checking for Prob.h" >&5 +echo $ECHO_N "checking for Prob.h... $ECHO_C" >&6 +if test "${ac_cv_header_Prob_h+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_cv_header_Prob_h=$ac_header_preproc +fi +echo "$as_me:$LINENO: result: $ac_cv_header_Prob_h" >&5 +echo "${ECHO_T}$ac_cv_header_Prob_h" >&6 + +fi +if test $ac_cv_header_Prob_h = yes; then + +cat >>confdefs.h <<\_ACEOF +#define HAVE_SRILM +_ACEOF + +else + { { echo "$as_me:$LINENO: error: Cannot find SRILM!" >&5 +echo "$as_me: error: Cannot find SRILM!" >&2;} + { (exit 1); exit 1; }; } +fi + + + + LIB_SRILM="-loolm -ldstruct -lmisc" + # ROOT/lib/i686-m64/liboolm.a + # ROOT/lib/i686-m64/libdstruct.a + # ROOT/lib/i686-m64/libmisc.a + MY_ARCH=`${with_srilm}/sbin/machine-type` + LDFLAGS="$LDFLAGS -L${with_srilm}/lib/${MY_ARCH}" + LIBS="$LIBS $LIB_SRILM" + FMTLIBS="$FMTLIBS liboolm.a libdstruct.a libmisc.a" + + +if true; then + SRI_LM_TRUE= + SRI_LM_FALSE='#' +else + SRI_LM_TRUE='#' + SRI_LM_FALSE= +fi + +fi + + + +LIBEVENT_URL=http://www.monkey.org/~provos/libevent/ + +echo "$as_me:$LINENO: checking for libevent directory" >&5 +echo $ECHO_N "checking for libevent directory... $ECHO_C" >&6 +if test "${ac_cv_libevent_dir+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + + saved_LIBS="$LIBS" + saved_LDFLAGS="$LDFLAGS" + saved_CPPFLAGS="$CPPFLAGS" + le_found=no + for ledir in $trylibeventdir "" $prefix /usr/local ; do + LDFLAGS="$saved_LDFLAGS" + LIBS="$saved_LIBS -levent" + + # Skip the directory if it isn't there. + if test ! -z "$ledir" -a ! -d "$ledir" ; then + continue; + fi + if test ! -z "$ledir" ; then + if test -d "$ledir/lib" ; then + LDFLAGS="-L$ledir/lib $LDFLAGS" + else + LDFLAGS="-L$ledir $LDFLAGS" + fi + if test -d "$ledir/include" ; then + CPPFLAGS="-I$ledir/include $CPPFLAGS" + else + CPPFLAGS="-I$ledir $CPPFLAGS" + fi + fi + # Can I compile and link it? + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <sys/time.h> +#include <sys/types.h> +#include <event.h> +int +main () +{ + event_init(); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + libevent_linked=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + libevent_linked=no +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + if test $libevent_linked = yes; then + if test ! -z "$ledir" ; then + ac_cv_libevent_dir=$ledir + else + ac_cv_libevent_dir="(system)" + fi + le_found=yes + break + fi + done + LIBS="$saved_LIBS" + LDFLAGS="$saved_LDFLAGS" + CPPFLAGS="$saved_CPPFLAGS" + if test $le_found = no ; then + { { echo "$as_me:$LINENO: error: libevent is required. You can get it from $LIBEVENT_URL + + If it's already installed, specify its path using --with-libevent=/dir/ +" >&5 +echo "$as_me: error: libevent is required. You can get it from $LIBEVENT_URL + + If it's already installed, specify its path using --with-libevent=/dir/ +" >&2;} + { (exit 1); exit 1; }; } + fi + +fi +echo "$as_me:$LINENO: result: $ac_cv_libevent_dir" >&5 +echo "${ECHO_T}$ac_cv_libevent_dir" >&6 +LIBS="$LIBS -levent" +if test $ac_cv_libevent_dir != "(system)"; then + if test -d "$ac_cv_libevent_dir/lib" ; then + LDFLAGS="-L$ac_cv_libevent_dir/lib $LDFLAGS" + le_libdir="$ac_cv_libevent_dir/lib" + else + LDFLAGS="-L$ac_cv_libevent_dir $LDFLAGS" + le_libdir="$ac_cv_libevent_dir" + fi + if test -d "$ac_cv_libevent_dir/include" ; then + CPPFLAGS="-I$ac_cv_libevent_dir/include $CPPFLAGS" + else + CPPFLAGS="-I$ac_cv_libevent_dir $CPPFLAGS" + fi +fi + + +echo "$as_me:$LINENO: checking for library containing socket" >&5 +echo $ECHO_N "checking for library containing socket... $ECHO_C" >&6 +if test "${ac_cv_search_socket+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_func_search_save_LIBS=$LIBS +ac_cv_search_socket=no +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char socket (); +int +main () +{ +socket (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_socket="none required" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +if test "$ac_cv_search_socket" = no; then + for ac_lib in socket; do + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char socket (); +int +main () +{ +socket (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_socket="-l$ac_lib" +break +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + done +fi +LIBS=$ac_func_search_save_LIBS +fi +echo "$as_me:$LINENO: result: $ac_cv_search_socket" >&5 +echo "${ECHO_T}$ac_cv_search_socket" >&6 +if test "$ac_cv_search_socket" != no; then + test "$ac_cv_search_socket" = "none required" || LIBS="$ac_cv_search_socket $LIBS" + +fi + +echo "$as_me:$LINENO: checking for library containing gethostbyname" >&5 +echo $ECHO_N "checking for library containing gethostbyname... $ECHO_C" >&6 +if test "${ac_cv_search_gethostbyname+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_func_search_save_LIBS=$LIBS +ac_cv_search_gethostbyname=no +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char gethostbyname (); +int +main () +{ +gethostbyname (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_gethostbyname="none required" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +if test "$ac_cv_search_gethostbyname" = no; then + for ac_lib in nsl; do + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char gethostbyname (); +int +main () +{ +gethostbyname (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_gethostbyname="-l$ac_lib" +break +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + done +fi +LIBS=$ac_func_search_save_LIBS +fi +echo "$as_me:$LINENO: result: $ac_cv_search_gethostbyname" >&5 +echo "${ECHO_T}$ac_cv_search_gethostbyname" >&6 +if test "$ac_cv_search_gethostbyname" != no; then + test "$ac_cv_search_gethostbyname" = "none required" || LIBS="$ac_cv_search_gethostbyname $LIBS" + +fi + +echo "$as_me:$LINENO: checking for library containing mallinfo" >&5 +echo $ECHO_N "checking for library containing mallinfo... $ECHO_C" >&6 +if test "${ac_cv_search_mallinfo+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_func_search_save_LIBS=$LIBS +ac_cv_search_mallinfo=no +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char mallinfo (); +int +main () +{ +mallinfo (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_mallinfo="none required" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +if test "$ac_cv_search_mallinfo" = no; then + for ac_lib in malloc; do + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char mallinfo (); +int +main () +{ +mallinfo (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_mallinfo="-l$ac_lib" +break +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + done +fi +LIBS=$ac_func_search_save_LIBS +fi +echo "$as_me:$LINENO: result: $ac_cv_search_mallinfo" >&5 +echo "${ECHO_T}$ac_cv_search_mallinfo" >&6 +if test "$ac_cv_search_mallinfo" != no; then + test "$ac_cv_search_mallinfo" = "none required" || LIBS="$ac_cv_search_mallinfo $LIBS" + +fi + + +echo "$as_me:$LINENO: checking for daemon" >&5 +echo $ECHO_N "checking for daemon... $ECHO_C" >&6 +if test "${ac_cv_func_daemon+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +/* Define daemon to an innocuous variant, in case <limits.h> declares daemon. + For example, HP-UX 11i <limits.h> declares gettimeofday. */ +#define daemon innocuous_daemon + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char daemon (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef daemon + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +{ +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char daemon (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined (__stub_daemon) || defined (__stub___daemon) +choke me +#else +char (*f) () = daemon; +#endif +#ifdef __cplusplus +} +#endif + +int +main () +{ +return f != daemon; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_func_daemon=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_func_daemon=no +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_func_daemon" >&5 +echo "${ECHO_T}$ac_cv_func_daemon" >&6 +if test $ac_cv_func_daemon = yes; then + +cat >>confdefs.h <<\_ACEOF +#define HAVE_DAEMON +_ACEOF + +else + DAEMON_OBJ=daemon.o +fi + + + +echo "$as_me:$LINENO: checking for stdbool.h that conforms to C99" >&5 +echo $ECHO_N "checking for stdbool.h that conforms to C99... $ECHO_C" >&6 +if test "${ac_cv_header_stdbool_h+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +#include <stdbool.h> +#ifndef bool +# error bool is not defined +#endif +#ifndef false +# error false is not defined +#endif +#if false +# error false is not 0 +#endif +#ifndef true +# error true is not defined +#endif +#if true != 1 +# error true is not 1 +#endif +#ifndef __bool_true_false_are_defined +# error __bool_true_false_are_defined is not defined +#endif + + struct s { _Bool s: 1; _Bool t; } s; + + char a[true == 1 ? 1 : -1]; + char b[false == 0 ? 1 : -1]; + char c[__bool_true_false_are_defined == 1 ? 1 : -1]; + char d[(bool) -0.5 == true ? 1 : -1]; + bool e = &s; + char f[(_Bool) -0.0 == false ? 1 : -1]; + char g[true]; + char h[sizeof (_Bool)]; + char i[sizeof s.t]; + +int +main () +{ + return !a + !b + !c + !d + !e + !f + !g + !h + !i; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_header_stdbool_h=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_header_stdbool_h=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_header_stdbool_h" >&5 +echo "${ECHO_T}$ac_cv_header_stdbool_h" >&6 +echo "$as_me:$LINENO: checking for _Bool" >&5 +echo $ECHO_N "checking for _Bool... $ECHO_C" >&6 +if test "${ac_cv_type__Bool+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ +if ((_Bool *) 0) + return 0; +if (sizeof (_Bool)) + return 0; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_type__Bool=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_type__Bool=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_type__Bool" >&5 +echo "${ECHO_T}$ac_cv_type__Bool" >&6 +if test $ac_cv_type__Bool = yes; then + +cat >>confdefs.h <<_ACEOF +#define HAVE__BOOL 1 +_ACEOF + + +fi + +if test $ac_cv_header_stdbool_h = yes; then + +cat >>confdefs.h <<\_ACEOF +#define HAVE_STDBOOL_H 1 +_ACEOF + +fi + +echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5 +echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6 +if test "${ac_cv_c_const+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ +/* FIXME: Include the comments suggested by Paul. */ +#ifndef __cplusplus + /* Ultrix mips cc rejects this. */ + typedef int charset[2]; + const charset x; + /* SunOS 4.1.1 cc rejects this. */ + char const *const *ccp; + char **p; + /* NEC SVR4.0.2 mips cc rejects this. */ + struct point {int x, y;}; + static struct point const zero = {0,0}; + /* AIX XL C 1.02.0.0 rejects this. + It does not let you subtract one const X* pointer from another in + an arm of an if-expression whose if-part is not a constant + expression */ + const char *g = "string"; + ccp = &g + (g ? g-g : 0); + /* HPUX 7.0 cc rejects these. */ + ++ccp; + p = (char**) ccp; + ccp = (char const *const *) p; + { /* SCO 3.2v4 cc rejects this. */ + char *t; + char const *s = 0 ? (char *) 0 : (char const *) 0; + + *t++ = 0; + } + { /* Someone thinks the Sun supposedly-ANSI compiler will reject this. */ + int x[] = {25, 17}; + const int *foo = &x[0]; + ++foo; + } + { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */ + typedef const int *iptr; + iptr p = 0; + ++p; + } + { /* AIX XL C 1.02.0.0 rejects this saying + "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */ + struct s { int j; const int *ap[3]; }; + struct s *b; b->j = 5; + } + { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */ + const int foo = 10; + } +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_c_const=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_c_const=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5 +echo "${ECHO_T}$ac_cv_c_const" >&6 +if test $ac_cv_c_const = no; then + +cat >>confdefs.h <<\_ACEOF +#define const +_ACEOF + +fi + +if test "${ac_cv_header_malloc_h+set}" = set; then + echo "$as_me:$LINENO: checking for malloc.h" >&5 +echo $ECHO_N "checking for malloc.h... $ECHO_C" >&6 +if test "${ac_cv_header_malloc_h+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +fi +echo "$as_me:$LINENO: result: $ac_cv_header_malloc_h" >&5 +echo "${ECHO_T}$ac_cv_header_malloc_h" >&6 +else + # Is the header compilable? +echo "$as_me:$LINENO: checking malloc.h usability" >&5 +echo $ECHO_N "checking malloc.h usability... $ECHO_C" >&6 +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +#include <malloc.h> +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_header_compiler=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_header_compiler=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +echo "${ECHO_T}$ac_header_compiler" >&6 + +# Is the header present? +echo "$as_me:$LINENO: checking malloc.h presence" >&5 +echo $ECHO_N "checking malloc.h presence... $ECHO_C" >&6 +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include <malloc.h> +_ACEOF +if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5 + (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null; then + if test -s conftest.err; then + ac_cpp_err=$ac_c_preproc_warn_flag + ac_cpp_err=$ac_cpp_err$ac_c_werror_flag + else + ac_cpp_err= + fi +else + ac_cpp_err=yes +fi +if test -z "$ac_cpp_err"; then + ac_header_preproc=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi +rm -f conftest.err conftest.$ac_ext +echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +echo "${ECHO_T}$ac_header_preproc" >&6 + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in + yes:no: ) + { echo "$as_me:$LINENO: WARNING: malloc.h: accepted by the compiler, rejected by the preprocessor!" >&5 +echo "$as_me: WARNING: malloc.h: accepted by the compiler, rejected by the preprocessor!" >&2;} + { echo "$as_me:$LINENO: WARNING: malloc.h: proceeding with the compiler's result" >&5 +echo "$as_me: WARNING: malloc.h: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { echo "$as_me:$LINENO: WARNING: malloc.h: present but cannot be compiled" >&5 +echo "$as_me: WARNING: malloc.h: present but cannot be compiled" >&2;} + { echo "$as_me:$LINENO: WARNING: malloc.h: check for missing prerequisite headers?" >&5 +echo "$as_me: WARNING: malloc.h: check for missing prerequisite headers?" >&2;} + { echo "$as_me:$LINENO: WARNING: malloc.h: see the Autoconf documentation" >&5 +echo "$as_me: WARNING: malloc.h: see the Autoconf documentation" >&2;} + { echo "$as_me:$LINENO: WARNING: malloc.h: section \"Present But Cannot Be Compiled\"" >&5 +echo "$as_me: WARNING: malloc.h: section \"Present But Cannot Be Compiled\"" >&2;} + { echo "$as_me:$LINENO: WARNING: malloc.h: proceeding with the preprocessor's result" >&5 +echo "$as_me: WARNING: malloc.h: proceeding with the preprocessor's result" >&2;} + { echo "$as_me:$LINENO: WARNING: malloc.h: in the future, the compiler will take precedence" >&5 +echo "$as_me: WARNING: malloc.h: in the future, the compiler will take precedence" >&2;} + ( + cat <<\_ASBOX +## ----------------------------------- ## +## Report this to the lmserver lists. ## +## ----------------------------------- ## +_ASBOX + ) | + sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +echo "$as_me:$LINENO: checking for malloc.h" >&5 +echo $ECHO_N "checking for malloc.h... $ECHO_C" >&6 +if test "${ac_cv_header_malloc_h+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_cv_header_malloc_h=$ac_header_preproc +fi +echo "$as_me:$LINENO: result: $ac_cv_header_malloc_h" >&5 +echo "${ECHO_T}$ac_cv_header_malloc_h" >&6 + +fi +if test $ac_cv_header_malloc_h = yes; then + +cat >>confdefs.h <<\_ACEOF +#define HAVE_MALLOC_H +_ACEOF + +fi + + +echo "$as_me:$LINENO: checking for struct mallinfo.arena" >&5 +echo $ECHO_N "checking for struct mallinfo.arena... $ECHO_C" >&6 +if test "${ac_cv_member_struct_mallinfo_arena+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +# include <malloc.h> + + + +int +main () +{ +static struct mallinfo ac_aggr; +if (ac_aggr.arena) +return 0; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_member_struct_mallinfo_arena=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +# include <malloc.h> + + + +int +main () +{ +static struct mallinfo ac_aggr; +if (sizeof ac_aggr.arena) +return 0; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_member_struct_mallinfo_arena=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_member_struct_mallinfo_arena=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_member_struct_mallinfo_arena" >&5 +echo "${ECHO_T}$ac_cv_member_struct_mallinfo_arena" >&6 +if test $ac_cv_member_struct_mallinfo_arena = yes; then + + +cat >>confdefs.h <<\_ACEOF +#define HAVE_STRUCT_MALLINFO +_ACEOF + + +fi + + + + +echo "$as_me:$LINENO: checking for socklen_t" >&5 +echo $ECHO_N "checking for socklen_t... $ECHO_C" >&6 +if test "${ac_cv_c_socklen_t+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + + #include <sys/types.h> + #include <sys/socket.h> + +int +main () +{ + + socklen_t foo; + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + + ac_cv_c_socklen_t=yes + +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + + ac_cv_c_socklen_t=no + +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext + +fi +echo "$as_me:$LINENO: result: $ac_cv_c_socklen_t" >&5 +echo "${ECHO_T}$ac_cv_c_socklen_t" >&6 +if test $ac_cv_c_socklen_t = no; then + +cat >>confdefs.h <<\_ACEOF +#define socklen_t int +_ACEOF + +fi + + + + +echo "$as_me:$LINENO: checking for endianness" >&5 +echo $ECHO_N "checking for endianness... $ECHO_C" >&6 +if test "${ac_cv_c_endian+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + + if test "$cross_compiling" = yes; then + { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling +See \`config.log' for more details." >&5 +echo "$as_me: error: cannot run test program while cross compiling +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + long val = 1; + char *c = (char *) &val; + exit(*c == 1); + + ; + return 0; +} + +_ACEOF +rm -f conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + + ac_cv_c_endian=big + +else + echo "$as_me: program exited with status $ac_status" >&5 +echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +( exit $ac_status ) + + ac_cv_c_endian=little + +fi +rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi + +fi +echo "$as_me:$LINENO: result: $ac_cv_c_endian" >&5 +echo "${ECHO_T}$ac_cv_c_endian" >&6 +if test $ac_cv_c_endian = big; then + +cat >>confdefs.h <<\_ACEOF +#define ENDIAN_BIG 1 +_ACEOF + +fi +if test $ac_cv_c_endian = little; then + +cat >>confdefs.h <<\_ACEOF +#define ENDIAN_LITTLE 1 +_ACEOF + +fi + + +# Check whether --enable-threads or --disable-threads was given. +if test "${enable_threads+set}" = set; then + enableval="$enable_threads" + +fi; +if test "x$enable_threads" == "xyes"; then + echo "$as_me:$LINENO: checking for library containing pthread_create" >&5 +echo $ECHO_N "checking for library containing pthread_create... $ECHO_C" >&6 +if test "${ac_cv_search_pthread_create+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_func_search_save_LIBS=$LIBS +ac_cv_search_pthread_create=no +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char pthread_create (); +int +main () +{ +pthread_create (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_pthread_create="none required" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +if test "$ac_cv_search_pthread_create" = no; then + for ac_lib in pthread; do + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char pthread_create (); +int +main () +{ +pthread_create (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_search_pthread_create="-l$ac_lib" +break +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + done +fi +LIBS=$ac_func_search_save_LIBS +fi +echo "$as_me:$LINENO: result: $ac_cv_search_pthread_create" >&5 +echo "${ECHO_T}$ac_cv_search_pthread_create" >&6 +if test "$ac_cv_search_pthread_create" != no; then + test "$ac_cv_search_pthread_create" = "none required" || LIBS="$ac_cv_search_pthread_create $LIBS" + +fi + + if test "x$ac_cv_search_pthread_create" != "xno"; then + +cat >>confdefs.h <<\_ACEOF +#define USE_THREADS +_ACEOF + + if test "$cross_compiling" = yes; then + { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling +See \`config.log' for more details." >&5 +echo "$as_me: error: cannot run test program while cross compiling +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ +#ifdef __SUNPRO_C + return 0; +#else + return 1; +#endif + + ; + return 0; +} + +_ACEOF +rm -f conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + + CFLAGS="-mt $CFLAGS" + +else + echo "$as_me: program exited with status $ac_status" >&5 +echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi + else + { { echo "$as_me:$LINENO: error: Can't enable threads without the POSIX thread library." >&5 +echo "$as_me: error: Can't enable threads without the POSIX thread library." >&2;} + { (exit 1); exit 1; }; } + fi +fi + + +for ac_func in mlockall +do +as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh` +echo "$as_me:$LINENO: checking for $ac_func" >&5 +echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6 +if eval "test \"\${$as_ac_var+set}\" = set"; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func. + For example, HP-UX 11i <limits.h> declares gettimeofday. */ +#define $ac_func innocuous_$ac_func + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char $ac_func (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef $ac_func + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +{ +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char $ac_func (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined (__stub_$ac_func) || defined (__stub___$ac_func) +choke me +#else +char (*f) () = $ac_func; +#endif +#ifdef __cplusplus +} +#endif + +int +main () +{ +return f != $ac_func; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + eval "$as_ac_var=yes" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +eval "$as_ac_var=no" +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5 +echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6 +if test `eval echo '${'$as_ac_var'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1 +_ACEOF + +fi +done + + +for ac_func in getpagesizes +do +as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh` +echo "$as_me:$LINENO: checking for $ac_func" >&5 +echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6 +if eval "test \"\${$as_ac_var+set}\" = set"; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func. + For example, HP-UX 11i <limits.h> declares gettimeofday. */ +#define $ac_func innocuous_$ac_func + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char $ac_func (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef $ac_func + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +{ +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char $ac_func (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined (__stub_$ac_func) || defined (__stub___$ac_func) +choke me +#else +char (*f) () = $ac_func; +#endif +#ifdef __cplusplus +} +#endif + +int +main () +{ +return f != $ac_func; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + eval "$as_ac_var=yes" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +eval "$as_ac_var=no" +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5 +echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6 +if test `eval echo '${'$as_ac_var'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1 +_ACEOF + +fi +done + + +for ac_func in memcntl +do +as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh` +echo "$as_me:$LINENO: checking for $ac_func" >&5 +echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6 +if eval "test \"\${$as_ac_var+set}\" = set"; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func. + For example, HP-UX 11i <limits.h> declares gettimeofday. */ +#define $ac_func innocuous_$ac_func + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char $ac_func (); below. + Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + <limits.h> exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + +#undef $ac_func + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +{ +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char $ac_func (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined (__stub_$ac_func) || defined (__stub___$ac_func) +choke me +#else +char (*f) () = $ac_func; +#endif +#ifdef __cplusplus +} +#endif + +int +main () +{ +return f != $ac_func; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + eval "$as_ac_var=yes" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +eval "$as_ac_var=no" +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5 +echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6 +if test `eval echo '${'$as_ac_var'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1 +_ACEOF + +fi +done + + + ac_config_files="$ac_config_files Makefile" + +cat >confcache <<\_ACEOF +# This file is a shell script that caches the results of configure +# tests run on this system so they can be shared between configure +# scripts and configure runs, see configure's option --config-cache. +# It is not useful on other systems. If it contains results you don't +# want to keep, you may remove or edit it. +# +# config.status only pays attention to the cache file if you give it +# the --recheck option to rerun configure. +# +# `ac_cv_env_foo' variables (set or unset) will be overridden when +# loading this file, other *unset* `ac_cv_foo' will be assigned the +# following values. + +_ACEOF + +# The following way of writing the cache mishandles newlines in values, +# but we know of no workaround that is simple, portable, and efficient. +# So, don't put newlines in cache variables' values. +# Ultrix sh set writes to stderr and can't be redirected directly, +# and sets the high bit in the cache file unless we assign to the vars. +{ + (set) 2>&1 | + case `(ac_space=' '; set | grep ac_space) 2>&1` in + *ac_space=\ *) + # `set' does not quote correctly, so add quotes (double-quote + # substitution turns \\\\ into \\, and sed turns \\ into \). + sed -n \ + "s/'/'\\\\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" + ;; + *) + # `set' quotes correctly as required by POSIX, so do not add quotes. + sed -n \ + "s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p" + ;; + esac; +} | + sed ' + t clear + : clear + s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ + t end + /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ + : end' >>confcache +if diff $cache_file confcache >/dev/null 2>&1; then :; else + if test -w $cache_file; then + test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file" + cat confcache >$cache_file + else + echo "not updating unwritable cache $cache_file" + fi +fi +rm -f confcache + +test "x$prefix" = xNONE && prefix=$ac_default_prefix +# Let make expand exec_prefix. +test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' + +# VPATH may cause trouble with some makes, so we remove $(srcdir), +# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and +# trailing colons and then remove the whole line if VPATH becomes empty +# (actually we leave an empty line to preserve line numbers). +if test "x$srcdir" = x.; then + ac_vpsub='/^[ ]*VPATH[ ]*=/{ +s/:*\$(srcdir):*/:/; +s/:*\${srcdir}:*/:/; +s/:*@srcdir@:*/:/; +s/^\([^=]*=[ ]*\):*/\1/; +s/:*$//; +s/^[^=]*=[ ]*$//; +}' +fi + +DEFS=-DHAVE_CONFIG_H + +ac_libobjs= +ac_ltlibobjs= +for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue + # 1. Remove the extension, and $U if already installed. + ac_i=`echo "$ac_i" | + sed 's/\$U\././;s/\.o$//;s/\.obj$//'` + # 2. Add them. + ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext" + ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo' +done +LIBOBJS=$ac_libobjs + +LTLIBOBJS=$ac_ltlibobjs + + +if test -z "${AMDEP_TRUE}" && test -z "${AMDEP_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"AMDEP\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +echo "$as_me: error: conditional \"AMDEP\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi +if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"am__fastdepCC\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +echo "$as_me: error: conditional \"am__fastdepCC\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi +if test -z "${am__fastdepCXX_TRUE}" && test -z "${am__fastdepCXX_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"am__fastdepCXX\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +echo "$as_me: error: conditional \"am__fastdepCXX\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi +if test -z "${SRI_LM_TRUE}" && test -z "${SRI_LM_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"SRI_LM\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +echo "$as_me: error: conditional \"SRI_LM\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi + +: ${CONFIG_STATUS=./config.status} +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files $CONFIG_STATUS" +{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5 +echo "$as_me: creating $CONFIG_STATUS" >&6;} +cat >$CONFIG_STATUS <<_ACEOF +#! $SHELL +# Generated by $as_me. +# Run this file to recreate the current configuration. +# Compiler output produced by configure, useful for debugging +# configure, is in config.log if it exists. + +debug=false +ac_cs_recheck=false +ac_cs_silent=false +SHELL=\${CONFIG_SHELL-$SHELL} +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF +## --------------------- ## +## M4sh Initialization. ## +## --------------------- ## + +# Be Bourne compatible +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' +elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then + set -o posix +fi +DUALCASE=1; export DUALCASE # for MKS sh + +# Support unset when possible. +if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then + as_unset=unset +else + as_unset=false +fi + + +# Work around bugs in pre-3.0 UWIN ksh. +$as_unset ENV MAIL MAILPATH +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +for as_var in \ + LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \ + LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \ + LC_TELEPHONE LC_TIME +do + if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then + eval $as_var=C; export $as_var + else + $as_unset $as_var + fi +done + +# Required to use basename. +if expr a : '\(a\)' >/dev/null 2>&1; then + as_expr=expr +else + as_expr=false +fi + +if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + + +# Name of the executable. +as_me=`$as_basename "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)$' \| \ + . : '\(.\)' 2>/dev/null || +echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; } + /^X\/\(\/\/\)$/{ s//\1/; q; } + /^X\/\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + + +# PATH needs CR, and LINENO needs CR and PATH. +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + echo "#! /bin/sh" >conf$$.sh + echo "exit 0" >>conf$$.sh + chmod +x conf$$.sh + if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then + PATH_SEPARATOR=';' + else + PATH_SEPARATOR=: + fi + rm -f conf$$.sh +fi + + + as_lineno_1=$LINENO + as_lineno_2=$LINENO + as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x$as_lineno_3" = "x$as_lineno_2" || { + # Find who we are. Look in the path if we contain no path at all + # relative or not. + case $0 in + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break +done + + ;; + esac + # We did not find ourselves, most probably we were run as `sh COMMAND' + # in which case we are not to be found in the path. + if test "x$as_myself" = x; then + as_myself=$0 + fi + if test ! -f "$as_myself"; then + { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5 +echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;} + { (exit 1); exit 1; }; } + fi + case $CONFIG_SHELL in + '') + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for as_base in sh bash ksh sh5; do + case $as_dir in + /*) + if ("$as_dir/$as_base" -c ' + as_lineno_1=$LINENO + as_lineno_2=$LINENO + as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x$as_lineno_3" = "x$as_lineno_2" ') 2>/dev/null; then + $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; } + $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; } + CONFIG_SHELL=$as_dir/$as_base + export CONFIG_SHELL + exec "$CONFIG_SHELL" "$0" ${1+"$@"} + fi;; + esac + done +done +;; + esac + + # Create $as_me.lineno as a copy of $as_myself, but with $LINENO + # uniformly replaced by the line number. The first 'sed' inserts a + # line-number line before each line; the second 'sed' does the real + # work. The second script uses 'N' to pair each line-number line + # with the numbered line, and appends trailing '-' during + # substitution so that $LINENO is not a special case at line end. + # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the + # second 'sed' script. Blame Lee E. McMahon for sed's syntax. :-) + sed '=' <$as_myself | + sed ' + N + s,$,-, + : loop + s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3, + t loop + s,-$,, + s,^['$as_cr_digits']*\n,, + ' >$as_me.lineno && + chmod +x $as_me.lineno || + { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5 +echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;} + { (exit 1); exit 1; }; } + + # Don't try to exec as it changes $[0], causing all sort of problems + # (the dirname of $[0] is not the place where we might find the + # original and so on. Autoconf is especially sensible to this). + . ./$as_me.lineno + # Exit status is that of the last command. + exit +} + + +case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in + *c*,-n*) ECHO_N= ECHO_C=' +' ECHO_T=' ' ;; + *c*,* ) ECHO_N=-n ECHO_C= ECHO_T= ;; + *) ECHO_N= ECHO_C='\c' ECHO_T= ;; +esac + +if expr a : '\(a\)' >/dev/null 2>&1; then + as_expr=expr +else + as_expr=false +fi + +rm -f conf$$ conf$$.exe conf$$.file +echo >conf$$.file +if ln -s conf$$.file conf$$ 2>/dev/null; then + # We could just check for DJGPP; but this test a) works b) is more generic + # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04). + if test -f conf$$.exe; then + # Don't use ln at all; we don't have any links + as_ln_s='cp -p' + else + as_ln_s='ln -s' + fi +elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.file + +if mkdir -p . 2>/dev/null; then + as_mkdir_p=: +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +as_executable_p="test -f" + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + +# IFS +# We need space, tab and new line, in precisely that order. +as_nl=' +' +IFS=" $as_nl" + +# CDPATH. +$as_unset CDPATH + +exec 6>&1 + +# Open the log real soon, to keep \$[0] and so on meaningful, and to +# report actual input values of CONFIG_FILES etc. instead of their +# values after options handling. Logging --version etc. is OK. +exec 5>>config.log +{ + echo + sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX +## Running $as_me. ## +_ASBOX +} >&5 +cat >&5 <<_CSEOF + +This file was extended by lmserver $as_me 1.0, which was +generated by GNU Autoconf 2.59. Invocation command line was + + CONFIG_FILES = $CONFIG_FILES + CONFIG_HEADERS = $CONFIG_HEADERS + CONFIG_LINKS = $CONFIG_LINKS + CONFIG_COMMANDS = $CONFIG_COMMANDS + $ $0 $@ + +_CSEOF +echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5 +echo >&5 +_ACEOF + +# Files that config.status was made for. +if test -n "$ac_config_files"; then + echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS +fi + +if test -n "$ac_config_headers"; then + echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS +fi + +if test -n "$ac_config_links"; then + echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS +fi + +if test -n "$ac_config_commands"; then + echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS +fi + +cat >>$CONFIG_STATUS <<\_ACEOF + +ac_cs_usage="\ +\`$as_me' instantiates files from templates according to the +current configuration. + +Usage: $0 [OPTIONS] [FILE]... + + -h, --help print this help, then exit + -V, --version print version number, then exit + -q, --quiet do not print progress messages + -d, --debug don't remove temporary files + --recheck update $as_me by reconfiguring in the same conditions + --file=FILE[:TEMPLATE] + instantiate the configuration file FILE + --header=FILE[:TEMPLATE] + instantiate the configuration header FILE + +Configuration files: +$config_files + +Configuration headers: +$config_headers + +Configuration commands: +$config_commands + +Report bugs to <bug-autoconf@gnu.org>." +_ACEOF + +cat >>$CONFIG_STATUS <<_ACEOF +ac_cs_version="\\ +lmserver config.status 1.0 +configured by $0, generated by GNU Autoconf 2.59, + with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\" + +Copyright (C) 2003 Free Software Foundation, Inc. +This config.status script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it." +srcdir=$srcdir +INSTALL="$INSTALL" +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF +# If no file are specified by the user, then we need to provide default +# value. By we need to know if files were specified by the user. +ac_need_defaults=: +while test $# != 0 +do + case $1 in + --*=*) + ac_option=`expr "x$1" : 'x\([^=]*\)='` + ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'` + ac_shift=: + ;; + -*) + ac_option=$1 + ac_optarg=$2 + ac_shift=shift + ;; + *) # This is not an option, so the user has probably given explicit + # arguments. + ac_option=$1 + ac_need_defaults=false;; + esac + + case $ac_option in + # Handling of the options. +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF + -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) + ac_cs_recheck=: ;; + --version | --vers* | -V ) + echo "$ac_cs_version"; exit 0 ;; + --he | --h) + # Conflict between --help and --header + { { echo "$as_me:$LINENO: error: ambiguous option: $1 +Try \`$0 --help' for more information." >&5 +echo "$as_me: error: ambiguous option: $1 +Try \`$0 --help' for more information." >&2;} + { (exit 1); exit 1; }; };; + --help | --hel | -h ) + echo "$ac_cs_usage"; exit 0 ;; + --debug | --d* | -d ) + debug=: ;; + --file | --fil | --fi | --f ) + $ac_shift + CONFIG_FILES="$CONFIG_FILES $ac_optarg" + ac_need_defaults=false;; + --header | --heade | --head | --hea ) + $ac_shift + CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg" + ac_need_defaults=false;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil | --si | --s) + ac_cs_silent=: ;; + + # This is an error. + -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1 +Try \`$0 --help' for more information." >&5 +echo "$as_me: error: unrecognized option: $1 +Try \`$0 --help' for more information." >&2;} + { (exit 1); exit 1; }; } ;; + + *) ac_config_targets="$ac_config_targets $1" ;; + + esac + shift +done + +ac_configure_extra_args= + +if $ac_cs_silent; then + exec 6>/dev/null + ac_configure_extra_args="$ac_configure_extra_args --silent" +fi + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF +if \$ac_cs_recheck; then + echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6 + exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion +fi + +_ACEOF + +cat >>$CONFIG_STATUS <<_ACEOF +# +# INIT-COMMANDS section. +# + +AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir" + +_ACEOF + + + +cat >>$CONFIG_STATUS <<\_ACEOF +for ac_config_target in $ac_config_targets +do + case "$ac_config_target" in + # Handling of arguments. + "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;; + "depfiles" ) CONFIG_COMMANDS="$CONFIG_COMMANDS depfiles" ;; + "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; + *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5 +echo "$as_me: error: invalid argument: $ac_config_target" >&2;} + { (exit 1); exit 1; }; };; + esac +done + +# If the user did not use the arguments to specify the items to instantiate, +# then the envvar interface is used. Set only those that are not. +# We use the long form for the default assignment because of an extremely +# bizarre bug on SunOS 4.1.3. +if $ac_need_defaults; then + test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files + test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers + test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands +fi + +# Have a temporary directory for convenience. Make it in the build tree +# simply because there is no reason to put it here, and in addition, +# creating and moving files from /tmp can sometimes cause problems. +# Create a temporary directory, and hook for its removal unless debugging. +$debug || +{ + trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0 + trap '{ (exit 1); exit 1; }' 1 2 13 15 +} + +# Create a (secure) tmp directory for tmp files. + +{ + tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` && + test -n "$tmp" && test -d "$tmp" +} || +{ + tmp=./confstat$$-$RANDOM + (umask 077 && mkdir $tmp) +} || +{ + echo "$me: cannot create a temporary directory in ." >&2 + { (exit 1); exit 1; } +} + +_ACEOF + +cat >>$CONFIG_STATUS <<_ACEOF + +# +# CONFIG_FILES section. +# + +# No need to generate the scripts if there are no CONFIG_FILES. +# This happens for instance when ./config.status config.h +if test -n "\$CONFIG_FILES"; then + # Protect against being on the right side of a sed subst in config.status. + sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g; + s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF +s,@SHELL@,$SHELL,;t t +s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t +s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t +s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t +s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t +s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t +s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t +s,@exec_prefix@,$exec_prefix,;t t +s,@prefix@,$prefix,;t t +s,@program_transform_name@,$program_transform_name,;t t +s,@bindir@,$bindir,;t t +s,@sbindir@,$sbindir,;t t +s,@libexecdir@,$libexecdir,;t t +s,@datadir@,$datadir,;t t +s,@sysconfdir@,$sysconfdir,;t t +s,@sharedstatedir@,$sharedstatedir,;t t +s,@localstatedir@,$localstatedir,;t t +s,@libdir@,$libdir,;t t +s,@includedir@,$includedir,;t t +s,@oldincludedir@,$oldincludedir,;t t +s,@infodir@,$infodir,;t t +s,@mandir@,$mandir,;t t +s,@build_alias@,$build_alias,;t t +s,@host_alias@,$host_alias,;t t +s,@target_alias@,$target_alias,;t t +s,@DEFS@,$DEFS,;t t +s,@ECHO_C@,$ECHO_C,;t t +s,@ECHO_N@,$ECHO_N,;t t +s,@ECHO_T@,$ECHO_T,;t t +s,@LIBS@,$LIBS,;t t +s,@build@,$build,;t t +s,@build_cpu@,$build_cpu,;t t +s,@build_vendor@,$build_vendor,;t t +s,@build_os@,$build_os,;t t +s,@host@,$host,;t t +s,@host_cpu@,$host_cpu,;t t +s,@host_vendor@,$host_vendor,;t t +s,@host_os@,$host_os,;t t +s,@target@,$target,;t t +s,@target_cpu@,$target_cpu,;t t +s,@target_vendor@,$target_vendor,;t t +s,@target_os@,$target_os,;t t +s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t +s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t +s,@INSTALL_DATA@,$INSTALL_DATA,;t t +s,@CYGPATH_W@,$CYGPATH_W,;t t +s,@PACKAGE@,$PACKAGE,;t t +s,@VERSION@,$VERSION,;t t +s,@ACLOCAL@,$ACLOCAL,;t t +s,@AUTOCONF@,$AUTOCONF,;t t +s,@AUTOMAKE@,$AUTOMAKE,;t t +s,@AUTOHEADER@,$AUTOHEADER,;t t +s,@MAKEINFO@,$MAKEINFO,;t t +s,@install_sh@,$install_sh,;t t +s,@STRIP@,$STRIP,;t t +s,@ac_ct_STRIP@,$ac_ct_STRIP,;t t +s,@INSTALL_STRIP_PROGRAM@,$INSTALL_STRIP_PROGRAM,;t t +s,@mkdir_p@,$mkdir_p,;t t +s,@AWK@,$AWK,;t t +s,@SET_MAKE@,$SET_MAKE,;t t +s,@am__leading_dot@,$am__leading_dot,;t t +s,@AMTAR@,$AMTAR,;t t +s,@am__tar@,$am__tar,;t t +s,@am__untar@,$am__untar,;t t +s,@CC@,$CC,;t t +s,@CFLAGS@,$CFLAGS,;t t +s,@LDFLAGS@,$LDFLAGS,;t t +s,@CPPFLAGS@,$CPPFLAGS,;t t +s,@ac_ct_CC@,$ac_ct_CC,;t t +s,@EXEEXT@,$EXEEXT,;t t +s,@OBJEXT@,$OBJEXT,;t t +s,@DEPDIR@,$DEPDIR,;t t +s,@am__include@,$am__include,;t t +s,@am__quote@,$am__quote,;t t +s,@AMDEP_TRUE@,$AMDEP_TRUE,;t t +s,@AMDEP_FALSE@,$AMDEP_FALSE,;t t +s,@AMDEPBACKSLASH@,$AMDEPBACKSLASH,;t t +s,@CCDEPMODE@,$CCDEPMODE,;t t +s,@am__fastdepCC_TRUE@,$am__fastdepCC_TRUE,;t t +s,@am__fastdepCC_FALSE@,$am__fastdepCC_FALSE,;t t +s,@CXX@,$CXX,;t t +s,@CXXFLAGS@,$CXXFLAGS,;t t +s,@ac_ct_CXX@,$ac_ct_CXX,;t t +s,@CXXDEPMODE@,$CXXDEPMODE,;t t +s,@am__fastdepCXX_TRUE@,$am__fastdepCXX_TRUE,;t t +s,@am__fastdepCXX_FALSE@,$am__fastdepCXX_FALSE,;t t +s,@CPP@,$CPP,;t t +s,@EGREP@,$EGREP,;t t +s,@SRI_LM_TRUE@,$SRI_LM_TRUE,;t t +s,@SRI_LM_FALSE@,$SRI_LM_FALSE,;t t +s,@DAEMON_OBJ@,$DAEMON_OBJ,;t t +s,@LIBOBJS@,$LIBOBJS,;t t +s,@LTLIBOBJS@,$LTLIBOBJS,;t t +CEOF + +_ACEOF + + cat >>$CONFIG_STATUS <<\_ACEOF + # Split the substitutions into bite-sized pieces for seds with + # small command number limits, like on Digital OSF/1 and HP-UX. + ac_max_sed_lines=48 + ac_sed_frag=1 # Number of current file. + ac_beg=1 # First line for current file. + ac_end=$ac_max_sed_lines # Line after last line for current file. + ac_more_lines=: + ac_sed_cmds= + while $ac_more_lines; do + if test $ac_beg -gt 1; then + sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag + else + sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag + fi + if test ! -s $tmp/subs.frag; then + ac_more_lines=false + else + # The purpose of the label and of the branching condition is to + # speed up the sed processing (if there are no `@' at all, there + # is no need to browse any of the substitutions). + # These are the two extra sed commands mentioned above. + (echo ':t + /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed + if test -z "$ac_sed_cmds"; then + ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed" + else + ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed" + fi + ac_sed_frag=`expr $ac_sed_frag + 1` + ac_beg=$ac_end + ac_end=`expr $ac_end + $ac_max_sed_lines` + fi + done + if test -z "$ac_sed_cmds"; then + ac_sed_cmds=cat + fi +fi # test -n "$CONFIG_FILES" + +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF +for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue + # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in". + case $ac_file in + - | *:- | *:-:* ) # input from stdin + cat >$tmp/stdin + ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + * ) ac_file_in=$ac_file.in ;; + esac + + # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories. + ac_dir=`(dirname "$ac_file") 2>/dev/null || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p "$ac_dir" + else + as_dir="$ac_dir" + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5 +echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;} + { (exit 1); exit 1; }; }; } + + ac_builddir=. + +if test "$ac_dir" != .; then + ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'` + # A "../" for each directory in $ac_dir_suffix. + ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'` +else + ac_dir_suffix= ac_top_builddir= +fi + +case $srcdir in + .) # No --srcdir option. We are building in place. + ac_srcdir=. + if test -z "$ac_top_builddir"; then + ac_top_srcdir=. + else + ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'` + fi ;; + [\\/]* | ?:[\\/]* ) # Absolute path. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir ;; + *) # Relative path. + ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_builddir$srcdir ;; +esac + +# Do not use `cd foo && pwd` to compute absolute paths, because +# the directories may not exist. +case `pwd` in +.) ac_abs_builddir="$ac_dir";; +*) + case "$ac_dir" in + .) ac_abs_builddir=`pwd`;; + [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";; + *) ac_abs_builddir=`pwd`/"$ac_dir";; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_builddir=${ac_top_builddir}.;; +*) + case ${ac_top_builddir}. in + .) ac_abs_top_builddir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;; + *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_srcdir=$ac_srcdir;; +*) + case $ac_srcdir in + .) ac_abs_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;; + *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_srcdir=$ac_top_srcdir;; +*) + case $ac_top_srcdir in + .) ac_abs_top_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;; + *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;; + esac;; +esac + + + case $INSTALL in + [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;; + *) ac_INSTALL=$ac_top_builddir$INSTALL ;; + esac + + if test x"$ac_file" != x-; then + { echo "$as_me:$LINENO: creating $ac_file" >&5 +echo "$as_me: creating $ac_file" >&6;} + rm -f "$ac_file" + fi + # Let's still pretend it is `configure' which instantiates (i.e., don't + # use $as_me), people would be surprised to read: + # /* config.h. Generated by config.status. */ + if test x"$ac_file" = x-; then + configure_input= + else + configure_input="$ac_file. " + fi + configure_input=$configure_input"Generated from `echo $ac_file_in | + sed 's,.*/,,'` by configure." + + # First look for the input files in the build tree, otherwise in the + # src tree. + ac_file_inputs=`IFS=: + for f in $ac_file_in; do + case $f in + -) echo $tmp/stdin ;; + [\\/$]*) + # Absolute (can't be DOS-style, as IFS=:) + test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + echo "$f";; + *) # Relative + if test -f "$f"; then + # Build tree + echo "$f" + elif test -f "$srcdir/$f"; then + # Source tree + echo "$srcdir/$f" + else + # /dev/null tree + { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + fi;; + esac + done` || { (exit 1); exit 1; } +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF + sed "$ac_vpsub +$extrasub +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF +:t +/@[a-zA-Z_][a-zA-Z_0-9]*@/!b +s,@configure_input@,$configure_input,;t t +s,@srcdir@,$ac_srcdir,;t t +s,@abs_srcdir@,$ac_abs_srcdir,;t t +s,@top_srcdir@,$ac_top_srcdir,;t t +s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t +s,@builddir@,$ac_builddir,;t t +s,@abs_builddir@,$ac_abs_builddir,;t t +s,@top_builddir@,$ac_top_builddir,;t t +s,@abs_top_builddir@,$ac_abs_top_builddir,;t t +s,@INSTALL@,$ac_INSTALL,;t t +" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out + rm -f $tmp/stdin + if test x"$ac_file" != x-; then + mv $tmp/out $ac_file + else + cat $tmp/out + rm -f $tmp/out + fi + +done +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF + +# +# CONFIG_HEADER section. +# + +# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where +# NAME is the cpp macro being defined and VALUE is the value it is being given. +# +# ac_d sets the value in "#define NAME VALUE" lines. +ac_dA='s,^\([ ]*\)#\([ ]*define[ ][ ]*\)' +ac_dB='[ ].*$,\1#\2' +ac_dC=' ' +ac_dD=',;t' +# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE". +ac_uA='s,^\([ ]*\)#\([ ]*\)undef\([ ][ ]*\)' +ac_uB='$,\1#\2define\3' +ac_uC=' ' +ac_uD=',;t' + +for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue + # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in". + case $ac_file in + - | *:- | *:-:* ) # input from stdin + cat >$tmp/stdin + ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; + * ) ac_file_in=$ac_file.in ;; + esac + + test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5 +echo "$as_me: creating $ac_file" >&6;} + + # First look for the input files in the build tree, otherwise in the + # src tree. + ac_file_inputs=`IFS=: + for f in $ac_file_in; do + case $f in + -) echo $tmp/stdin ;; + [\\/$]*) + # Absolute (can't be DOS-style, as IFS=:) + test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + # Do quote $f, to prevent DOS paths from being IFS'd. + echo "$f";; + *) # Relative + if test -f "$f"; then + # Build tree + echo "$f" + elif test -f "$srcdir/$f"; then + # Source tree + echo "$srcdir/$f" + else + # /dev/null tree + { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 +echo "$as_me: error: cannot find input file: $f" >&2;} + { (exit 1); exit 1; }; } + fi;; + esac + done` || { (exit 1); exit 1; } + # Remove the trailing spaces. + sed 's/[ ]*$//' $ac_file_inputs >$tmp/in + +_ACEOF + +# Transform confdefs.h into two sed scripts, `conftest.defines' and +# `conftest.undefs', that substitutes the proper values into +# config.h.in to produce config.h. The first handles `#define' +# templates, and the second `#undef' templates. +# And first: Protect against being on the right side of a sed subst in +# config.status. Protect against being in an unquoted here document +# in config.status. +rm -f conftest.defines conftest.undefs +# Using a here document instead of a string reduces the quoting nightmare. +# Putting comments in sed scripts is not portable. +# +# `end' is used to avoid that the second main sed command (meant for +# 0-ary CPP macros) applies to n-ary macro definitions. +# See the Autoconf documentation for `clear'. +cat >confdef2sed.sed <<\_ACEOF +s/[\\&,]/\\&/g +s,[\\$`],\\&,g +t clear +: clear +s,^[ ]*#[ ]*define[ ][ ]*\([^ (][^ (]*\)\(([^)]*)\)[ ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp +t end +s,^[ ]*#[ ]*define[ ][ ]*\([^ ][^ ]*\)[ ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp +: end +_ACEOF +# If some macros were called several times there might be several times +# the same #defines, which is useless. Nevertheless, we may not want to +# sort them, since we want the *last* AC-DEFINE to be honored. +uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines +sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs +rm -f confdef2sed.sed + +# This sed command replaces #undef with comments. This is necessary, for +# example, in the case of _POSIX_SOURCE, which is predefined and required +# on some systems where configure will not decide to define it. +cat >>conftest.undefs <<\_ACEOF +s,^[ ]*#[ ]*undef[ ][ ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */, +_ACEOF + +# Break up conftest.defines because some shells have a limit on the size +# of here documents, and old seds have small limits too (100 cmds). +echo ' # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS +echo ' if grep "^[ ]*#[ ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS +echo ' # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS +echo ' :' >>$CONFIG_STATUS +rm -f conftest.tail +while grep . conftest.defines >/dev/null +do + # Write a limited-size here document to $tmp/defines.sed. + echo ' cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS + # Speed up: don't consider the non `#define' lines. + echo '/^[ ]*#[ ]*define/!b' >>$CONFIG_STATUS + # Work around the forget-to-reset-the-flag bug. + echo 't clr' >>$CONFIG_STATUS + echo ': clr' >>$CONFIG_STATUS + sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS + echo 'CEOF + sed -f $tmp/defines.sed $tmp/in >$tmp/out + rm -f $tmp/in + mv $tmp/out $tmp/in +' >>$CONFIG_STATUS + sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail + rm -f conftest.defines + mv conftest.tail conftest.defines +done +rm -f conftest.defines +echo ' fi # grep' >>$CONFIG_STATUS +echo >>$CONFIG_STATUS + +# Break up conftest.undefs because some shells have a limit on the size +# of here documents, and old seds have small limits too (100 cmds). +echo ' # Handle all the #undef templates' >>$CONFIG_STATUS +rm -f conftest.tail +while grep . conftest.undefs >/dev/null +do + # Write a limited-size here document to $tmp/undefs.sed. + echo ' cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS + # Speed up: don't consider the non `#undef' + echo '/^[ ]*#[ ]*undef/!b' >>$CONFIG_STATUS + # Work around the forget-to-reset-the-flag bug. + echo 't clr' >>$CONFIG_STATUS + echo ': clr' >>$CONFIG_STATUS + sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS + echo 'CEOF + sed -f $tmp/undefs.sed $tmp/in >$tmp/out + rm -f $tmp/in + mv $tmp/out $tmp/in +' >>$CONFIG_STATUS + sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail + rm -f conftest.undefs + mv conftest.tail conftest.undefs +done +rm -f conftest.undefs + +cat >>$CONFIG_STATUS <<\_ACEOF + # Let's still pretend it is `configure' which instantiates (i.e., don't + # use $as_me), people would be surprised to read: + # /* config.h. Generated by config.status. */ + if test x"$ac_file" = x-; then + echo "/* Generated by configure. */" >$tmp/config.h + else + echo "/* $ac_file. Generated by configure. */" >$tmp/config.h + fi + cat $tmp/in >>$tmp/config.h + rm -f $tmp/in + if test x"$ac_file" != x-; then + if diff $ac_file $tmp/config.h >/dev/null 2>&1; then + { echo "$as_me:$LINENO: $ac_file is unchanged" >&5 +echo "$as_me: $ac_file is unchanged" >&6;} + else + ac_dir=`(dirname "$ac_file") 2>/dev/null || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p "$ac_dir" + else + as_dir="$ac_dir" + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5 +echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;} + { (exit 1); exit 1; }; }; } + + rm -f $ac_file + mv $tmp/config.h $ac_file + fi + else + cat $tmp/config.h + rm -f $tmp/config.h + fi +# Compute $ac_file's index in $config_headers. +_am_stamp_count=1 +for _am_header in $config_headers :; do + case $_am_header in + $ac_file | $ac_file:* ) + break ;; + * ) + _am_stamp_count=`expr $_am_stamp_count + 1` ;; + esac +done +echo "timestamp for $ac_file" >`(dirname $ac_file) 2>/dev/null || +$as_expr X$ac_file : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X$ac_file : 'X\(//\)[^/]' \| \ + X$ac_file : 'X\(//\)$' \| \ + X$ac_file : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X$ac_file | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'`/stamp-h$_am_stamp_count +done +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF + +# +# CONFIG_COMMANDS section. +# +for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue + ac_dest=`echo "$ac_file" | sed 's,:.*,,'` + ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'` + ac_dir=`(dirname "$ac_dest") 2>/dev/null || +$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_dest" : 'X\(//\)[^/]' \| \ + X"$ac_dest" : 'X\(//\)$' \| \ + X"$ac_dest" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$ac_dest" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p "$ac_dir" + else + as_dir="$ac_dir" + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5 +echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;} + { (exit 1); exit 1; }; }; } + + ac_builddir=. + +if test "$ac_dir" != .; then + ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'` + # A "../" for each directory in $ac_dir_suffix. + ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'` +else + ac_dir_suffix= ac_top_builddir= +fi + +case $srcdir in + .) # No --srcdir option. We are building in place. + ac_srcdir=. + if test -z "$ac_top_builddir"; then + ac_top_srcdir=. + else + ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'` + fi ;; + [\\/]* | ?:[\\/]* ) # Absolute path. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir ;; + *) # Relative path. + ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_builddir$srcdir ;; +esac + +# Do not use `cd foo && pwd` to compute absolute paths, because +# the directories may not exist. +case `pwd` in +.) ac_abs_builddir="$ac_dir";; +*) + case "$ac_dir" in + .) ac_abs_builddir=`pwd`;; + [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";; + *) ac_abs_builddir=`pwd`/"$ac_dir";; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_builddir=${ac_top_builddir}.;; +*) + case ${ac_top_builddir}. in + .) ac_abs_top_builddir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;; + *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_srcdir=$ac_srcdir;; +*) + case $ac_srcdir in + .) ac_abs_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;; + *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;; + esac;; +esac +case $ac_abs_builddir in +.) ac_abs_top_srcdir=$ac_top_srcdir;; +*) + case $ac_top_srcdir in + .) ac_abs_top_srcdir=$ac_abs_builddir;; + [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;; + *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;; + esac;; +esac + + + { echo "$as_me:$LINENO: executing $ac_dest commands" >&5 +echo "$as_me: executing $ac_dest commands" >&6;} + case $ac_dest in + depfiles ) test x"$AMDEP_TRUE" != x"" || for mf in $CONFIG_FILES; do + # Strip MF so we end up with the name of the file. + mf=`echo "$mf" | sed -e 's/:.*$//'` + # Check whether this is an Automake generated Makefile or not. + # We used to match only the files named `Makefile.in', but + # some people rename them; so instead we look at the file content. + # Grep'ing the first line is not enough: some people post-process + # each Makefile.in and add a new line on top of each file to say so. + # So let's grep whole file. + if grep '^#.*generated by automake' $mf > /dev/null 2>&1; then + dirpart=`(dirname "$mf") 2>/dev/null || +$as_expr X"$mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$mf" : 'X\(//\)[^/]' \| \ + X"$mf" : 'X\(//\)$' \| \ + X"$mf" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$mf" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + else + continue + fi + # Extract the definition of DEPDIR, am__include, and am__quote + # from the Makefile without running `make'. + DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` + test -z "$DEPDIR" && continue + am__include=`sed -n 's/^am__include = //p' < "$mf"` + test -z "am__include" && continue + am__quote=`sed -n 's/^am__quote = //p' < "$mf"` + # When using ansi2knr, U may be empty or an underscore; expand it + U=`sed -n 's/^U = //p' < "$mf"` + # Find all dependency output files, they are included files with + # $(DEPDIR) in their names. We invoke sed twice because it is the + # simplest approach to changing $(DEPDIR) to its actual value in the + # expansion. + for file in `sed -n " + s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ + sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do + # Make sure the directory exists. + test -f "$dirpart/$file" && continue + fdir=`(dirname "$file") 2>/dev/null || +$as_expr X"$file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$file" : 'X\(//\)[^/]' \| \ + X"$file" : 'X\(//\)$' \| \ + X"$file" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + { if $as_mkdir_p; then + mkdir -p $dirpart/$fdir + else + as_dir=$dirpart/$fdir + as_dirs= + while test ! -d "$as_dir"; do + as_dirs="$as_dir $as_dirs" + as_dir=`(dirname "$as_dir") 2>/dev/null || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| \ + . : '\(.\)' 2>/dev/null || +echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } + /^X\(\/\/\)[^/].*/{ s//\1/; q; } + /^X\(\/\/\)$/{ s//\1/; q; } + /^X\(\/\).*/{ s//\1/; q; } + s/.*/./; q'` + done + test ! -n "$as_dirs" || mkdir $as_dirs + fi || { { echo "$as_me:$LINENO: error: cannot create directory $dirpart/$fdir" >&5 +echo "$as_me: error: cannot create directory $dirpart/$fdir" >&2;} + { (exit 1); exit 1; }; }; } + + # echo "creating $dirpart/$file" + echo '# dummy' > "$dirpart/$file" + done +done + ;; + esac +done +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF + +{ (exit 0); exit 0; } +_ACEOF +chmod +x $CONFIG_STATUS +ac_clean_files=$ac_clean_files_save + + +# configure is writing to config.log, and then calls config.status. +# config.status does its own redirection, appending to config.log. +# Unfortunately, on DOS this fails, as config.log is still kept open +# by configure, so config.status won't be able to write to it; its +# output is simply discarded. So we exec the FD to /dev/null, +# effectively closing config.log, so it can be properly (re)opened and +# appended to by config.status. When coming back to configure, we +# need to make the FD available again. +if test "$no_create" != yes; then + ac_cs_success=: + ac_config_status_args= + test "$silent" = yes && + ac_config_status_args="$ac_config_status_args --quiet" + exec 5>/dev/null + $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false + exec 5>>config.log + # Use ||, not &&, to avoid exiting from the if with $? = 1, which + # would make configure fail if this is the last instruction. + $ac_cs_success || { (exit 1); exit 1; } +fi + diff --git a/contrib/lmserver/configure.ac b/contrib/lmserver/configure.ac new file mode 100644 index 000000000..62ab5dc02 --- /dev/null +++ b/contrib/lmserver/configure.ac @@ -0,0 +1,235 @@ +AC_PREREQ(2.52) +AC_INIT(lmserver, 1.0) +AC_CANONICAL_SYSTEM +AC_CONFIG_SRCDIR(lmserver.c) +AM_INIT_AUTOMAKE(AC_PACKAGE_NAME, AC_PACKAGE_VERSION) +AM_CONFIG_HEADER(config.h) + +AC_PROG_CC +AC_PROG_CXX +AM_PROG_CC_C_O +AC_PROG_INSTALL + +AC_ARG_WITH(srilm, + [AC_HELP_STRING([--with-srilm=PATH], [(optional) path to SRI's LM toolkit])], + [with_srilm=$withval], + [with_srilm=no] +) + +AC_ARG_ENABLE(64bit, + [AS_HELP_STRING([--enable-64bit],[build 64bit verison])]) +if test "x$enable_64bit" == "xyes" +then + org_cflags=$CFLAGS + CFLAGS=-m64 + AC_RUN_IFELSE( + [AC_LANG_PROGRAM([], [dnl +return sizeof(void*) == 8 ? 0 : 1; + ]) + ],[ + CFLAGS="-m64 $org_cflags" + ],[ + AC_MSG_ERROR([Don't know how to build a 64-bit object.]) + ]) +fi + +trylibeventdir="" +AC_ARG_WITH(libevent, + [ --with-libevent=PATH Specify path to libevent installation ], + [ + if test "x$withval" != "xno" ; then + trylibeventdir=$withval + fi + ] +) + +if test "x$with_srilm" != 'xno' +then + SAVE_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS -I${with_srilm}/include" + + AC_CHECK_HEADER(Prob.h, + [AC_DEFINE([HAVE_SRILM], [], [flag for SRILM])], + [AC_MSG_ERROR([Cannot find SRILM!])]) + + LIB_SRILM="-loolm -ldstruct -lmisc" + # ROOT/lib/i686-m64/liboolm.a + # ROOT/lib/i686-m64/libdstruct.a + # ROOT/lib/i686-m64/libmisc.a + MY_ARCH=`${with_srilm}/sbin/machine-type` + LDFLAGS="$LDFLAGS -L${with_srilm}/lib/${MY_ARCH}" + LIBS="$LIBS $LIB_SRILM" + FMTLIBS="$FMTLIBS liboolm.a libdstruct.a libmisc.a" + AM_CONDITIONAL([SRI_LM], true) +fi + + +dnl ------------------------------------------------------ +dnl libevent detection. swiped from Tor. modified a bit. + +LIBEVENT_URL=http://www.monkey.org/~provos/libevent/ + +AC_CACHE_CHECK([for libevent directory], ac_cv_libevent_dir, [ + saved_LIBS="$LIBS" + saved_LDFLAGS="$LDFLAGS" + saved_CPPFLAGS="$CPPFLAGS" + le_found=no + for ledir in $trylibeventdir "" $prefix /usr/local ; do + LDFLAGS="$saved_LDFLAGS" + LIBS="$saved_LIBS -levent" + + # Skip the directory if it isn't there. + if test ! -z "$ledir" -a ! -d "$ledir" ; then + continue; + fi + if test ! -z "$ledir" ; then + if test -d "$ledir/lib" ; then + LDFLAGS="-L$ledir/lib $LDFLAGS" + else + LDFLAGS="-L$ledir $LDFLAGS" + fi + if test -d "$ledir/include" ; then + CPPFLAGS="-I$ledir/include $CPPFLAGS" + else + CPPFLAGS="-I$ledir $CPPFLAGS" + fi + fi + # Can I compile and link it? + AC_TRY_LINK([#include <sys/time.h> +#include <sys/types.h> +#include <event.h>], [ event_init(); ], + [ libevent_linked=yes ], [ libevent_linked=no ]) + if test $libevent_linked = yes; then + if test ! -z "$ledir" ; then + ac_cv_libevent_dir=$ledir + else + ac_cv_libevent_dir="(system)" + fi + le_found=yes + break + fi + done + LIBS="$saved_LIBS" + LDFLAGS="$saved_LDFLAGS" + CPPFLAGS="$saved_CPPFLAGS" + if test $le_found = no ; then + AC_MSG_ERROR([libevent is required. You can get it from $LIBEVENT_URL + + If it's already installed, specify its path using --with-libevent=/dir/ +]) + fi +]) +LIBS="$LIBS -levent" +if test $ac_cv_libevent_dir != "(system)"; then + if test -d "$ac_cv_libevent_dir/lib" ; then + LDFLAGS="-L$ac_cv_libevent_dir/lib $LDFLAGS" + le_libdir="$ac_cv_libevent_dir/lib" + else + LDFLAGS="-L$ac_cv_libevent_dir $LDFLAGS" + le_libdir="$ac_cv_libevent_dir" + fi + if test -d "$ac_cv_libevent_dir/include" ; then + CPPFLAGS="-I$ac_cv_libevent_dir/include $CPPFLAGS" + else + CPPFLAGS="-I$ac_cv_libevent_dir $CPPFLAGS" + fi +fi + +dnl ---------------------------------------------------------------------------- + +AC_SEARCH_LIBS(socket, socket) +AC_SEARCH_LIBS(gethostbyname, nsl) +AC_SEARCH_LIBS(mallinfo, malloc) + +AC_CHECK_FUNC(daemon,AC_DEFINE([HAVE_DAEMON],,[Define this if you have daemon()]),[DAEMON_OBJ=daemon.o]) +AC_SUBST(DAEMON_OBJ) + +AC_HEADER_STDBOOL +AC_C_CONST +AC_CHECK_HEADER(malloc.h, AC_DEFINE(HAVE_MALLOC_H,,[do we have malloc.h?])) +AC_CHECK_MEMBER([struct mallinfo.arena], [ + AC_DEFINE(HAVE_STRUCT_MALLINFO,,[do we have stuct mallinfo?]) + ], ,[ +# include <malloc.h> + ] +) + +dnl From licq: Copyright (c) 2000 Dirk Mueller +dnl Check if the type socklen_t is defined anywhere +AC_DEFUN([AC_C_SOCKLEN_T], +[AC_CACHE_CHECK(for socklen_t, ac_cv_c_socklen_t, +[ + AC_TRY_COMPILE([ + #include <sys/types.h> + #include <sys/socket.h> + ],[ + socklen_t foo; + ],[ + ac_cv_c_socklen_t=yes + ],[ + ac_cv_c_socklen_t=no + ]) +]) +if test $ac_cv_c_socklen_t = no; then + AC_DEFINE(socklen_t, int, [define to int if socklen_t not available]) +fi +]) + +AC_C_SOCKLEN_T + +dnl Check if we're a little-endian or a big-endian system, needed by hash code +AC_DEFUN([AC_C_ENDIAN], +[AC_CACHE_CHECK(for endianness, ac_cv_c_endian, +[ + AC_RUN_IFELSE( + [AC_LANG_PROGRAM([], [dnl + long val = 1; + char *c = (char *) &val; + exit(*c == 1); + ]) + ],[ + ac_cv_c_endian=big + ],[ + ac_cv_c_endian=little + ]) +]) +if test $ac_cv_c_endian = big; then + AC_DEFINE(ENDIAN_BIG, 1, [machine is bigendian]) +fi +if test $ac_cv_c_endian = little; then + AC_DEFINE(ENDIAN_LITTLE, 1, [machine is littleendian]) +fi +]) + +AC_C_ENDIAN + +dnl Check whether the user wants threads or not +AC_ARG_ENABLE(threads, + [AS_HELP_STRING([--enable-threads],[support multithreaded execution])]) +if test "x$enable_threads" == "xyes"; then + AC_SEARCH_LIBS(pthread_create, pthread) + if test "x$ac_cv_search_pthread_create" != "xno"; then + AC_DEFINE([USE_THREADS],,[Define this if you want to use pthreads]) + dnl Sun compilers need the -mt flag! + AC_RUN_IFELSE( + [AC_LANG_PROGRAM([], [dnl +#ifdef __SUNPRO_C + return 0; +#else + return 1; +#endif + ]) + ],[ + CFLAGS="-mt $CFLAGS" + ]) + else + AC_MSG_ERROR([Can't enable threads without the POSIX thread library.]) + fi +fi + +AC_CHECK_FUNCS(mlockall) +AC_CHECK_FUNCS(getpagesizes) +AC_CHECK_FUNCS(memcntl) + +AC_CONFIG_FILES(Makefile) +AC_OUTPUT diff --git a/contrib/lmserver/daemon.c b/contrib/lmserver/daemon.c new file mode 100644 index 000000000..9cb7884d0 --- /dev/null +++ b/contrib/lmserver/daemon.c @@ -0,0 +1,68 @@ +/* $Header: /cvsroot/wikipedia/willow/src/bin/willow/daemon.c,v 1.1 2005/05/02 19:15:21 kateturner Exp $ */ +/* $NetBSD: daemon.c,v 1.9 2003/08/07 16:42:46 agc Exp $ */ +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined __SUNPRO_C || defined __DECC || defined __HP_cc +# pragma ident "@(#)$Header: /cvsroot/wikipedia/willow/src/bin/willow/daemon.c,v 1.1 2005/05/02 19:15:21 kateturner Exp $" +# pragma ident "$NetBSD: daemon.c,v 1.9 2003/08/07 16:42:46 agc Exp $" +#endif + +#include <fcntl.h> +#include <stdlib.h> +#include <unistd.h> + +int daemon(int nochdir, int noclose) +{ + int fd; + + switch (fork()) { + case -1: + return (-1); + case 0: + break; + default: + _exit(EXIT_SUCCESS); + } + + if (setsid() == -1) + return (-1); + + if (nochdir == 0) + (void)chdir("/"); + + if (noclose == 0 && (fd = open("/dev/null", O_RDWR, 0)) != -1) { + (void)dup2(fd, STDIN_FILENO); + (void)dup2(fd, STDOUT_FILENO); + (void)dup2(fd, STDERR_FILENO); + if (fd > STDERR_FILENO) + (void)close(fd); + } + return (0); +} diff --git a/contrib/lmserver/depcomp b/contrib/lmserver/depcomp new file mode 100755 index 000000000..e5f9736c7 --- /dev/null +++ b/contrib/lmserver/depcomp @@ -0,0 +1,589 @@ +#! /bin/sh +# depcomp - compile a program generating dependencies as side-effects + +scriptversion=2007-03-29.01 + +# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007 Free Software +# Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# Originally written by Alexandre Oliva <oliva@dcc.unicamp.br>. + +case $1 in + '') + echo "$0: No command. Try \`$0 --help' for more information." 1>&2 + exit 1; + ;; + -h | --h*) + cat <<\EOF +Usage: depcomp [--help] [--version] PROGRAM [ARGS] + +Run PROGRAMS ARGS to compile a file, generating dependencies +as side-effects. + +Environment variables: + depmode Dependency tracking mode. + source Source file read by `PROGRAMS ARGS'. + object Object file output by `PROGRAMS ARGS'. + DEPDIR directory where to store dependencies. + depfile Dependency file to output. + tmpdepfile Temporary file to use when outputing dependencies. + libtool Whether libtool is used (yes/no). + +Report bugs to <bug-automake@gnu.org>. +EOF + exit $? + ;; + -v | --v*) + echo "depcomp $scriptversion" + exit $? + ;; +esac + +if test -z "$depmode" || test -z "$source" || test -z "$object"; then + echo "depcomp: Variables source, object and depmode must be set" 1>&2 + exit 1 +fi + +# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po. +depfile=${depfile-`echo "$object" | + sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`} +tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`} + +rm -f "$tmpdepfile" + +# Some modes work just like other modes, but use different flags. We +# parameterize here, but still list the modes in the big case below, +# to make depend.m4 easier to write. Note that we *cannot* use a case +# here, because this file can only contain one case statement. +if test "$depmode" = hp; then + # HP compiler uses -M and no extra arg. + gccflag=-M + depmode=gcc +fi + +if test "$depmode" = dashXmstdout; then + # This is just like dashmstdout with a different argument. + dashmflag=-xM + depmode=dashmstdout +fi + +case "$depmode" in +gcc3) +## gcc 3 implements dependency tracking that does exactly what +## we want. Yay! Note: for some reason libtool 1.4 doesn't like +## it if -MD -MP comes after the -MF stuff. Hmm. +## Unfortunately, FreeBSD c89 acceptance of flags depends upon +## the command line argument order; so add the flags where they +## appear in depend2.am. Note that the slowdown incurred here +## affects only configure: in makefiles, %FASTDEP% shortcuts this. + for arg + do + case $arg in + -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;; + *) set fnord "$@" "$arg" ;; + esac + shift # fnord + shift # $arg + done + "$@" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + mv "$tmpdepfile" "$depfile" + ;; + +gcc) +## There are various ways to get dependency output from gcc. Here's +## why we pick this rather obscure method: +## - Don't want to use -MD because we'd like the dependencies to end +## up in a subdir. Having to rename by hand is ugly. +## (We might end up doing this anyway to support other compilers.) +## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like +## -MM, not -M (despite what the docs say). +## - Using -M directly means running the compiler twice (even worse +## than renaming). + if test -z "$gccflag"; then + gccflag=-MD, + fi + "$@" -Wp,"$gccflag$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + echo "$object : \\" > "$depfile" + alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz +## The second -e expression handles DOS-style file names with drive letters. + sed -e 's/^[^:]*: / /' \ + -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile" +## This next piece of magic avoids the `deleted header file' problem. +## The problem is that when a header file which appears in a .P file +## is deleted, the dependency causes make to die (because there is +## typically no way to rebuild the header). We avoid this by adding +## dummy dependencies for each header file. Too bad gcc doesn't do +## this for us directly. + tr ' ' ' +' < "$tmpdepfile" | +## Some versions of gcc put a space before the `:'. On the theory +## that the space means something, we add a space to the output as +## well. +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +hp) + # This case exists only to let depend.m4 do its work. It works by + # looking at the text of this script. This case will never be run, + # since it is checked for above. + exit 1 + ;; + +sgi) + if test "$libtool" = yes; then + "$@" "-Wp,-MDupdate,$tmpdepfile" + else + "$@" -MDupdate "$tmpdepfile" + fi + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + + if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files + echo "$object : \\" > "$depfile" + + # Clip off the initial element (the dependent). Don't try to be + # clever and replace this with sed code, as IRIX sed won't handle + # lines with more than a fixed number of characters (4096 in + # IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines; + # the IRIX cc adds comments like `#:fec' to the end of the + # dependency line. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \ + tr ' +' ' ' >> $depfile + echo >> $depfile + + # The second pass generates a dummy entry for each header file. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \ + >> $depfile + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +aix) + # The C for AIX Compiler uses -M and outputs the dependencies + # in a .u file. In older versions, this file always lives in the + # current directory. Also, the AIX compiler puts `$object:' at the + # start of each line; $object doesn't have directory information. + # Version 6 uses the directory in both cases. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + if test "$libtool" = yes; then + tmpdepfile1=$dir$base.u + tmpdepfile2=$base.u + tmpdepfile3=$dir.libs/$base.u + "$@" -Wc,-M + else + tmpdepfile1=$dir$base.u + tmpdepfile2=$dir$base.u + tmpdepfile3=$dir$base.u + "$@" -M + fi + stat=$? + + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + # Each line is of the form `foo.o: dependent.h'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile" + # That's a tab and a space in the []. + sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile" + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +icc) + # Intel's C compiler understands `-MD -MF file'. However on + # icc -MD -MF foo.d -c -o sub/foo.o sub/foo.c + # ICC 7.0 will fill foo.d with something like + # foo.o: sub/foo.c + # foo.o: sub/foo.h + # which is wrong. We want: + # sub/foo.o: sub/foo.c + # sub/foo.o: sub/foo.h + # sub/foo.c: + # sub/foo.h: + # ICC 7.1 will output + # foo.o: sub/foo.c sub/foo.h + # and will wrap long lines using \ : + # foo.o: sub/foo.c ... \ + # sub/foo.h ... \ + # ... + + "$@" -MD -MF "$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + # Each line is of the form `foo.o: dependent.h', + # or `foo.o: dep1.h dep2.h \', or ` dep3.h dep4.h \'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed "s,^[^:]*:,$object :," < "$tmpdepfile" > "$depfile" + # Some versions of the HPUX 10.20 sed can't process this invocation + # correctly. Breaking it into two sed invocations is a workaround. + sed 's,^[^:]*: \(.*\)$,\1,;s/^\\$//;/^$/d;/:$/d' < "$tmpdepfile" | + sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +hp2) + # The "hp" stanza above does not work with aCC (C++) and HP's ia64 + # compilers, which have integrated preprocessors. The correct option + # to use with these is +Maked; it writes dependencies to a file named + # 'foo.d', which lands next to the object file, wherever that + # happens to be. + # Much of this is similar to the tru64 case; see comments there. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + if test "$libtool" = yes; then + tmpdepfile1=$dir$base.d + tmpdepfile2=$dir.libs/$base.d + "$@" -Wc,+Maked + else + tmpdepfile1=$dir$base.d + tmpdepfile2=$dir$base.d + "$@" +Maked + fi + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile" + # Add `dependent.h:' lines. + sed -ne '2,${; s/^ *//; s/ \\*$//; s/$/:/; p;}' "$tmpdepfile" >> "$depfile" + else + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" "$tmpdepfile2" + ;; + +tru64) + # The Tru64 compiler uses -MD to generate dependencies as a side + # effect. `cc -MD -o foo.o ...' puts the dependencies into `foo.o.d'. + # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put + # dependencies in `foo.d' instead, so we check for that too. + # Subdirectories are respected. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + + if test "$libtool" = yes; then + # With Tru64 cc, shared objects can also be used to make a + # static library. This mechanism is used in libtool 1.4 series to + # handle both shared and static libraries in a single compilation. + # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d. + # + # With libtool 1.5 this exception was removed, and libtool now + # generates 2 separate objects for the 2 libraries. These two + # compilations output dependencies in $dir.libs/$base.o.d and + # in $dir$base.o.d. We have to check for both files, because + # one of the two compilations can be disabled. We should prefer + # $dir$base.o.d over $dir.libs/$base.o.d because the latter is + # automatically cleaned when .libs/ is deleted, while ignoring + # the former would cause a distcleancheck panic. + tmpdepfile1=$dir.libs/$base.lo.d # libtool 1.4 + tmpdepfile2=$dir$base.o.d # libtool 1.5 + tmpdepfile3=$dir.libs/$base.o.d # libtool 1.5 + tmpdepfile4=$dir.libs/$base.d # Compaq CCC V6.2-504 + "$@" -Wc,-MD + else + tmpdepfile1=$dir$base.o.d + tmpdepfile2=$dir$base.d + tmpdepfile3=$dir$base.d + tmpdepfile4=$dir$base.d + "$@" -MD + fi + + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile" + # That's a tab and a space in the []. + sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile" + else + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +#nosideeffect) + # This comment above is used by automake to tell side-effect + # dependency tracking mechanisms from slower ones. + +dashmstdout) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout, regardless of -o. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + test -z "$dashmflag" && dashmflag=-M + # Require at least two characters before searching for `:' + # in the target name. This is to cope with DOS-style filenames: + # a dependency such as `c:/foo/bar' could be seen as target `c' otherwise. + "$@" $dashmflag | + sed 's:^[ ]*[^: ][^:][^:]*\:[ ]*:'"$object"'\: :' > "$tmpdepfile" + rm -f "$depfile" + cat < "$tmpdepfile" > "$depfile" + tr ' ' ' +' < "$tmpdepfile" | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +dashXmstdout) + # This case only exists to satisfy depend.m4. It is never actually + # run, as this mode is specially recognized in the preamble. + exit 1 + ;; + +makedepend) + "$@" || exit $? + # Remove any Libtool call + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + # X makedepend + shift + cleared=no + for arg in "$@"; do + case $cleared in + no) + set ""; shift + cleared=yes ;; + esac + case "$arg" in + -D*|-I*) + set fnord "$@" "$arg"; shift ;; + # Strip any option that makedepend may not understand. Remove + # the object too, otherwise makedepend will parse it as a source file. + -*|$object) + ;; + *) + set fnord "$@" "$arg"; shift ;; + esac + done + obj_suffix="`echo $object | sed 's/^.*\././'`" + touch "$tmpdepfile" + ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@" + rm -f "$depfile" + cat < "$tmpdepfile" > "$depfile" + sed '1,2d' "$tmpdepfile" | tr ' ' ' +' | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" "$tmpdepfile".bak + ;; + +cpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + "$@" -E | + sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ + -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' | + sed '$ s: \\$::' > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + cat < "$tmpdepfile" >> "$depfile" + sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +msvisualcpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout, regardless of -o, + # because we must use -o when running libtool. + "$@" || exit $? + IFS=" " + for arg + do + case "$arg" in + "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI") + set fnord "$@" + shift + shift + ;; + *) + set fnord "$@" "$arg" + shift + shift + ;; + esac + done + "$@" -E | + sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::echo "`cygpath -u \\"\1\\"`":p' | sort | uniq > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + . "$tmpdepfile" | sed 's% %\\ %g' | sed -n '/^\(.*\)$/ s:: \1 \\:p' >> "$depfile" + echo " " >> "$depfile" + . "$tmpdepfile" | sed 's% %\\ %g' | sed -n '/^\(.*\)$/ s::\1\::p' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +none) + exec "$@" + ;; + +*) + echo "Unknown depmode $depmode" 1>&2 + exit 1 + ;; +esac + +exit 0 + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/contrib/lmserver/examples/LMClient.java b/contrib/lmserver/examples/LMClient.java new file mode 100644 index 000000000..ee1a42e28 --- /dev/null +++ b/contrib/lmserver/examples/LMClient.java @@ -0,0 +1,55 @@ +import java.io.DataInputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.Socket; +import java.net.URI; +import java.net.URISyntaxException; + +public class LMClient { + + private Socket sock; + private DataInputStream input; + private OutputStreamWriter output; + + public LMClient(URI u) throws IOException { + sock = new Socket(u.getHost(), u.getPort()); + System.err.println(sock); + input = new DataInputStream(sock.getInputStream()); + output = new OutputStreamWriter(sock.getOutputStream(), "UTF8"); + } + + public float wordLogProb(String word, String context) throws IOException { + return wordLogProb(word, context.split("\\s+")); + } + + public float wordLogProb(String word, String[] context) throws IOException { + StringBuffer sb = new StringBuffer(); + sb.append("prob "); + sb.append(word); + for (int i = context.length-1; i >= 0; --i) { + sb.append(' ').append(context[i]); + } + sb.append("\r\n"); + output.write(sb.toString()); + output.flush(); + byte b1 = input.readByte(); + byte b2 = input.readByte(); + byte b3 = input.readByte(); + byte b4 = input.readByte(); + Float f = Float.intBitsToFloat( (((b4 & 0xff) << 24) | ((b3 & 0xff) << 16) | ((b2 & 0xff) << 8) | (b1 & 0xff)) ); + input.readByte(); input.readByte(); + return f; + } + + public static void main(String[] args) { + try { + LMClient lm = new LMClient(new URI("lm://csubmit02.umiacs.umd.edu:6666")); + System.err.println(lm.wordLogProb("want", "<s> the old man")); + System.err.println(lm.wordLogProb("wants", "<s> the old man")); + } catch (URISyntaxException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/contrib/lmserver/examples/LMClient.pm b/contrib/lmserver/examples/LMClient.pm new file mode 100644 index 000000000..78f1e03ab --- /dev/null +++ b/contrib/lmserver/examples/LMClient.pm @@ -0,0 +1,37 @@ +package LMClient; + +use IO::Socket; + +sub new { + my ($class, $cstr) = @_; + my $self = {}; + $cstr =~ s/^!//; + my ($host, $port) = split /\:/, $cstr; + die "Please specify connection string as host:port" unless ($host && $port); + + $self->{'SOCK'} = new IO::Socket::INET( + PeerAddr => $host, + PeerPort => $port, + Proto => 'tcp') or die "Couldn't create connection to $host:$port -- is memcached running?\n"; + + bless $self, $class; + return $self; +} + +sub word_prob { + my ($self, $word, $context) = @_; + my @cwords = reverse split /\s+/, $context; + my $qstr = "prob $word @cwords"; + my $s = $self->{'SOCK'}; + print $s "$qstr\r\n"; + my $r = <$s>; + my $x= unpack "f", $r; + return $x; +} + +sub close { + my ($self) = @_; + close $self->{'SOCK'}; +} + +1; diff --git a/contrib/lmserver/examples/lmclient.cc b/contrib/lmserver/examples/lmclient.cc new file mode 100644 index 000000000..4ce5be380 --- /dev/null +++ b/contrib/lmserver/examples/lmclient.cc @@ -0,0 +1,103 @@ +#include "Prob.h" +#include "Ngram.h" +#include "Vocab.h" + +#include <sstream> +#include <string> +#include <iostream> +#include <stdio.h> +#include <unistd.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <netinet/in.h> +#include <netdb.h> +#include <string.h> +#include <map> + +struct Cache { + map<int, Cache> tree; + float prob; + Cache() : prob(0) {} +}; + +struct LMClient { + Vocab* voc; + int sock, port; + char *s; + struct hostent *hp; + struct sockaddr_in server; + char res[8]; + + LMClient(Vocab* v, const char* host) : voc(v), port(6666) { + s = strchr(host, ':'); + + if (s != NULL) { + *s = '\0'; + s+=1; + port = atoi(s); + } + + sock = socket(AF_INET, SOCK_STREAM, 0); + + hp = gethostbyname(host); + if (hp == NULL) { + fprintf(stderr, "unknown host %s\n", host); + exit(1); + } + + bzero((char *)&server, sizeof(server)); + bcopy(hp->h_addr, (char *)&server.sin_addr, hp->h_length); + server.sin_family = hp->h_addrtype; + server.sin_port = htons(port); + + int errors = 0; + while (connect(sock, (struct sockaddr *)&server, sizeof(server)) < 0) { + cerr << "Error: connect()\n"; + sleep(1); + errors++; + if (errors > 5) exit(1); + } + std::cerr << "Connected to LM on " << host << " on port " << port << std::endl; + } + float wordProb(int word, int* context) { + Cache* cur = &cache; + int i = 0; + while (context[i] > 0) { + cur = &cur->tree[context[i++]]; + } + cur = &cur->tree[word]; + if (cur->prob) { return cur->prob; } + + i = 0; + ostringstream os; + os << "prob " << voc->getWord((VocabIndex)word); + while (context[i] > 0) { + os << ' ' << voc->getWord((VocabIndex)context[i++]); + } + os << endl; + string out = os.str(); + write(sock, out.c_str(), out.size()); + int r = read(sock, res, 6); + int errors = 0; + int cnt = 0; + while (1) { + if (r < 0) { + errors++; sleep(1); + cerr << "Error: read()\n"; + if (errors > 5) exit(1); + } else if (r==0 || res[cnt] == '\n') { break; } + else { + cnt += r; + if (cnt==6) break; + read(sock, &res[cnt], 6-cnt); + } + } + cur->prob = *reinterpret_cast<float*>(res); + return cur->prob; + } + void clear() { + cache.tree.clear(); + } + Cache cache; +}; + diff --git a/contrib/lmserver/examples/query_lmserver.pl b/contrib/lmserver/examples/query_lmserver.pl new file mode 100755 index 000000000..3573dcf5c --- /dev/null +++ b/contrib/lmserver/examples/query_lmserver.pl @@ -0,0 +1,16 @@ +#!/usr/bin/perl -w +use strict; + +use LMClient; +my $lmclient = new LMClient('localhost:11211'); +my $lp1 = $lmclient->word_prob("wants","<s> the old man"); +my $lp2 = $lmclient->word_prob("want","<s> the old man"); +print "$lp1 $lp2\n"; +if ($lp1 > $lp2) { + print "Sentence 1 is more probable\n"; + } else { + print "Sentence 2 is more probable\n"; + } +print "done\n"; + + diff --git a/contrib/lmserver/install-sh b/contrib/lmserver/install-sh new file mode 100755 index 000000000..a5897de6e --- /dev/null +++ b/contrib/lmserver/install-sh @@ -0,0 +1,519 @@ +#!/bin/sh +# install - install a program, script, or datafile + +scriptversion=2006-12-25.00 + +# This originates from X11R5 (mit/util/scripts/install.sh), which was +# later released in X11R6 (xc/config/util/install.sh) with the +# following copyright and license. +# +# Copyright (C) 1994 X Consortium +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- +# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name of the X Consortium shall not +# be used in advertising or otherwise to promote the sale, use or other deal- +# ings in this Software without prior written authorization from the X Consor- +# tium. +# +# +# FSF changes to this file are in the public domain. +# +# Calling this script install-sh is preferred over install.sh, to prevent +# `make' implicit rules from creating a file called install from it +# when there is no Makefile. +# +# This script is compatible with the BSD install script, but was written +# from scratch. + +nl=' +' +IFS=" "" $nl" + +# set DOITPROG to echo to test this script + +# Don't use :- since 4.3BSD and earlier shells don't like it. +doit=${DOITPROG-} +if test -z "$doit"; then + doit_exec=exec +else + doit_exec=$doit +fi + +# Put in absolute file names if you don't have them in your path; +# or use environment vars. + +chgrpprog=${CHGRPPROG-chgrp} +chmodprog=${CHMODPROG-chmod} +chownprog=${CHOWNPROG-chown} +cmpprog=${CMPPROG-cmp} +cpprog=${CPPROG-cp} +mkdirprog=${MKDIRPROG-mkdir} +mvprog=${MVPROG-mv} +rmprog=${RMPROG-rm} +stripprog=${STRIPPROG-strip} + +posix_glob='?' +initialize_posix_glob=' + test "$posix_glob" != "?" || { + if (set -f) 2>/dev/null; then + posix_glob= + else + posix_glob=: + fi + } +' + +posix_mkdir= + +# Desired mode of installed file. +mode=0755 + +chgrpcmd= +chmodcmd=$chmodprog +chowncmd= +mvcmd=$mvprog +rmcmd="$rmprog -f" +stripcmd= + +src= +dst= +dir_arg= +dst_arg= + +copy_on_change=false +no_target_directory= + +usage="\ +Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE + or: $0 [OPTION]... SRCFILES... DIRECTORY + or: $0 [OPTION]... -t DIRECTORY SRCFILES... + or: $0 [OPTION]... -d DIRECTORIES... + +In the 1st form, copy SRCFILE to DSTFILE. +In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. +In the 4th, create DIRECTORIES. + +Options: + --help display this help and exit. + --version display version info and exit. + + -c (ignored) + -C install only if different (preserve the last data modification time) + -d create directories instead of installing files. + -g GROUP $chgrpprog installed files to GROUP. + -m MODE $chmodprog installed files to MODE. + -o USER $chownprog installed files to USER. + -s $stripprog installed files. + -t DIRECTORY install into DIRECTORY. + -T report an error if DSTFILE is a directory. + +Environment variables override the default commands: + CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG + RMPROG STRIPPROG +" + +while test $# -ne 0; do + case $1 in + -c) ;; + + -C) copy_on_change=true;; + + -d) dir_arg=true;; + + -g) chgrpcmd="$chgrpprog $2" + shift;; + + --help) echo "$usage"; exit $?;; + + -m) mode=$2 + case $mode in + *' '* | *' '* | *' +'* | *'*'* | *'?'* | *'['*) + echo "$0: invalid mode: $mode" >&2 + exit 1;; + esac + shift;; + + -o) chowncmd="$chownprog $2" + shift;; + + -s) stripcmd=$stripprog;; + + -t) dst_arg=$2 + shift;; + + -T) no_target_directory=true;; + + --version) echo "$0 $scriptversion"; exit $?;; + + --) shift + break;; + + -*) echo "$0: invalid option: $1" >&2 + exit 1;; + + *) break;; + esac + shift +done + +if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then + # When -d is used, all remaining arguments are directories to create. + # When -t is used, the destination is already specified. + # Otherwise, the last argument is the destination. Remove it from $@. + for arg + do + if test -n "$dst_arg"; then + # $@ is not empty: it contains at least $arg. + set fnord "$@" "$dst_arg" + shift # fnord + fi + shift # arg + dst_arg=$arg + done +fi + +if test $# -eq 0; then + if test -z "$dir_arg"; then + echo "$0: no input file specified." >&2 + exit 1 + fi + # It's OK to call `install-sh -d' without argument. + # This can happen when creating conditional directories. + exit 0 +fi + +if test -z "$dir_arg"; then + trap '(exit $?); exit' 1 2 13 15 + + # Set umask so as not to create temps with too-generous modes. + # However, 'strip' requires both read and write access to temps. + case $mode in + # Optimize common cases. + *644) cp_umask=133;; + *755) cp_umask=22;; + + *[0-7]) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw='% 200' + fi + cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; + *) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw=,u+rw + fi + cp_umask=$mode$u_plus_rw;; + esac +fi + +for src +do + # Protect names starting with `-'. + case $src in + -*) src=./$src;; + esac + + if test -n "$dir_arg"; then + dst=$src + dstdir=$dst + test -d "$dstdir" + dstdir_status=$? + else + + # Waiting for this to be detected by the "$cpprog $src $dsttmp" command + # might cause directories to be created, which would be especially bad + # if $src (and thus $dsttmp) contains '*'. + if test ! -f "$src" && test ! -d "$src"; then + echo "$0: $src does not exist." >&2 + exit 1 + fi + + if test -z "$dst_arg"; then + echo "$0: no destination specified." >&2 + exit 1 + fi + + dst=$dst_arg + # Protect names starting with `-'. + case $dst in + -*) dst=./$dst;; + esac + + # If destination is a directory, append the input filename; won't work + # if double slashes aren't ignored. + if test -d "$dst"; then + if test -n "$no_target_directory"; then + echo "$0: $dst_arg: Is a directory" >&2 + exit 1 + fi + dstdir=$dst + dst=$dstdir/`basename "$src"` + dstdir_status=0 + else + # Prefer dirname, but fall back on a substitute if dirname fails. + dstdir=` + (dirname "$dst") 2>/dev/null || + expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$dst" : 'X\(//\)[^/]' \| \ + X"$dst" : 'X\(//\)$' \| \ + X"$dst" : 'X\(/\)' \| . 2>/dev/null || + echo X"$dst" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q' + ` + + test -d "$dstdir" + dstdir_status=$? + fi + fi + + obsolete_mkdir_used=false + + if test $dstdir_status != 0; then + case $posix_mkdir in + '') + # Create intermediate dirs using mode 755 as modified by the umask. + # This is like FreeBSD 'install' as of 1997-10-28. + umask=`umask` + case $stripcmd.$umask in + # Optimize common cases. + *[2367][2367]) mkdir_umask=$umask;; + .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; + + *[0-7]) + mkdir_umask=`expr $umask + 22 \ + - $umask % 100 % 40 + $umask % 20 \ + - $umask % 10 % 4 + $umask % 2 + `;; + *) mkdir_umask=$umask,go-w;; + esac + + # With -d, create the new directory with the user-specified mode. + # Otherwise, rely on $mkdir_umask. + if test -n "$dir_arg"; then + mkdir_mode=-m$mode + else + mkdir_mode= + fi + + posix_mkdir=false + case $umask in + *[123567][0-7][0-7]) + # POSIX mkdir -p sets u+wx bits regardless of umask, which + # is incompatible with FreeBSD 'install' when (umask & 300) != 0. + ;; + *) + tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ + trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 + + if (umask $mkdir_umask && + exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 + then + if test -z "$dir_arg" || { + # Check for POSIX incompatibilities with -m. + # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or + # other-writeable bit of parent directory when it shouldn't. + # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. + ls_ld_tmpdir=`ls -ld "$tmpdir"` + case $ls_ld_tmpdir in + d????-?r-*) different_mode=700;; + d????-?--*) different_mode=755;; + *) false;; + esac && + $mkdirprog -m$different_mode -p -- "$tmpdir" && { + ls_ld_tmpdir_1=`ls -ld "$tmpdir"` + test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" + } + } + then posix_mkdir=: + fi + rmdir "$tmpdir/d" "$tmpdir" + else + # Remove any dirs left behind by ancient mkdir implementations. + rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null + fi + trap '' 0;; + esac;; + esac + + if + $posix_mkdir && ( + umask $mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" + ) + then : + else + + # The umask is ridiculous, or mkdir does not conform to POSIX, + # or it failed possibly due to a race condition. Create the + # directory the slow way, step by step, checking for races as we go. + + case $dstdir in + /*) prefix='/';; + -*) prefix='./';; + *) prefix='';; + esac + + eval "$initialize_posix_glob" + + oIFS=$IFS + IFS=/ + $posix_glob set -f + set fnord $dstdir + shift + $posix_glob set +f + IFS=$oIFS + + prefixes= + + for d + do + test -z "$d" && continue + + prefix=$prefix$d + if test -d "$prefix"; then + prefixes= + else + if $posix_mkdir; then + (umask=$mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break + # Don't fail if two instances are running concurrently. + test -d "$prefix" || exit 1 + else + case $prefix in + *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; + *) qprefix=$prefix;; + esac + prefixes="$prefixes '$qprefix'" + fi + fi + prefix=$prefix/ + done + + if test -n "$prefixes"; then + # Don't fail if two instances are running concurrently. + (umask $mkdir_umask && + eval "\$doit_exec \$mkdirprog $prefixes") || + test -d "$dstdir" || exit 1 + obsolete_mkdir_used=true + fi + fi + fi + + if test -n "$dir_arg"; then + { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && + { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || + test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 + else + + # Make a couple of temp file names in the proper directory. + dsttmp=$dstdir/_inst.$$_ + rmtmp=$dstdir/_rm.$$_ + + # Trap to clean up those temp files at exit. + trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 + + # Copy the file name to the temp name. + (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + + # and set any options; do chmod last to preserve setuid bits. + # + # If any of these fail, we abort the whole thing. If we want to + # ignore errors from any of these, just make sure not to ignore + # errors from the above "$doit $cpprog $src $dsttmp" command. + # + { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && + { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && + { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && + + # If -C, don't bother to copy if it wouldn't change the file. + if $copy_on_change && + old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && + new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && + + eval "$initialize_posix_glob" && + $posix_glob set -f && + set X $old && old=:$2:$4:$5:$6 && + set X $new && new=:$2:$4:$5:$6 && + $posix_glob set +f && + + test "$old" = "$new" && + $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 + then + rm -f "$dsttmp" + else + # Rename the file to the real destination. + $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || + + # The rename failed, perhaps because mv can't rename something else + # to itself, or perhaps because mv is so ancient that it does not + # support -f. + { + # Now remove or move aside any old file at destination location. + # We try this two ways since rm can't unlink itself on some + # systems and the destination file might be busy for other + # reasons. In this case, the final cleanup might fail but the new + # file should still install successfully. + { + test ! -f "$dst" || + $doit $rmcmd -f "$dst" 2>/dev/null || + { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && + { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } + } || + { echo "$0: cannot unlink or rename $dst" >&2 + (exit 1); exit 1 + } + } && + + # Now rename the file to the real destination. + $doit $mvcmd "$dsttmp" "$dst" + } + fi || exit 1 + + trap '' 0 + fi +done + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/contrib/lmserver/lmserver.c b/contrib/lmserver/lmserver.c new file mode 100644 index 000000000..d3aa685cc --- /dev/null +++ b/contrib/lmserver/lmserver.c @@ -0,0 +1,2140 @@ +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * memcached - memory caching daemon + * + * http://www.danga.com/memcached/ + * + * Copyright 2003 Danga Interactive, Inc. All rights reserved. + * + * Use and distribution licensed under the BSD license. See + * the LICENSE file for full text. + * + * Authors: + * Anatoly Vorobey <mellon@pobox.com> + * Brad Fitzpatrick <brad@danga.com> +std * + * $Id$ + */ +#include "lmserver.h" +#include "srilm.h" +#include <sys/stat.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <signal.h> +#include <sys/resource.h> +#include <sys/uio.h> + +/* some POSIX systems need the following definition + * to get mlockall flags out of sys/mman.h. */ +#ifndef _P1003_1B_VISIBLE +#define _P1003_1B_VISIBLE +#endif +/* need this to get IOV_MAX on some platforms. */ +#ifndef __need_IOV_MAX +#define __need_IOV_MAX +#endif +#include <pwd.h> +#include <sys/mman.h> +#include <fcntl.h> +#include <netinet/tcp.h> +#include <arpa/inet.h> +#include <errno.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <time.h> +#include <assert.h> +#include <limits.h> + +#ifdef HAVE_MALLOC_H +/* OpenBSD has a malloc.h, but warns to use stdlib.h instead */ +#ifndef __OpenBSD__ +#include <malloc.h> +#endif +#endif + +/* FreeBSD 4.x doesn't have IOV_MAX exposed. */ +#ifndef IOV_MAX +#if defined(__FreeBSD__) || defined(__APPLE__) +# define IOV_MAX 1024 +#endif +#endif + +/* + * forward declarations + */ +static void drive_machine(conn *c); +static int new_socket(struct addrinfo *ai); +static int server_socket(const int port, const bool is_udp); +static int try_read_command(conn *c); +static int try_read_network(conn *c); +static int try_read_udp(conn *c); + +/* stats */ +static void stats_reset(void); +static void stats_init(void); + +/* defaults */ +static void settings_init(void); + +/* event handling, network IO */ +static void event_handler(const int fd, const short which, void *arg); +static void conn_close(conn *c); +static void conn_init(void); +static void accept_new_conns(const bool do_accept); +static bool update_event(conn *c, const int new_flags); +static void complete_nread(conn *c); +static void process_command(conn *c, char *command); +static int transmit(conn *c); +static int ensure_iov_space(conn *c); +static int add_iov(conn *c, const void *buf, int len); +static int add_msghdr(conn *c); + +/* time handling */ +static void set_current_time(void); /* update the global variable holding + global 32-bit seconds-since-start time + (to avoid 64 bit time_t) */ + +static void conn_free(conn *c); + +/** exported globals **/ +struct stats stats; +struct settings settings; + +/** file scope variables **/ +static item **todelete = NULL; +static int delcurr; +static int deltotal; +static conn *listen_conn = NULL; +static struct event_base *main_base; + +#define TRANSMIT_COMPLETE 0 +#define TRANSMIT_INCOMPLETE 1 +#define TRANSMIT_SOFT_ERROR 2 +#define TRANSMIT_HARD_ERROR 3 + +static int *buckets = 0; /* bucket->generation array for a managed instance */ + +#define REALTIME_MAXDELTA 60*60*24*30 +/* + * given time value that's either unix time or delta from current unix time, return + * unix time. Use the fact that delta can't exceed one month (and real time value can't + * be that low). + */ +static rel_time_t realtime(const time_t exptime) { + /* no. of seconds in 30 days - largest possible delta exptime */ + + if (exptime == 0) return 0; /* 0 means never expire */ + + if (exptime > REALTIME_MAXDELTA) { + /* if item expiration is at/before the server started, give it an + expiration time of 1 second after the server started. + (because 0 means don't expire). without this, we'd + underflow and wrap around to some large value way in the + future, effectively making items expiring in the past + really expiring never */ + if (exptime <= stats.started) + return (rel_time_t)1; + return (rel_time_t)(exptime - stats.started); + } else { + return (rel_time_t)(exptime + current_time); + } +} + +static void stats_init(void) { + stats.curr_items = stats.total_items = stats.curr_conns = stats.total_conns = stats.conn_structs = 0; + stats.get_cmds = stats.set_cmds = stats.get_hits = stats.get_misses = stats.evictions = 0; + stats.curr_bytes = stats.bytes_read = stats.bytes_written = 0; + + /* make the time we started always be 2 seconds before we really + did, so time(0) - time.started is never zero. if so, things + like 'settings.oldest_live' which act as booleans as well as + values are now false in boolean context... */ + stats.started = time(0) - 2; +} + +static void stats_reset(void) { + STATS_LOCK(); + stats.total_items = stats.total_conns = 0; + stats.get_cmds = stats.set_cmds = stats.get_hits = stats.get_misses = stats.evictions = 0; + stats.bytes_read = stats.bytes_written = 0; + STATS_UNLOCK(); +} + +static void settings_init(void) { + settings.srilm = NULL; + settings.srilm_order = 3; + settings.access=0700; + settings.port = 11211; + settings.udpport = 0; + /* By default this string should be NULL for getaddrinfo() */ + settings.inter = NULL; + settings.maxbytes = 64 * 1024 * 1024; /* default is 64MB */ + settings.maxconns = 1024; /* to limit connections-related memory to about 5MB */ + settings.verbose = 0; + settings.oldest_live = 0; + settings.evict_to_free = 1; /* push old items out of cache when memory runs out */ + settings.socketpath = NULL; /* by default, not using a unix socket */ + settings.managed = false; + settings.factor = 1.25; + settings.chunk_size = 48; /* space for a modest key and value */ +#ifdef USE_THREADS + settings.num_threads = 4; +#else + settings.num_threads = 1; +#endif + settings.detail_enabled = 0; +} + +/* returns true if a deleted item's delete-locked-time is over, and it + should be removed from the namespace */ +static bool item_delete_lock_over (item *it) { + assert(it->it_flags & ITEM_DELETED); + return (current_time >= it->exptime); +} + +/* + * Adds a message header to a connection. + * + * Returns 0 on success, -1 on out-of-memory. + */ +static int add_msghdr(conn *c) +{ + struct msghdr *msg; + + assert(c != NULL); + + if (c->msgsize == c->msgused) { + msg = realloc(c->msglist, c->msgsize * 2 * sizeof(struct msghdr)); + if (! msg) + return -1; + c->msglist = msg; + c->msgsize *= 2; + } + + msg = c->msglist + c->msgused; + + /* this wipes msg_iovlen, msg_control, msg_controllen, and + msg_flags, the last 3 of which aren't defined on solaris: */ + memset(msg, 0, sizeof(struct msghdr)); + + msg->msg_iov = &c->iov[c->iovused]; + + if (c->request_addr_size > 0) { + msg->msg_name = &c->request_addr; + msg->msg_namelen = c->request_addr_size; + } + + c->msgbytes = 0; + c->msgused++; + + if (c->udp) { + /* Leave room for the UDP header, which we'll fill in later. */ + return add_iov(c, NULL, UDP_HEADER_SIZE); + } + + return 0; +} + + +/* + * Free list management for connections. + */ + +static conn **freeconns; +static int freetotal; +static int freecurr; + + +static void conn_init(void) { + freetotal = 200; + freecurr = 0; + if ((freeconns = (conn **)malloc(sizeof(conn *) * freetotal)) == NULL) { + fprintf(stderr, "malloc()\n"); + } + return; +} + +/* + * Returns a connection from the freelist, if any. Should call this using + * conn_from_freelist() for thread safety. + */ +conn *do_conn_from_freelist() { + conn *c; + + if (freecurr > 0) { + c = freeconns[--freecurr]; + } else { + c = NULL; + } + + return c; +} + +/* + * Adds a connection to the freelist. 0 = success. Should call this using + * conn_add_to_freelist() for thread safety. + */ +bool do_conn_add_to_freelist(conn *c) { + if (freecurr < freetotal) { + freeconns[freecurr++] = c; + return false; + } else { + /* try to enlarge free connections array */ + conn **new_freeconns = realloc(freeconns, sizeof(conn *) * freetotal * 2); + if (new_freeconns) { + freetotal *= 2; + freeconns = new_freeconns; + freeconns[freecurr++] = c; + return false; + } + } + return true; +} + +conn *conn_new(const int sfd, const int init_state, const int event_flags, + const int read_buffer_size, const bool is_udp, struct event_base *base) { + conn *c = conn_from_freelist(); + + if (NULL == c) { + if (!(c = (conn *)calloc(1, sizeof(conn)))) { + fprintf(stderr, "calloc()\n"); + return NULL; + } + + c->rbuf = c->wbuf = 0; + c->ilist = 0; + c->suffixlist = 0; + c->iov = 0; + c->msglist = 0; + c->hdrbuf = 0; + + c->rsize = read_buffer_size; + c->wsize = DATA_BUFFER_SIZE; + c->isize = ITEM_LIST_INITIAL; + c->suffixsize = SUFFIX_LIST_INITIAL; + c->iovsize = IOV_LIST_INITIAL; + c->msgsize = MSG_LIST_INITIAL; + c->hdrsize = 0; + + c->rbuf = (char *)malloc((size_t)c->rsize); + c->wbuf = (char *)malloc((size_t)c->wsize); + c->ilist = (item **)malloc(sizeof(item *) * c->isize); + c->suffixlist = (char **)malloc(sizeof(char *) * c->suffixsize); + c->iov = (struct iovec *)malloc(sizeof(struct iovec) * c->iovsize); + c->msglist = (struct msghdr *)malloc(sizeof(struct msghdr) * c->msgsize); + + if (c->rbuf == 0 || c->wbuf == 0 || c->ilist == 0 || c->iov == 0 || + c->msglist == 0 || c->suffixlist == 0) { + conn_free(c); + fprintf(stderr, "malloc()\n"); + return NULL; + } + + STATS_LOCK(); + stats.conn_structs++; + STATS_UNLOCK(); + } + + if (settings.verbose > 1) { + if (init_state == conn_listening) + fprintf(stderr, "<%d server listening\n", sfd); + else if (is_udp) + fprintf(stderr, "<%d server listening (udp)\n", sfd); + else + fprintf(stderr, "<%d new client connection\n", sfd); + } + + c->sfd = sfd; + c->udp = is_udp; + c->state = init_state; + c->rlbytes = 0; + c->rbytes = c->wbytes = 0; + c->wcurr = c->wbuf; + c->rcurr = c->rbuf; + c->ritem = 0; + c->icurr = c->ilist; + c->suffixcurr = c->suffixlist; + c->ileft = 0; + c->suffixleft = 0; + c->iovused = 0; + c->msgcurr = 0; + c->msgused = 0; + + c->write_and_go = conn_read; + c->write_and_free = 0; + c->item = 0; + c->bucket = -1; + c->gen = 0; + + c->noreply = false; + + event_set(&c->event, sfd, event_flags, event_handler, (void *)c); + event_base_set(base, &c->event); + c->ev_flags = event_flags; + + if (event_add(&c->event, 0) == -1) { + if (conn_add_to_freelist(c)) { + conn_free(c); + } + perror("event_add"); + return NULL; + } + + STATS_LOCK(); + stats.curr_conns++; + stats.total_conns++; + STATS_UNLOCK(); + + return c; +} + +static void conn_cleanup(conn *c) { + assert(c != NULL); + + if (c->write_and_free) { + free(c->write_and_free); + c->write_and_free = 0; + } +} + +/* + * Frees a connection. + */ +void conn_free(conn *c) { + if (c) { + if (c->hdrbuf) + free(c->hdrbuf); + if (c->msglist) + free(c->msglist); + if (c->rbuf) + free(c->rbuf); + if (c->wbuf) + free(c->wbuf); + if (c->ilist) + free(c->ilist); + if (c->suffixlist) + free(c->suffixlist); + if (c->iov) + free(c->iov); + free(c); + } +} + +static void conn_close(conn *c) { + assert(c != NULL); + + /* delete the event, the socket and the conn */ + event_del(&c->event); + + if (settings.verbose > 1) + fprintf(stderr, "<%d connection closed.\n", c->sfd); + + close(c->sfd); + accept_new_conns(true); + conn_cleanup(c); + + /* if the connection has big buffers, just free it */ + if (c->rsize > READ_BUFFER_HIGHWAT || conn_add_to_freelist(c)) { + conn_free(c); + } + + STATS_LOCK(); + stats.curr_conns--; + STATS_UNLOCK(); + + return; +} + + +/* + * Shrinks a connection's buffers if they're too big. This prevents + * periodic large "get" requests from permanently chewing lots of server + * memory. + * + * This should only be called in between requests since it can wipe output + * buffers! + */ +static void conn_shrink(conn *c) { + assert(c != NULL); + + if (c->udp) + return; + + if (c->rsize > READ_BUFFER_HIGHWAT && c->rbytes < DATA_BUFFER_SIZE) { + char *newbuf; + + if (c->rcurr != c->rbuf) + memmove(c->rbuf, c->rcurr, (size_t)c->rbytes); + + newbuf = (char *)realloc((void *)c->rbuf, DATA_BUFFER_SIZE); + + if (newbuf) { + c->rbuf = newbuf; + c->rsize = DATA_BUFFER_SIZE; + } + /* TODO check other branch... */ + c->rcurr = c->rbuf; + } + + if (c->isize > ITEM_LIST_HIGHWAT) { + item **newbuf = (item**) realloc((void *)c->ilist, ITEM_LIST_INITIAL * sizeof(c->ilist[0])); + if (newbuf) { + c->ilist = newbuf; + c->isize = ITEM_LIST_INITIAL; + } + /* TODO check error condition? */ + } + + if (c->msgsize > MSG_LIST_HIGHWAT) { + struct msghdr *newbuf = (struct msghdr *) realloc((void *)c->msglist, MSG_LIST_INITIAL * sizeof(c->msglist[0])); + if (newbuf) { + c->msglist = newbuf; + c->msgsize = MSG_LIST_INITIAL; + } + /* TODO check error condition? */ + } + + if (c->iovsize > IOV_LIST_HIGHWAT) { + struct iovec *newbuf = (struct iovec *) realloc((void *)c->iov, IOV_LIST_INITIAL * sizeof(c->iov[0])); + if (newbuf) { + c->iov = newbuf; + c->iovsize = IOV_LIST_INITIAL; + } + /* TODO check return value */ + } +} + +/* + * Sets a connection's current state in the state machine. Any special + * processing that needs to happen on certain state transitions can + * happen here. + */ +static void conn_set_state(conn *c, int state) { + assert(c != NULL); + + if (state != c->state) { + if (state == conn_read) { + conn_shrink(c); + //assoc_move_next_bucket(); + } + c->state = state; + } +} + +/* + * Ensures that there is room for another struct iovec in a connection's + * iov list. + * + * Returns 0 on success, -1 on out-of-memory. + */ +static int ensure_iov_space(conn *c) { + assert(c != NULL); + + if (c->iovused >= c->iovsize) { + int i, iovnum; + struct iovec *new_iov = (struct iovec *)realloc(c->iov, + (c->iovsize * 2) * sizeof(struct iovec)); + if (! new_iov) + return -1; + c->iov = new_iov; + c->iovsize *= 2; + + /* Point all the msghdr structures at the new list. */ + for (i = 0, iovnum = 0; i < c->msgused; i++) { + c->msglist[i].msg_iov = &c->iov[iovnum]; + iovnum += c->msglist[i].msg_iovlen; + } + } + + return 0; +} + + +/* + * Adds data to the list of pending data that will be written out to a + * connection. + * + * Returns 0 on success, -1 on out-of-memory. + */ + +static int add_iov(conn *c, const void *buf, int len) { + struct msghdr *m; + int leftover; + bool limit_to_mtu; + + assert(c != NULL); + + do { + m = &c->msglist[c->msgused - 1]; + + /* + * Limit UDP packets, and the first payloads of TCP replies, to + * UDP_MAX_PAYLOAD_SIZE bytes. + */ + limit_to_mtu = c->udp || (1 == c->msgused); + + /* We may need to start a new msghdr if this one is full. */ + if (m->msg_iovlen == IOV_MAX || + (limit_to_mtu && c->msgbytes >= UDP_MAX_PAYLOAD_SIZE)) { + add_msghdr(c); + m = &c->msglist[c->msgused - 1]; + } + + if (ensure_iov_space(c) != 0) + return -1; + + /* If the fragment is too big to fit in the datagram, split it up */ + if (limit_to_mtu && len + c->msgbytes > UDP_MAX_PAYLOAD_SIZE) { + leftover = len + c->msgbytes - UDP_MAX_PAYLOAD_SIZE; + len -= leftover; + } else { + leftover = 0; + } + + m = &c->msglist[c->msgused - 1]; + m->msg_iov[m->msg_iovlen].iov_base = (void *)buf; + m->msg_iov[m->msg_iovlen].iov_len = len; + + c->msgbytes += len; + c->iovused++; + m->msg_iovlen++; + + buf = ((char *)buf) + len; + len = leftover; + } while (leftover > 0); + + return 0; +} + + +/* + * Constructs a set of UDP headers and attaches them to the outgoing messages. + */ +static int build_udp_headers(conn *c) { + int i; + unsigned char *hdr; + + assert(c != NULL); + + if (c->msgused > c->hdrsize) { + void *new_hdrbuf; + if (c->hdrbuf) + new_hdrbuf = realloc(c->hdrbuf, c->msgused * 2 * UDP_HEADER_SIZE); + else + new_hdrbuf = malloc(c->msgused * 2 * UDP_HEADER_SIZE); + if (! new_hdrbuf) + return -1; + c->hdrbuf = (unsigned char *)new_hdrbuf; + c->hdrsize = c->msgused * 2; + } + + hdr = c->hdrbuf; + for (i = 0; i < c->msgused; i++) { + c->msglist[i].msg_iov[0].iov_base = hdr; + c->msglist[i].msg_iov[0].iov_len = UDP_HEADER_SIZE; + *hdr++ = c->request_id / 256; + *hdr++ = c->request_id % 256; + *hdr++ = i / 256; + *hdr++ = i % 256; + *hdr++ = c->msgused / 256; + *hdr++ = c->msgused % 256; + *hdr++ = 0; + *hdr++ = 0; + assert((void *) hdr == (void *)c->msglist[i].msg_iov[0].iov_base + UDP_HEADER_SIZE); + } + + return 0; +} + + +static void out_string(conn *c, const char *str) { + size_t len; + + assert(c != NULL); + + if (c->noreply) { + if (settings.verbose > 1) + fprintf(stderr, ">%d NOREPLY %s\n", c->sfd, str); + c->noreply = false; + conn_set_state(c, conn_read); + return; + } + + if (settings.verbose > 1) + fprintf(stderr, ">%d %s\n", c->sfd, str); + + len = strlen(str); + if ((len + 2) > c->wsize) { + /* ought to be always enough. just fail for simplicity */ + str = "SERVER_ERROR output line too long"; + len = strlen(str); + } + + memcpy(c->wbuf, str, len); + memcpy(c->wbuf + len, "\r\n", 2); + c->wbytes = len + 2; + c->wcurr = c->wbuf; + + conn_set_state(c, conn_write); + c->write_and_go = conn_read; + return; +} + +typedef struct token_s { + char *value; + size_t length; +} token_t; + +#define COMMAND_TOKEN 0 +#define SUBCOMMAND_TOKEN 1 +#define KEY_TOKEN 1 +#define KEY_MAX_LENGTH 250 + +#define MAX_TOKENS 8 + +/* + * Tokenize the command string by replacing whitespace with '\0' and update + * the token array tokens with pointer to start of each token and length. + * Returns total number of tokens. The last valid token is the terminal + * token (value points to the first unprocessed character of the string and + * length zero). + * + * Usage example: + * + * while(tokenize_command(command, ncommand, tokens, max_tokens) > 0) { + * for(int ix = 0; tokens[ix].length != 0; ix++) { + * ... + * } + * ncommand = tokens[ix].value - command; + * command = tokens[ix].value; + * } + */ +static size_t tokenize_command(char *command, token_t *tokens, const size_t max_tokens) { + char *s, *e; + size_t ntokens = 0; + + assert(command != NULL && tokens != NULL && max_tokens > 1); + + for (s = e = command; ntokens < max_tokens - 1; ++e) { + if (*e == ' ') { + if (s != e) { + tokens[ntokens].value = s; + tokens[ntokens].length = e - s; + ntokens++; + *e = '\0'; + } + s = e + 1; + } + else if (*e == '\0') { + if (s != e) { + tokens[ntokens].value = s; + tokens[ntokens].length = e - s; + ntokens++; + } + + break; /* string end */ + } + } + + /* + * If we scanned the whole string, the terminal value pointer is null, + * otherwise it is the first unprocessed character. + */ + tokens[ntokens].value = *e == '\0' ? NULL : e; + tokens[ntokens].length = 0; + ntokens++; + + return ntokens; +} + +/* set up a connection to write a buffer then free it, used for stats */ +static void write_and_free(conn *c, char *buf, int bytes) { + if (buf) { + c->write_and_free = buf; + c->wcurr = buf; + c->wbytes = bytes; + conn_set_state(c, conn_write); + c->write_and_go = conn_read; + } else { + out_string(c, "SERVER_ERROR out of memory writing stats"); + } +} + +static inline void set_noreply_maybe(conn *c, token_t *tokens, size_t ntokens) +{ + int noreply_index = ntokens - 2; + + /* + NOTE: this function is not the first place where we are going to + send the reply. We could send it instead from process_command() + if the request line has wrong number of tokens. However parsing + malformed line for "noreply" option is not reliable anyway, so + it can't be helped. + */ + if (tokens[noreply_index].value + && strcmp(tokens[noreply_index].value, "noreply") == 0) { + c->noreply = true; + } +} + +inline static void process_stats_detail(conn *c, const char *command) { + assert(c != NULL); + + if (strcmp(command, "on") == 0) { + settings.detail_enabled = 1; + out_string(c, "OK"); + } + else if (strcmp(command, "off") == 0) { + settings.detail_enabled = 0; + out_string(c, "OK"); + } else { + out_string(c, "CLIENT_ERROR usage: stats detail on|off|dump"); + } +} + +static void process_stat(conn *c, token_t *tokens, const size_t ntokens) { + rel_time_t now = current_time; + char *command; + char *subcommand; + + assert(c != NULL); + + if(ntokens < 2) { + out_string(c, "CLIENT_ERROR bad command line"); + return; + } + + command = tokens[COMMAND_TOKEN].value; + + if (ntokens == 2 && strcmp(command, "stats") == 0) { + char temp[1024]; + pid_t pid = getpid(); + char *pos = temp; + +#ifndef WIN32 + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); +#endif /* !WIN32 */ + + STATS_LOCK(); + pos += sprintf(pos, "STAT pid %u\r\n", pid); + pos += sprintf(pos, "STAT uptime %u\r\n", now); + pos += sprintf(pos, "STAT time %ld\r\n", now + stats.started); + pos += sprintf(pos, "STAT version " VERSION "\r\n"); + pos += sprintf(pos, "STAT pointer_size %d\r\n", 8 * sizeof(void *)); +#ifndef WIN32 + pos += sprintf(pos, "STAT rusage_user %ld.%06ld\r\n", usage.ru_utime.tv_sec, usage.ru_utime.tv_usec); + pos += sprintf(pos, "STAT rusage_system %ld.%06ld\r\n", usage.ru_stime.tv_sec, usage.ru_stime.tv_usec); +#endif /* !WIN32 */ + pos += sprintf(pos, "STAT curr_items %u\r\n", stats.curr_items); + pos += sprintf(pos, "STAT total_items %u\r\n", stats.total_items); + pos += sprintf(pos, "STAT bytes %llu\r\n", stats.curr_bytes); + pos += sprintf(pos, "STAT curr_connections %u\r\n", stats.curr_conns - 1); /* ignore listening conn */ + pos += sprintf(pos, "STAT total_connections %u\r\n", stats.total_conns); + pos += sprintf(pos, "STAT connection_structures %u\r\n", stats.conn_structs); + pos += sprintf(pos, "STAT cmd_get %llu\r\n", stats.get_cmds); + pos += sprintf(pos, "STAT cmd_set %llu\r\n", stats.set_cmds); + pos += sprintf(pos, "STAT get_hits %llu\r\n", stats.get_hits); + pos += sprintf(pos, "STAT get_misses %llu\r\n", stats.get_misses); + pos += sprintf(pos, "STAT evictions %llu\r\n", stats.evictions); + pos += sprintf(pos, "STAT bytes_read %llu\r\n", stats.bytes_read); + pos += sprintf(pos, "STAT bytes_written %llu\r\n", stats.bytes_written); + pos += sprintf(pos, "STAT limit_maxbytes %llu\r\n", (uint64_t) settings.maxbytes); + pos += sprintf(pos, "STAT threads %u\r\n", settings.num_threads); + pos += sprintf(pos, "END"); + STATS_UNLOCK(); + out_string(c, temp); + return; + } + + subcommand = tokens[SUBCOMMAND_TOKEN].value; + + if (strcmp(subcommand, "reset") == 0) { + stats_reset(); + out_string(c, "RESET"); + return; + } + +#ifdef HAVE_MALLOC_H +#ifdef HAVE_STRUCT_MALLINFO + if (strcmp(subcommand, "malloc") == 0) { + char temp[512]; + struct mallinfo info; + char *pos = temp; + + info = mallinfo(); + pos += sprintf(pos, "STAT arena_size %d\r\n", info.arena); + pos += sprintf(pos, "STAT free_chunks %d\r\n", info.ordblks); + pos += sprintf(pos, "STAT fastbin_blocks %d\r\n", info.smblks); + pos += sprintf(pos, "STAT mmapped_regions %d\r\n", info.hblks); + pos += sprintf(pos, "STAT mmapped_space %d\r\n", info.hblkhd); + pos += sprintf(pos, "STAT max_total_alloc %d\r\n", info.usmblks); + pos += sprintf(pos, "STAT fastbin_space %d\r\n", info.fsmblks); + pos += sprintf(pos, "STAT total_alloc %d\r\n", info.uordblks); + pos += sprintf(pos, "STAT total_free %d\r\n", info.fordblks); + pos += sprintf(pos, "STAT releasable_space %d\r\nEND", info.keepcost); + out_string(c, temp); + return; + } +#endif /* HAVE_STRUCT_MALLINFO */ +#endif /* HAVE_MALLOC_H */ + out_string(c, "ERROR"); +} + +static inline void process_srilm_command(conn *c, token_t *tokens, size_t ntokens) { + int context[6]; + int i = 1; + int j = ntokens - 3; + while (tokens[i].length) { + context[i-1] = srilm_getvoc(tokens[i].value); + ++i; + } + float p = -999.0f; + if (context[0] != -1) { + context[i-1] = -1; + p = srilm_wordprob(context[0], &context[1]); + } + + memcpy(c->wbuf, &p, sizeof(float)); + memcpy(c->wbuf + sizeof(float), "\r\n", 2); + c->wbytes = sizeof(float) + 2; + c->wcurr = c->wbuf; + + conn_set_state(c, conn_write); + c->write_and_go = conn_read; +} + +static void process_command(conn *c, char *command) { + + token_t tokens[MAX_TOKENS]; + size_t ntokens; + int comm; + + assert(c != NULL); + + if (settings.verbose > 1) + fprintf(stderr, "<%d %s\n", c->sfd, command); + + /* + * for commands set/add/replace, we build an item and read the data + * directly into it, then continue in nread_complete(). + */ + + c->msgcurr = 0; + c->msgused = 0; + c->iovused = 0; + if (add_msghdr(c) != 0) { + out_string(c, "SERVER_ERROR out of memory preparing response"); + return; + } + + ntokens = tokenize_command(command, tokens, MAX_TOKENS); + if (ntokens >1 && + strcmp(tokens[COMMAND_TOKEN].value, "prob") == 0) { + process_srilm_command(c, tokens, ntokens); + } else if (ntokens >= 2 && (strcmp(tokens[COMMAND_TOKEN].value, "stats") == 0)) { + + process_stat(c, tokens, ntokens); + + } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "version") == 0)) { + + out_string(c, "VERSION " VERSION); + + } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "quit") == 0)) { + + conn_set_state(c, conn_closing); + + } else { + out_string(c, "ERROR"); + } + return; +} + +/* + * if we have a complete line in the buffer, process it. + */ +static int try_read_command(conn *c) { + char *el, *cont; + + assert(c != NULL); + assert(c->rcurr <= (c->rbuf + c->rsize)); + + if (c->rbytes == 0) + return 0; + el = memchr(c->rcurr, '\n', c->rbytes); + if (!el) + return 0; + cont = el + 1; + if ((el - c->rcurr) > 1 && *(el - 1) == '\r') { + el--; + } + *el = '\0'; + + assert(cont <= (c->rcurr + c->rbytes)); + + process_command(c, c->rcurr); + + c->rbytes -= (cont - c->rcurr); + c->rcurr = cont; + + assert(c->rcurr <= (c->rbuf + c->rsize)); + + return 1; +} + +/* + * read a UDP request. + * return 0 if there's nothing to read. + */ +static int try_read_udp(conn *c) { + int res; + + assert(c != NULL); + + c->request_addr_size = sizeof(c->request_addr); + res = recvfrom(c->sfd, c->rbuf, c->rsize, + 0, &c->request_addr, &c->request_addr_size); + if (res > 8) { + unsigned char *buf = (unsigned char *)c->rbuf; + STATS_LOCK(); + stats.bytes_read += res; + STATS_UNLOCK(); + + /* Beginning of UDP packet is the request ID; save it. */ + c->request_id = buf[0] * 256 + buf[1]; + + /* If this is a multi-packet request, drop it. */ + if (buf[4] != 0 || buf[5] != 1) { + out_string(c, "SERVER_ERROR multi-packet request not supported"); + return 0; + } + + /* Don't care about any of the rest of the header. */ + res -= 8; + memmove(c->rbuf, c->rbuf + 8, res); + + c->rbytes += res; + c->rcurr = c->rbuf; + return 1; + } + return 0; +} + +/* + * read from network as much as we can, handle buffer overflow and connection + * close. + * before reading, move the remaining incomplete fragment of a command + * (if any) to the beginning of the buffer. + * return 0 if there's nothing to read on the first read. + */ +static int try_read_network(conn *c) { + int gotdata = 0; + int res; + + assert(c != NULL); + + if (c->rcurr != c->rbuf) { + if (c->rbytes != 0) /* otherwise there's nothing to copy */ + memmove(c->rbuf, c->rcurr, c->rbytes); + c->rcurr = c->rbuf; + } + + while (1) { + if (c->rbytes >= c->rsize) { + char *new_rbuf = realloc(c->rbuf, c->rsize * 2); + if (!new_rbuf) { + if (settings.verbose > 0) + fprintf(stderr, "Couldn't realloc input buffer\n"); + c->rbytes = 0; /* ignore what we read */ + out_string(c, "SERVER_ERROR out of memory reading request"); + c->write_and_go = conn_closing; + return 1; + } + c->rcurr = c->rbuf = new_rbuf; + c->rsize *= 2; + } + + /* unix socket mode doesn't need this, so zeroed out. but why + * is this done for every command? presumably for UDP + * mode. */ + if (!settings.socketpath) { + c->request_addr_size = sizeof(c->request_addr); + } else { + c->request_addr_size = 0; + } + + int avail = c->rsize - c->rbytes; + res = read(c->sfd, c->rbuf + c->rbytes, avail); + if (res > 0) { + STATS_LOCK(); + stats.bytes_read += res; + STATS_UNLOCK(); + gotdata = 1; + c->rbytes += res; + if (res == avail) { + continue; + } else { + break; + } + } + if (res == 0) { + /* connection closed */ + conn_set_state(c, conn_closing); + return 1; + } + if (res == -1) { + if (errno == EAGAIN || errno == EWOULDBLOCK) break; + /* Should close on unhandled errors. */ + conn_set_state(c, conn_closing); + return 1; + } + } + return gotdata; +} + +static bool update_event(conn *c, const int new_flags) { + assert(c != NULL); + + struct event_base *base = c->event.ev_base; + if (c->ev_flags == new_flags) + return true; + if (event_del(&c->event) == -1) return false; + event_set(&c->event, c->sfd, new_flags, event_handler, (void *)c); + event_base_set(base, &c->event); + c->ev_flags = new_flags; + if (event_add(&c->event, 0) == -1) return false; + return true; +} + +/* + * Sets whether we are listening for new connections or not. + */ +void accept_new_conns(const bool do_accept) { + conn *next; + + if (! is_listen_thread()) + return; + + for (next = listen_conn; next; next = next->next) { + if (do_accept) { + update_event(next, EV_READ | EV_PERSIST); + if (listen(next->sfd, 1024) != 0) { + perror("listen"); + } + } + else { + update_event(next, 0); + if (listen(next->sfd, 0) != 0) { + perror("listen"); + } + } + } +} + + +/* + * Transmit the next chunk of data from our list of msgbuf structures. + * + * Returns: + * TRANSMIT_COMPLETE All done writing. + * TRANSMIT_INCOMPLETE More data remaining to write. + * TRANSMIT_SOFT_ERROR Can't write any more right now. + * TRANSMIT_HARD_ERROR Can't write (c->state is set to conn_closing) + */ +static int transmit(conn *c) { + assert(c != NULL); + + if (c->msgcurr < c->msgused && + c->msglist[c->msgcurr].msg_iovlen == 0) { + /* Finished writing the current msg; advance to the next. */ + c->msgcurr++; + } + if (c->msgcurr < c->msgused) { + ssize_t res; + struct msghdr *m = &c->msglist[c->msgcurr]; + + res = sendmsg(c->sfd, m, 0); + if (res > 0) { + STATS_LOCK(); + stats.bytes_written += res; + STATS_UNLOCK(); + + /* We've written some of the data. Remove the completed + iovec entries from the list of pending writes. */ + while (m->msg_iovlen > 0 && res >= m->msg_iov->iov_len) { + res -= m->msg_iov->iov_len; + m->msg_iovlen--; + m->msg_iov++; + } + + /* Might have written just part of the last iovec entry; + adjust it so the next write will do the rest. */ + if (res > 0) { + m->msg_iov->iov_base += res; + m->msg_iov->iov_len -= res; + } + return TRANSMIT_INCOMPLETE; + } + if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + if (!update_event(c, EV_WRITE | EV_PERSIST)) { + if (settings.verbose > 0) + fprintf(stderr, "Couldn't update event\n"); + conn_set_state(c, conn_closing); + return TRANSMIT_HARD_ERROR; + } + return TRANSMIT_SOFT_ERROR; + } + /* if res==0 or res==-1 and error is not EAGAIN or EWOULDBLOCK, + we have a real error, on which we close the connection */ + if (settings.verbose > 0) + perror("Failed to write, and not due to blocking"); + + if (c->udp) + conn_set_state(c, conn_read); + else + conn_set_state(c, conn_closing); + return TRANSMIT_HARD_ERROR; + } else { + return TRANSMIT_COMPLETE; + } +} + +static void drive_machine(conn *c) { + bool stop = false; + int sfd, flags = 1; + socklen_t addrlen; + struct sockaddr_storage addr; + int res; + + assert(c != NULL); + + while (!stop) { + + switch(c->state) { + case conn_listening: + addrlen = sizeof(addr); + if ((sfd = accept(c->sfd, (struct sockaddr *)&addr, &addrlen)) == -1) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + /* these are transient, so don't log anything */ + stop = true; + } else if (errno == EMFILE) { + if (settings.verbose > 0) + fprintf(stderr, "Too many open connections\n"); + accept_new_conns(false); + stop = true; + } else { + perror("accept()"); + stop = true; + } + break; + } + if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 || + fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) { + perror("setting O_NONBLOCK"); + close(sfd); + break; + } + dispatch_conn_new(sfd, conn_read, EV_READ | EV_PERSIST, + DATA_BUFFER_SIZE, false); + break; + + case conn_read: + if (try_read_command(c) != 0) { + continue; + } + if ((c->udp ? try_read_udp(c) : try_read_network(c)) != 0) { + continue; + } + /* we have no command line and no data to read from network */ + if (!update_event(c, EV_READ | EV_PERSIST)) { + if (settings.verbose > 0) + fprintf(stderr, "Couldn't update event\n"); + conn_set_state(c, conn_closing); + break; + } + stop = true; + break; + + case conn_nread: + assert(!"nread should not be possible"); + break; + + case conn_swallow: + /* we are reading sbytes and throwing them away */ + if (c->sbytes == 0) { + conn_set_state(c, conn_read); + break; + } + + /* first check if we have leftovers in the conn_read buffer */ + if (c->rbytes > 0) { + int tocopy = c->rbytes > c->sbytes ? c->sbytes : c->rbytes; + c->sbytes -= tocopy; + c->rcurr += tocopy; + c->rbytes -= tocopy; + break; + } + + /* now try reading from the socket */ + res = read(c->sfd, c->rbuf, c->rsize > c->sbytes ? c->sbytes : c->rsize); + if (res > 0) { + STATS_LOCK(); + stats.bytes_read += res; + STATS_UNLOCK(); + c->sbytes -= res; + break; + } + if (res == 0) { /* end of stream */ + conn_set_state(c, conn_closing); + break; + } + if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + if (!update_event(c, EV_READ | EV_PERSIST)) { + if (settings.verbose > 0) + fprintf(stderr, "Couldn't update event\n"); + conn_set_state(c, conn_closing); + break; + } + stop = true; + break; + } + /* otherwise we have a real error, on which we close the connection */ + if (settings.verbose > 0) + fprintf(stderr, "Failed to read, and not due to blocking\n"); + conn_set_state(c, conn_closing); + break; + + case conn_write: + /* + * We want to write out a simple response. If we haven't already, + * assemble it into a msgbuf list (this will be a single-entry + * list for TCP or a two-entry list for UDP). + */ + if (c->iovused == 0 || (c->udp && c->iovused == 1)) { + if (add_iov(c, c->wcurr, c->wbytes) != 0 || + (c->udp && build_udp_headers(c) != 0)) { + if (settings.verbose > 0) + fprintf(stderr, "Couldn't build response\n"); + conn_set_state(c, conn_closing); + break; + } + } + + /* fall through... */ + + case conn_mwrite: + switch (transmit(c)) { + case TRANSMIT_COMPLETE: + if (c->state == conn_write) { + if (c->write_and_free) { + free(c->write_and_free); + c->write_and_free = 0; + } + conn_set_state(c, c->write_and_go); + } else { + if (settings.verbose > 0) + fprintf(stderr, "Unexpected state %d\n", c->state); + conn_set_state(c, conn_closing); + } + break; + + case TRANSMIT_INCOMPLETE: + case TRANSMIT_HARD_ERROR: + break; /* Continue in state machine. */ + + case TRANSMIT_SOFT_ERROR: + stop = true; + break; + } + break; + + case conn_closing: + if (c->udp) + conn_cleanup(c); + else + conn_close(c); + stop = true; + break; + } + } + + return; +} + +void event_handler(const int fd, const short which, void *arg) { + conn *c; + + c = (conn *)arg; + assert(c != NULL); + + c->which = which; + + /* sanity */ + if (fd != c->sfd) { + if (settings.verbose > 0) + fprintf(stderr, "Catastrophic: event fd doesn't match conn fd!\n"); + conn_close(c); + return; + } + + drive_machine(c); + + /* wait for next event */ + return; +} + +static int new_socket(struct addrinfo *ai) { + int sfd; + int flags; + + if ((sfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol)) == -1) { + perror("socket()"); + return -1; + } + + if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 || + fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) { + perror("setting O_NONBLOCK"); + close(sfd); + return -1; + } + return sfd; +} + + +/* + * Sets a socket's send buffer size to the maximum allowed by the system. + */ +static void maximize_sndbuf(const int sfd) { + socklen_t intsize = sizeof(int); + int last_good = 0; + int min, max, avg; + int old_size; + + /* Start with the default size. */ + if (getsockopt(sfd, SOL_SOCKET, SO_SNDBUF, &old_size, &intsize) != 0) { + if (settings.verbose > 0) + perror("getsockopt(SO_SNDBUF)"); + return; + } + + /* Binary-search for the real maximum. */ + min = old_size; + max = MAX_SENDBUF_SIZE; + + while (min <= max) { + avg = ((unsigned int)(min + max)) / 2; + if (setsockopt(sfd, SOL_SOCKET, SO_SNDBUF, (void *)&avg, intsize) == 0) { + last_good = avg; + min = avg + 1; + } else { + max = avg - 1; + } + } + + if (settings.verbose > 1) + fprintf(stderr, "<%d send buffer was %d, now %d\n", sfd, old_size, last_good); +} + +static int server_socket(const int port, const bool is_udp) { + int sfd; + struct linger ling = {0, 0}; + struct addrinfo *ai; + struct addrinfo *next; + struct addrinfo hints; + char port_buf[NI_MAXSERV]; + int error; + int success = 0; + + int flags =1; + + /* + * the memset call clears nonstandard fields in some impementations + * that otherwise mess things up. + */ + memset(&hints, 0, sizeof (hints)); + hints.ai_flags = AI_PASSIVE|AI_ADDRCONFIG; + if (is_udp) + { + hints.ai_protocol = IPPROTO_UDP; + hints.ai_socktype = SOCK_DGRAM; + hints.ai_family = AF_INET; /* This left here because of issues with OSX 10.5 */ + } else { + hints.ai_family = AF_UNSPEC; + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + } + + snprintf(port_buf, NI_MAXSERV, "%d", port); + error= getaddrinfo(settings.inter, port_buf, &hints, &ai); + if (error != 0) { + if (error != EAI_SYSTEM) + fprintf(stderr, "getaddrinfo(): %s\n", gai_strerror(error)); + else + perror("getaddrinfo()"); + + return 1; + } + + for (next= ai; next; next= next->ai_next) { + conn *listen_conn_add; + if ((sfd = new_socket(next)) == -1) { + freeaddrinfo(ai); + return 1; + } + + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags)); + if (is_udp) { + maximize_sndbuf(sfd); + } else { + setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags)); + setsockopt(sfd, SOL_SOCKET, SO_LINGER, (void *)&ling, sizeof(ling)); + setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, (void *)&flags, sizeof(flags)); + } + + if (bind(sfd, next->ai_addr, next->ai_addrlen) == -1) { + if (errno != EADDRINUSE) { + perror("bind()"); + close(sfd); + freeaddrinfo(ai); + return 1; + } + close(sfd); + continue; + } else { + success++; + if (!is_udp && listen(sfd, 1024) == -1) { + perror("listen()"); + close(sfd); + freeaddrinfo(ai); + return 1; + } + } + + if (is_udp) + { + int c; + + for (c = 0; c < settings.num_threads; c++) { + /* this is guaranteed to hit all threads because we round-robin */ + dispatch_conn_new(sfd, conn_read, EV_READ | EV_PERSIST, + UDP_READ_BUFFER_SIZE, 1); + } + } else { + if (!(listen_conn_add = conn_new(sfd, conn_listening, + EV_READ | EV_PERSIST, 1, false, main_base))) { + fprintf(stderr, "failed to create listening connection\n"); + exit(EXIT_FAILURE); + } + + listen_conn_add->next = listen_conn; + listen_conn = listen_conn_add; + } + } + + freeaddrinfo(ai); + + /* Return zero iff we detected no errors in starting up connections */ + return success == 0; +} + +static int new_socket_unix(void) { + int sfd; + int flags; + + if ((sfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { + perror("socket()"); + return -1; + } + + if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 || + fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) { + perror("setting O_NONBLOCK"); + close(sfd); + return -1; + } + return sfd; +} + +static int server_socket_unix(const char *path, int access_mask) { + int sfd; + struct linger ling = {0, 0}; + struct sockaddr_un addr; + struct stat tstat; + int flags =1; + int old_umask; + + if (!path) { + return 1; + } + + if ((sfd = new_socket_unix()) == -1) { + return 1; + } + + /* + * Clean up a previous socket file if we left it around + */ + if (lstat(path, &tstat) == 0) { + if (S_ISSOCK(tstat.st_mode)) + unlink(path); + } + + setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags)); + setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags)); + setsockopt(sfd, SOL_SOCKET, SO_LINGER, (void *)&ling, sizeof(ling)); + + /* + * the memset call clears nonstandard fields in some impementations + * that otherwise mess things up. + */ + memset(&addr, 0, sizeof(addr)); + + addr.sun_family = AF_UNIX; + strcpy(addr.sun_path, path); + old_umask=umask( ~(access_mask&0777)); + if (bind(sfd, (struct sockaddr *)&addr, sizeof(addr)) == -1) { + perror("bind()"); + close(sfd); + umask(old_umask); + return 1; + } + umask(old_umask); + if (listen(sfd, 1024) == -1) { + perror("listen()"); + close(sfd); + return 1; + } + if (!(listen_conn = conn_new(sfd, conn_listening, + EV_READ | EV_PERSIST, 1, false, main_base))) { + fprintf(stderr, "failed to create listening connection\n"); + exit(EXIT_FAILURE); + } + + return 0; +} + +/* + * We keep the current time of day in a global variable that's updated by a + * timer event. This saves us a bunch of time() system calls (we really only + * need to get the time once a second, whereas there can be tens of thousands + * of requests a second) and allows us to use server-start-relative timestamps + * rather than absolute UNIX timestamps, a space savings on systems where + * sizeof(time_t) > sizeof(unsigned int). + */ +volatile rel_time_t current_time; +static struct event clockevent; + +/* time-sensitive callers can call it by hand with this, outside the normal ever-1-second timer */ +static void set_current_time(void) { + struct timeval timer; + + gettimeofday(&timer, NULL); + current_time = (rel_time_t) (timer.tv_sec - stats.started); +} + +static void clock_handler(const int fd, const short which, void *arg) { + struct timeval t = {.tv_sec = 1, .tv_usec = 0}; + static bool initialized = false; + + if (initialized) { + /* only delete the event if it's actually there. */ + evtimer_del(&clockevent); + } else { + initialized = true; + } + + evtimer_set(&clockevent, clock_handler, 0); + event_base_set(main_base, &clockevent); + evtimer_add(&clockevent, &t); + + set_current_time(); +} + +static void usage(void) { + printf(PACKAGE " " VERSION "\n"); + printf("-p <num> TCP port number to listen on (default: 11211)\n" + "-U <num> UDP port number to listen on (default: 0, off)\n" + "-s <file> unix socket path to listen on (disables network support)\n" + "-a <mask> access mask for unix socket, in octal (default 0700)\n" + "-l <ip_addr> interface to listen on, default is INDRR_ANY\n" + "-d run as a daemon\n" + "-r maximize core file limit\n" + "-u <username> assume identity of <username> (only when run as root)\n" + "-m <num> max memory to use for items in megabytes, default is 64 MB\n" + "-M return error on memory exhausted (rather than removing items)\n" + "-c <num> max simultaneous connections, default is 1024\n" + "-k lock down all paged memory. Note that there is a\n" + " limit on how much memory you may lock. Trying to\n" + " allocate more than that would fail, so be sure you\n" + " set the limit correctly for the user you started\n" + " the daemon with (not for -u <username> user;\n" + " under sh this is done with 'ulimit -S -l NUM_KB').\n" + "-v verbose (print errors/warnings while in event loop)\n" + "-vv very verbose (also print client commands/reponses)\n" + "-h print this help and exit\n" + "-i print memcached and libevent license\n" + "-b run a managed instanced (mnemonic: buckets)\n" + "-P <file> save PID in <file>, only used with -d option\n" + "-f <factor> chunk size growth factor, default 1.25\n" + "-n <bytes> minimum space allocated for key+value+flags, default 48\n" + +#if defined(HAVE_GETPAGESIZES) && defined(HAVE_MEMCNTL) + "-L Try to use large memory pages (if available). Increasing\n" + " the memory page size could reduce the number of TLB misses\n" + " and improve the performance. In order to get large pages\n" + " from the OS, memcached will allocate the total item-cache\n" + " in one large chunk.\n" +#endif + ); + +#ifdef USE_THREADS + printf("-t <num> number of threads to use, default 4\n"); +#endif + return; +} + +static void usage_license(void) { + printf(PACKAGE " " VERSION "\n\n"); + printf( + "Copyright (c) 2003, Danga Interactive, Inc. <http://www.danga.com/>\n" + "All rights reserved.\n" + "\n" + "Redistribution and use in source and binary forms, with or without\n" + "modification, are permitted provided that the following conditions are\n" + "met:\n" + "\n" + " * Redistributions of source code must retain the above copyright\n" + "notice, this list of conditions and the following disclaimer.\n" + "\n" + " * Redistributions in binary form must reproduce the above\n" + "copyright notice, this list of conditions and the following disclaimer\n" + "in the documentation and/or other materials provided with the\n" + "distribution.\n" + "\n" + " * Neither the name of the Danga Interactive nor the names of its\n" + "contributors may be used to endorse or promote products derived from\n" + "this software without specific prior written permission.\n" + "\n" + "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" + "\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" + "LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" + "A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" + "OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" + "SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" + "LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" + "DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" + "THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" + "(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" + "OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + "\n" + "\n" + "This product includes software developed by Niels Provos.\n" + "\n" + "[ libevent ]\n" + "\n" + "Copyright 2000-2003 Niels Provos <provos@citi.umich.edu>\n" + "All rights reserved.\n" + "\n" + "Redistribution and use in source and binary forms, with or without\n" + "modification, are permitted provided that the following conditions\n" + "are met:\n" + "1. Redistributions of source code must retain the above copyright\n" + " notice, this list of conditions and the following disclaimer.\n" + "2. Redistributions in binary form must reproduce the above copyright\n" + " notice, this list of conditions and the following disclaimer in the\n" + " documentation and/or other materials provided with the distribution.\n" + "3. All advertising materials mentioning features or use of this software\n" + " must display the following acknowledgement:\n" + " This product includes software developed by Niels Provos.\n" + "4. The name of the author may not be used to endorse or promote products\n" + " derived from this software without specific prior written permission.\n" + "\n" + "THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n" + "IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n" + "OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n" + "IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n" + "INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n" + "NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" + "DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" + "THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" + "(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n" + "THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + ); + + return; +} + +static void save_pid(const pid_t pid, const char *pid_file) { + FILE *fp; + if (pid_file == NULL) + return; + + if ((fp = fopen(pid_file, "w")) == NULL) { + fprintf(stderr, "Could not open the pid file %s for writing\n", pid_file); + return; + } + + fprintf(fp,"%ld\n", (long)pid); + if (fclose(fp) == -1) { + fprintf(stderr, "Could not close the pid file %s.\n", pid_file); + return; + } +} + +static void remove_pidfile(const char *pid_file) { + if (pid_file == NULL) + return; + + if (unlink(pid_file) != 0) { + fprintf(stderr, "Could not remove the pid file %s.\n", pid_file); + } + +} + + +static void sig_handler(const int sig) { + printf("SIGINT handled.\n"); + exit(EXIT_SUCCESS); +} + +#if defined(HAVE_GETPAGESIZES) && defined(HAVE_MEMCNTL) +/* + * On systems that supports multiple page sizes we may reduce the + * number of TLB-misses by using the biggest available page size + */ +int enable_large_pages(void) { + int ret = -1; + size_t sizes[32]; + int avail = getpagesizes(sizes, 32); + if (avail != -1) { + size_t max = sizes[0]; + struct memcntl_mha arg = {0}; + int ii; + + for (ii = 1; ii < avail; ++ii) { + if (max < sizes[ii]) { + max = sizes[ii]; + } + } + + arg.mha_flags = 0; + arg.mha_pagesize = max; + arg.mha_cmd = MHA_MAPSIZE_BSSBRK; + + if (memcntl(0, 0, MC_HAT_ADVISE, (caddr_t)&arg, 0, 0) == -1) { + fprintf(stderr, "Failed to set large pages: %s\n", + strerror(errno)); + fprintf(stderr, "Will use default page size\n"); + } else { + ret = 0; + } + } else { + fprintf(stderr, "Failed to get supported pagesizes: %s\n", + strerror(errno)); + fprintf(stderr, "Will use default page size\n"); + } + + return ret; +} +#endif + +int main (int argc, char **argv) { + int c; + int x; + bool lock_memory = false; + bool daemonize = false; + bool preallocate = false; + int maxcore = 0; + char *username = NULL; + char *pid_file = NULL; + struct passwd *pw; + struct sigaction sa; + struct rlimit rlim; + /* listening socket */ + static int *l_socket = NULL; + + /* udp socket */ + static int *u_socket = NULL; + static int u_socket_count = 0; + + /* handle SIGINT */ + signal(SIGINT, sig_handler); + + /* init settings */ + settings_init(); + + /* set stderr non-buffering (for running under, say, daemontools) */ + setbuf(stderr, NULL); + + /* process arguments */ + while ((c = getopt(argc, argv, "x:o:a:bp:s:U:m:Mc:khirvdl:u:P:f:s:n:t:L")) != -1) { + switch (c) { + case 'x': + settings.srilm = optarg; + break; + case 'o': + settings.srilm_order = atoi(optarg); + break; + case 'a': + /* access for unix domain socket, as octal mask (like chmod)*/ + settings.access= strtol(optarg,NULL,8); + break; + + case 'U': + settings.udpport = atoi(optarg); + break; + case 'b': + settings.managed = true; + break; + case 'p': + settings.port = atoi(optarg); + break; + case 's': + settings.socketpath = optarg; + break; + case 'm': + settings.maxbytes = ((size_t)atoi(optarg)) * 1024 * 1024; + break; + case 'M': + settings.evict_to_free = 0; + break; + case 'c': + settings.maxconns = atoi(optarg); + break; + case 'h': + usage(); + exit(EXIT_SUCCESS); + case 'i': + usage_license(); + exit(EXIT_SUCCESS); + case 'k': + lock_memory = true; + break; + case 'v': + settings.verbose++; + break; + case 'l': + settings.inter= strdup(optarg); + break; + case 'd': + daemonize = true; + break; + case 'r': + maxcore = 1; + break; + case 'u': + username = optarg; + break; + case 'P': + pid_file = optarg; + break; + case 'f': + settings.factor = atof(optarg); + if (settings.factor <= 1.0) { + fprintf(stderr, "Factor must be greater than 1\n"); + return 1; + } + break; + case 'n': + settings.chunk_size = atoi(optarg); + if (settings.chunk_size == 0) { + fprintf(stderr, "Chunk size must be greater than 0\n"); + return 1; + } + break; + case 't': + settings.num_threads = atoi(optarg); + if (settings.num_threads == 0) { + fprintf(stderr, "Number of threads must be greater than 0\n"); + return 1; + } + break; +#if defined(HAVE_GETPAGESIZES) && defined(HAVE_MEMCNTL) + case 'L' : + if (enable_large_pages() == 0) { + preallocate = true; + } + break; +#endif + default: + fprintf(stderr, "Illegal argument \"%c\"\n", c); + return 1; + } + } + + if (maxcore != 0) { + struct rlimit rlim_new; + /* + * First try raising to infinity; if that fails, try bringing + * the soft limit to the hard. + */ + if (getrlimit(RLIMIT_CORE, &rlim) == 0) { + rlim_new.rlim_cur = rlim_new.rlim_max = RLIM_INFINITY; + if (setrlimit(RLIMIT_CORE, &rlim_new)!= 0) { + /* failed. try raising just to the old max */ + rlim_new.rlim_cur = rlim_new.rlim_max = rlim.rlim_max; + (void)setrlimit(RLIMIT_CORE, &rlim_new); + } + } + /* + * getrlimit again to see what we ended up with. Only fail if + * the soft limit ends up 0, because then no core files will be + * created at all. + */ + + if ((getrlimit(RLIMIT_CORE, &rlim) != 0) || rlim.rlim_cur == 0) { + fprintf(stderr, "failed to ensure corefile creation\n"); + exit(EXIT_FAILURE); + } + } + + /* + * If needed, increase rlimits to allow as many connections + * as needed. + */ + + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { + fprintf(stderr, "failed to getrlimit number of files\n"); + exit(EXIT_FAILURE); + } else { + int maxfiles = settings.maxconns; + if (rlim.rlim_cur < maxfiles) + rlim.rlim_cur = maxfiles + 3; + if (rlim.rlim_max < rlim.rlim_cur) + rlim.rlim_max = rlim.rlim_cur; + if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) { + fprintf(stderr, "failed to set rlimit for open files. Try running as root or requesting smaller maxconns value.\n"); + exit(EXIT_FAILURE); + } + } + + /* daemonize if requested */ + /* if we want to ensure our ability to dump core, don't chdir to / */ + if (daemonize) { + int res; + res = daemon(maxcore, settings.verbose); + if (res == -1) { + fprintf(stderr, "failed to daemon() in order to daemonize\n"); + return 1; + } + } + + /* lock paged memory if needed */ + if (lock_memory) { +#ifdef HAVE_MLOCKALL + int res = mlockall(MCL_CURRENT | MCL_FUTURE); + if (res != 0) { + fprintf(stderr, "warning: -k invalid, mlockall() failed: %s\n", + strerror(errno)); + } +#else + fprintf(stderr, "warning: -k invalid, mlockall() not supported on this platform. proceeding without.\n"); +#endif + } + + /* lose root privileges if we have them */ + if (getuid() == 0 || geteuid() == 0) { + if (username == 0 || *username == '\0') { + fprintf(stderr, "can't run as root without the -u switch\n"); + return 1; + } + if ((pw = getpwnam(username)) == 0) { + fprintf(stderr, "can't find the user %s to switch to\n", username); + return 1; + } + if (setgid(pw->pw_gid) < 0 || setuid(pw->pw_uid) < 0) { + fprintf(stderr, "failed to assume identity of user %s\n", username); + return 1; + } + } + + /* initialize main thread libevent instance */ + main_base = event_init(); + + /* initialize other stuff */ + stats_init(); + conn_init(); + if (!settings.srilm) { + fprintf(stderr, "please specify a LM file with -x\n"); + exit(EXIT_FAILURE); + } + srilm_init(settings.srilm, settings.srilm_order); + + /* managed instance? alloc and zero a bucket array */ + if (settings.managed) { + buckets = malloc(sizeof(int) * MAX_BUCKETS); + if (buckets == 0) { + fprintf(stderr, "failed to allocate the bucket array"); + exit(EXIT_FAILURE); + } + memset(buckets, 0, sizeof(int) * MAX_BUCKETS); + } + + /* + * ignore SIGPIPE signals; we can use errno==EPIPE if we + * need that information + */ + sa.sa_handler = SIG_IGN; + sa.sa_flags = 0; + if (sigemptyset(&sa.sa_mask) == -1 || + sigaction(SIGPIPE, &sa, 0) == -1) { + perror("failed to ignore SIGPIPE; sigaction"); + exit(EXIT_FAILURE); + } + /* start up worker threads if MT mode */ + thread_init(settings.num_threads, main_base); + /* save the PID in if we're a daemon, do this after thread_init due to + a file descriptor handling bug somewhere in libevent */ + if (daemonize) + save_pid(getpid(), pid_file); + /* initialise clock event */ + clock_handler(0, 0, 0); + + /* create unix mode sockets after dropping privileges */ + if (settings.socketpath != NULL) { + if (server_socket_unix(settings.socketpath,settings.access)) { + fprintf(stderr, "failed to listen\n"); + exit(EXIT_FAILURE); + } + } + + /* create the listening socket, bind it, and init */ + if (settings.socketpath == NULL) { + int udp_port; + + if (server_socket(settings.port, 0)) { + fprintf(stderr, "failed to listen\n"); + exit(EXIT_FAILURE); + } + /* + * initialization order: first create the listening sockets + * (may need root on low ports), then drop root if needed, + * then daemonise if needed, then init libevent (in some cases + * descriptors created by libevent wouldn't survive forking). + */ + udp_port = settings.udpport ? settings.udpport : settings.port; + + /* create the UDP listening socket and bind it */ + if (server_socket(udp_port, 1)) { + fprintf(stderr, "failed to listen on UDP port %d\n", settings.udpport); + exit(EXIT_FAILURE); + } + } + + /* enter the event loop */ + event_base_loop(main_base, 0); + /* remove the PID file if we're a daemon */ + if (daemonize) + remove_pidfile(pid_file); + /* Clean up strdup() call for bind() address */ + if (settings.inter) + free(settings.inter); + if (l_socket) + free(l_socket); + if (u_socket) + free(u_socket); + + return 0; +} diff --git a/contrib/lmserver/lmserver.h b/contrib/lmserver/lmserver.h new file mode 100644 index 000000000..1afdd133b --- /dev/null +++ b/contrib/lmserver/lmserver.h @@ -0,0 +1,375 @@ +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* $Id$ */ +#ifndef lmserver_lmserver_h +#define lmserver_lmserver_h + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <netinet/in.h> +#include <event.h> +#include <netdb.h> + +#define DATA_BUFFER_SIZE 2048 +#define UDP_READ_BUFFER_SIZE 65536 +#define UDP_MAX_PAYLOAD_SIZE 1400 +#define UDP_HEADER_SIZE 8 +#define MAX_SENDBUF_SIZE (256 * 1024 * 1024) +/* I'm told the max legnth of a 64-bit num converted to string is 20 bytes. + * Plus a few for spaces, \r\n, \0 */ +#define SUFFIX_SIZE 24 + +/** Initial size of list of items being returned by "get". */ +#define ITEM_LIST_INITIAL 200 + +/** Initial size of list of CAS suffixes appended to "gets" lines. */ +#define SUFFIX_LIST_INITIAL 20 + +/** Initial size of the sendmsg() scatter/gather array. */ +#define IOV_LIST_INITIAL 400 + +/** Initial number of sendmsg() argument structures to allocate. */ +#define MSG_LIST_INITIAL 10 + +/** High water marks for buffer shrinking */ +#define READ_BUFFER_HIGHWAT 8192 +#define ITEM_LIST_HIGHWAT 400 +#define IOV_LIST_HIGHWAT 600 +#define MSG_LIST_HIGHWAT 100 + +/* Get a consistent bool type */ +#if HAVE_STDBOOL_H +# include <stdbool.h> +#else + typedef enum {false = 0, true = 1} bool; +#endif + +#if HAVE_STDINT_H +# include <stdint.h> +#else + typedef unsigned char uint8_t; +#endif + +/* unistd.h is here */ +#if HAVE_UNISTD_H +# include <unistd.h> +#endif + +/** Time relative to server start. Smaller than time_t on 64-bit systems. */ +typedef unsigned int rel_time_t; + +struct stats { + unsigned int curr_items; + unsigned int total_items; + uint64_t curr_bytes; + unsigned int curr_conns; + unsigned int total_conns; + unsigned int conn_structs; + uint64_t get_cmds; + uint64_t set_cmds; + uint64_t get_hits; + uint64_t get_misses; + uint64_t evictions; + time_t started; /* when the process was started */ + uint64_t bytes_read; + uint64_t bytes_written; +}; + +#define MAX_VERBOSITY_LEVEL 2 + +struct settings { + size_t maxbytes; + int maxconns; + int port; + int udpport; + char *srilm; + int srilm_order; + char *inter; + int verbose; + rel_time_t oldest_live; /* ignore existing items older than this */ + bool managed; /* if 1, a tracker manages virtual buckets */ + int evict_to_free; + char *socketpath; /* path to unix socket if using local socket */ + int access; /* access mask (a la chmod) for unix domain socket */ + double factor; /* chunk size growth factor */ + int chunk_size; + int num_threads; /* number of libevent threads to run */ + char prefix_delimiter; /* character that marks a key prefix (for stats) */ + int detail_enabled; /* nonzero if we're collecting detailed stats */ +}; + +extern struct stats stats; +extern struct settings settings; + +#define ITEM_LINKED 1 +#define ITEM_DELETED 2 + +/* temp */ +#define ITEM_SLABBED 4 + +typedef struct _stritem { + struct _stritem *next; + struct _stritem *prev; + struct _stritem *h_next; /* hash chain next */ + rel_time_t time; /* least recent access */ + rel_time_t exptime; /* expire time */ + int nbytes; /* size of data */ + unsigned short refcount; + uint8_t nsuffix; /* length of flags-and-length string */ + uint8_t it_flags; /* ITEM_* above */ + uint8_t slabs_clsid;/* which slab class we're in */ + uint8_t nkey; /* key length, w/terminating null and padding */ + uint64_t cas_id; /* the CAS identifier */ + void * end[]; + /* then null-terminated key */ + /* then " flags length\r\n" (no terminating null) */ + /* then data with terminating \r\n (no terminating null; it's binary!) */ +} item; + +#define ITEM_key(item) ((char*)&((item)->end[0])) + +/* warning: don't use these macros with a function, as it evals its arg twice */ +#define ITEM_suffix(item) ((char*) &((item)->end[0]) + (item)->nkey + 1) +#define ITEM_data(item) ((char*) &((item)->end[0]) + (item)->nkey + 1 + (item)->nsuffix) +#define ITEM_ntotal(item) (sizeof(struct _stritem) + (item)->nkey + 1 + (item)->nsuffix + (item)->nbytes) + +enum conn_states { + conn_listening, /** the socket which listens for connections */ + conn_read, /** reading in a command line */ + conn_write, /** writing out a simple response */ + conn_nread, /** reading in a fixed number of bytes */ + conn_swallow, /** swallowing unnecessary bytes w/o storing */ + conn_closing, /** closing this connection */ + conn_mwrite, /** writing out many items sequentially */ +}; + +#define NREAD_ADD 1 +#define NREAD_SET 2 +#define NREAD_REPLACE 3 +#define NREAD_APPEND 4 +#define NREAD_PREPEND 5 +#define NREAD_CAS 6 + +typedef struct conn conn; +struct conn { + int sfd; + int state; + struct event event; + short ev_flags; + short which; /** which events were just triggered */ + + char *rbuf; /** buffer to read commands into */ + char *rcurr; /** but if we parsed some already, this is where we stopped */ + int rsize; /** total allocated size of rbuf */ + int rbytes; /** how much data, starting from rcur, do we have unparsed */ + + char *wbuf; + char *wcurr; + int wsize; + int wbytes; + int write_and_go; /** which state to go into after finishing current write */ + void *write_and_free; /** free this memory after finishing writing */ + + char *ritem; /** when we read in an item's value, it goes here */ + int rlbytes; + + /* data for the nread state */ + + /** + * item is used to hold an item structure created after reading the command + * line of set/add/replace commands, but before we finished reading the actual + * data. The data is read into ITEM_data(item) to avoid extra copying. + */ + + void *item; /* for commands set/add/replace */ + int item_comm; /* which one is it: set/add/replace */ + + /* data for the swallow state */ + int sbytes; /* how many bytes to swallow */ + + /* data for the mwrite state */ + struct iovec *iov; + int iovsize; /* number of elements allocated in iov[] */ + int iovused; /* number of elements used in iov[] */ + + struct msghdr *msglist; + int msgsize; /* number of elements allocated in msglist[] */ + int msgused; /* number of elements used in msglist[] */ + int msgcurr; /* element in msglist[] being transmitted now */ + int msgbytes; /* number of bytes in current msg */ + + item **ilist; /* list of items to write out */ + int isize; + item **icurr; + int ileft; + + char **suffixlist; + int suffixsize; + char **suffixcurr; + int suffixleft; + + /* data for UDP clients */ + bool udp; /* is this is a UDP "connection" */ + int request_id; /* Incoming UDP request ID, if this is a UDP "connection" */ + struct sockaddr request_addr; /* Who sent the most recent request */ + socklen_t request_addr_size; + unsigned char *hdrbuf; /* udp packet headers */ + int hdrsize; /* number of headers' worth of space is allocated */ + + int binary; /* are we in binary mode */ + int bucket; /* bucket number for the next command, if running as + a managed instance. -1 (_not_ 0) means invalid. */ + int gen; /* generation requested for the bucket */ + bool noreply; /* True if the reply should not be sent. */ + conn *next; /* Used for generating a list of conn structures */ +}; + +/* number of virtual buckets for a managed instance */ +#define MAX_BUCKETS 32768 + +/* current time of day (updated periodically) */ +extern volatile rel_time_t current_time; + +/* + * Functions + */ + +conn *do_conn_from_freelist(); +bool do_conn_add_to_freelist(conn *c); +conn *conn_new(const int sfd, const int init_state, const int event_flags, const int read_buffer_size, const bool is_udp, struct event_base *base); + + +#include "stats.h" +//#include "slabs.h" +//#include "assoc.h" +//#include "items.h" +//#include "memcached_dtrace.h" + +/* + * In multithreaded mode, we wrap certain functions with lock management and + * replace the logic of some other functions. All wrapped functions have + * "mt_" and "do_" variants. In multithreaded mode, the plain version of a + * function is #define-d to the "mt_" variant, which often just grabs a + * lock and calls the "do_" function. In singlethreaded mode, the "do_" + * function is called directly. + * + * Functions such as the libevent-related calls that need to do cross-thread + * communication in multithreaded mode (rather than actually doing the work + * in the current thread) are called via "dispatch_" frontends, which are + * also #define-d to directly call the underlying code in singlethreaded mode. + */ +#ifdef USE_THREADS + +void thread_init(int nthreads, struct event_base *main_base); +int dispatch_event_add(int thread, conn *c); +void dispatch_conn_new(int sfd, int init_state, int event_flags, int read_buffer_size, int is_udp); + +/* Lock wrappers for cache functions that are called from main loop. */ +char *mt_add_delta(conn *c, item *item, const int incr, const int64_t delta, + char *buf); +void mt_assoc_move_next_bucket(void); +conn *mt_conn_from_freelist(void); +bool mt_conn_add_to_freelist(conn *c); +char *mt_suffix_from_freelist(void); +bool mt_suffix_add_to_freelist(char *s); +char *mt_defer_delete(item *it, time_t exptime); +int mt_is_listen_thread(void); +item *mt_item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes); +char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes); +void mt_item_flush_expired(void); +item *mt_item_get_notedeleted(const char *key, const size_t nkey, bool *delete_locked); +int mt_item_link(item *it); +void mt_item_remove(item *it); +int mt_item_replace(item *it, item *new_it); +char *mt_item_stats(int *bytes); +char *mt_item_stats_sizes(int *bytes); +void mt_item_unlink(item *it); +void mt_item_update(item *it); +void mt_run_deferred_deletes(void); +void *mt_slabs_alloc(size_t size, unsigned int id); +void mt_slabs_free(void *ptr, size_t size, unsigned int id); +int mt_slabs_reassign(unsigned char srcid, unsigned char dstid); +char *mt_slabs_stats(int *buflen); +void mt_stats_lock(void); +void mt_stats_unlock(void); +int mt_store_item(item *item, int comm); + + +# define add_delta(c,x,y,z,a) mt_add_delta(c,x,y,z,a) +# define assoc_move_next_bucket() mt_assoc_move_next_bucket() +# define conn_from_freelist() mt_conn_from_freelist() +# define conn_add_to_freelist(x) mt_conn_add_to_freelist(x) +# define suffix_from_freelist() mt_suffix_from_freelist() +# define suffix_add_to_freelist(x) mt_suffix_add_to_freelist(x) +# define defer_delete(x,y) mt_defer_delete(x,y) +# define is_listen_thread() mt_is_listen_thread() +# define item_alloc(x,y,z,a,b) mt_item_alloc(x,y,z,a,b) +# define item_cachedump(x,y,z) mt_item_cachedump(x,y,z) +# define item_flush_expired() mt_item_flush_expired() +# define item_get_notedeleted(x,y,z) mt_item_get_notedeleted(x,y,z) +# define item_link(x) mt_item_link(x) +# define item_remove(x) mt_item_remove(x) +# define item_replace(x,y) mt_item_replace(x,y) +# define item_stats(x) mt_item_stats(x) +# define item_stats_sizes(x) mt_item_stats_sizes(x) +# define item_update(x) mt_item_update(x) +# define item_unlink(x) mt_item_unlink(x) +# define run_deferred_deletes() mt_run_deferred_deletes() +# define slabs_alloc(x,y) mt_slabs_alloc(x,y) +# define slabs_free(x,y,z) mt_slabs_free(x,y,z) +# define slabs_reassign(x,y) mt_slabs_reassign(x,y) +# define slabs_stats(x) mt_slabs_stats(x) +# define store_item(x,y) mt_store_item(x,y) + +# define STATS_LOCK() mt_stats_lock() +# define STATS_UNLOCK() mt_stats_unlock() + +#else /* !USE_THREADS */ + +# define add_delta(c,x,y,z,a) do_add_delta(c,x,y,z,a) +# define assoc_move_next_bucket() do_assoc_move_next_bucket() +# define conn_from_freelist() do_conn_from_freelist() +# define conn_add_to_freelist(x) do_conn_add_to_freelist(x) +# define suffix_from_freelist() do_suffix_from_freelist() +# define suffix_add_to_freelist(x) do_suffix_add_to_freelist(x) +# define defer_delete(x,y) do_defer_delete(x,y) +# define dispatch_conn_new(x,y,z,a,b) conn_new(x,y,z,a,b,main_base) +# define dispatch_event_add(t,c) event_add(&(c)->event, 0) +# define is_listen_thread() 1 +# define item_alloc(x,y,z,a,b) do_item_alloc(x,y,z,a,b) +# define item_cachedump(x,y,z) do_item_cachedump(x,y,z) +# define item_flush_expired() do_item_flush_expired() +# define item_get_notedeleted(x,y,z) do_item_get_notedeleted(x,y,z) +# define item_link(x) do_item_link(x) +# define item_remove(x) do_item_remove(x) +# define item_replace(x,y) do_item_replace(x,y) +# define item_stats(x) do_item_stats(x) +# define item_stats_sizes(x) do_item_stats_sizes(x) +# define item_unlink(x) do_item_unlink(x) +# define item_update(x) do_item_update(x) +# define run_deferred_deletes() do_run_deferred_deletes() +# define slabs_alloc(x,y) do_slabs_alloc(x,y) +# define slabs_free(x,y,z) do_slabs_free(x,y,z) +# define slabs_reassign(x,y) do_slabs_reassign(x,y) +# define slabs_stats(x) do_slabs_stats(x) +# define store_item(x,y) do_store_item(x,y) +# define thread_init(x,y) 0 + +# define STATS_LOCK() /**/ +# define STATS_UNLOCK() /**/ + +#endif /* !USE_THREADS */ + +/* If supported, give compiler hints for branch prediction. */ +#if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) +#define __builtin_expect(x, expected_value) (x) +#endif + +#define likely(x) __builtin_expect((x),1) +#define unlikely(x) __builtin_expect((x),0) + +#endif diff --git a/contrib/lmserver/missing b/contrib/lmserver/missing new file mode 100755 index 000000000..1c8ff7049 --- /dev/null +++ b/contrib/lmserver/missing @@ -0,0 +1,367 @@ +#! /bin/sh +# Common stub for a few missing GNU programs while installing. + +scriptversion=2006-05-10.23 + +# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006 +# Free Software Foundation, Inc. +# Originally by Fran,cois Pinard <pinard@iro.umontreal.ca>, 1996. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +if test $# -eq 0; then + echo 1>&2 "Try \`$0 --help' for more information" + exit 1 +fi + +run=: +sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p' +sed_minuso='s/.* -o \([^ ]*\).*/\1/p' + +# In the cases where this matters, `missing' is being run in the +# srcdir already. +if test -f configure.ac; then + configure_ac=configure.ac +else + configure_ac=configure.in +fi + +msg="missing on your system" + +case $1 in +--run) + # Try to run requested program, and just exit if it succeeds. + run= + shift + "$@" && exit 0 + # Exit code 63 means version mismatch. This often happens + # when the user try to use an ancient version of a tool on + # a file that requires a minimum version. In this case we + # we should proceed has if the program had been absent, or + # if --run hadn't been passed. + if test $? = 63; then + run=: + msg="probably too old" + fi + ;; + + -h|--h|--he|--hel|--help) + echo "\ +$0 [OPTION]... PROGRAM [ARGUMENT]... + +Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an +error status if there is no known handling for PROGRAM. + +Options: + -h, --help display this help and exit + -v, --version output version information and exit + --run try to run the given command, and emulate it if it fails + +Supported PROGRAM values: + aclocal touch file \`aclocal.m4' + autoconf touch file \`configure' + autoheader touch file \`config.h.in' + autom4te touch the output file, or create a stub one + automake touch all \`Makefile.in' files + bison create \`y.tab.[ch]', if possible, from existing .[ch] + flex create \`lex.yy.c', if possible, from existing .c + help2man touch the output file + lex create \`lex.yy.c', if possible, from existing .c + makeinfo touch the output file + tar try tar, gnutar, gtar, then tar without non-portable flags + yacc create \`y.tab.[ch]', if possible, from existing .[ch] + +Send bug reports to <bug-automake@gnu.org>." + exit $? + ;; + + -v|--v|--ve|--ver|--vers|--versi|--versio|--version) + echo "missing $scriptversion (GNU Automake)" + exit $? + ;; + + -*) + echo 1>&2 "$0: Unknown \`$1' option" + echo 1>&2 "Try \`$0 --help' for more information" + exit 1 + ;; + +esac + +# Now exit if we have it, but it failed. Also exit now if we +# don't have it and --version was passed (most likely to detect +# the program). +case $1 in + lex|yacc) + # Not GNU programs, they don't have --version. + ;; + + tar) + if test -n "$run"; then + echo 1>&2 "ERROR: \`tar' requires --run" + exit 1 + elif test "x$2" = "x--version" || test "x$2" = "x--help"; then + exit 1 + fi + ;; + + *) + if test -z "$run" && ($1 --version) > /dev/null 2>&1; then + # We have it, but it failed. + exit 1 + elif test "x$2" = "x--version" || test "x$2" = "x--help"; then + # Could not run --version or --help. This is probably someone + # running `$TOOL --version' or `$TOOL --help' to check whether + # $TOOL exists and not knowing $TOOL uses missing. + exit 1 + fi + ;; +esac + +# If it does not exist, or fails to run (possibly an outdated version), +# try to emulate it. +case $1 in + aclocal*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`acinclude.m4' or \`${configure_ac}'. You might want + to install the \`Automake' and \`Perl' packages. Grab them from + any GNU archive site." + touch aclocal.m4 + ;; + + autoconf) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`${configure_ac}'. You might want to install the + \`Autoconf' and \`GNU m4' packages. Grab them from any GNU + archive site." + touch configure + ;; + + autoheader) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`acconfig.h' or \`${configure_ac}'. You might want + to install the \`Autoconf' and \`GNU m4' packages. Grab them + from any GNU archive site." + files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}` + test -z "$files" && files="config.h" + touch_files= + for f in $files; do + case $f in + *:*) touch_files="$touch_files "`echo "$f" | + sed -e 's/^[^:]*://' -e 's/:.*//'`;; + *) touch_files="$touch_files $f.in";; + esac + done + touch $touch_files + ;; + + automake*) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'. + You might want to install the \`Automake' and \`Perl' packages. + Grab them from any GNU archive site." + find . -type f -name Makefile.am -print | + sed 's/\.am$/.in/' | + while read f; do touch "$f"; done + ;; + + autom4te) + echo 1>&2 "\ +WARNING: \`$1' is needed, but is $msg. + You might have modified some files without having the + proper tools for further handling them. + You can get \`$1' as part of \`Autoconf' from any GNU + archive site." + + file=`echo "$*" | sed -n "$sed_output"` + test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` + if test -f "$file"; then + touch $file + else + test -z "$file" || exec >$file + echo "#! /bin/sh" + echo "# Created by GNU Automake missing as a replacement of" + echo "# $ $@" + echo "exit 0" + chmod +x $file + exit 1 + fi + ;; + + bison|yacc) + echo 1>&2 "\ +WARNING: \`$1' $msg. You should only need it if + you modified a \`.y' file. You may need the \`Bison' package + in order for those modifications to take effect. You can get + \`Bison' from any GNU archive site." + rm -f y.tab.c y.tab.h + if test $# -ne 1; then + eval LASTARG="\${$#}" + case $LASTARG in + *.y) + SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'` + if test -f "$SRCFILE"; then + cp "$SRCFILE" y.tab.c + fi + SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'` + if test -f "$SRCFILE"; then + cp "$SRCFILE" y.tab.h + fi + ;; + esac + fi + if test ! -f y.tab.h; then + echo >y.tab.h + fi + if test ! -f y.tab.c; then + echo 'main() { return 0; }' >y.tab.c + fi + ;; + + lex|flex) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified a \`.l' file. You may need the \`Flex' package + in order for those modifications to take effect. You can get + \`Flex' from any GNU archive site." + rm -f lex.yy.c + if test $# -ne 1; then + eval LASTARG="\${$#}" + case $LASTARG in + *.l) + SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'` + if test -f "$SRCFILE"; then + cp "$SRCFILE" lex.yy.c + fi + ;; + esac + fi + if test ! -f lex.yy.c; then + echo 'main() { return 0; }' >lex.yy.c + fi + ;; + + help2man) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified a dependency of a manual page. You may need the + \`Help2man' package in order for those modifications to take + effect. You can get \`Help2man' from any GNU archive site." + + file=`echo "$*" | sed -n "$sed_output"` + test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` + if test -f "$file"; then + touch $file + else + test -z "$file" || exec >$file + echo ".ab help2man is required to generate this page" + exit 1 + fi + ;; + + makeinfo) + echo 1>&2 "\ +WARNING: \`$1' is $msg. You should only need it if + you modified a \`.texi' or \`.texinfo' file, or any other file + indirectly affecting the aspect of the manual. The spurious + call might also be the consequence of using a buggy \`make' (AIX, + DU, IRIX). You might want to install the \`Texinfo' package or + the \`GNU make' package. Grab either from any GNU archive site." + # The file to touch is that specified with -o ... + file=`echo "$*" | sed -n "$sed_output"` + test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` + if test -z "$file"; then + # ... or it is the one specified with @setfilename ... + infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'` + file=`sed -n ' + /^@setfilename/{ + s/.* \([^ ]*\) *$/\1/ + p + q + }' $infile` + # ... or it is derived from the source name (dir/f.texi becomes f.info) + test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info + fi + # If the file does not exist, the user really needs makeinfo; + # let's fail without touching anything. + test -f $file || exit 1 + touch $file + ;; + + tar) + shift + + # We have already tried tar in the generic part. + # Look for gnutar/gtar before invocation to avoid ugly error + # messages. + if (gnutar --version > /dev/null 2>&1); then + gnutar "$@" && exit 0 + fi + if (gtar --version > /dev/null 2>&1); then + gtar "$@" && exit 0 + fi + firstarg="$1" + if shift; then + case $firstarg in + *o*) + firstarg=`echo "$firstarg" | sed s/o//` + tar "$firstarg" "$@" && exit 0 + ;; + esac + case $firstarg in + *h*) + firstarg=`echo "$firstarg" | sed s/h//` + tar "$firstarg" "$@" && exit 0 + ;; + esac + fi + + echo 1>&2 "\ +WARNING: I can't seem to be able to run \`tar' with the given arguments. + You may want to install GNU tar or Free paxutils, or check the + command line arguments." + exit 1 + ;; + + *) + echo 1>&2 "\ +WARNING: \`$1' is needed, and is $msg. + You might have modified some files without having the + proper tools for further handling them. Check the \`README' file, + it often tells you about the needed prerequisites for installing + this package. You may also peek at any GNU archive site, in case + some other package would contain this missing \`$1' program." + exit 1 + ;; +esac + +exit 0 + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/contrib/lmserver/srilm.cc b/contrib/lmserver/srilm.cc new file mode 100644 index 000000000..657bed3c4 --- /dev/null +++ b/contrib/lmserver/srilm.cc @@ -0,0 +1,29 @@ +#include <cassert> +#include <iostream> +#include "Ngram.h" + +using namespace std; +Vocab vocab; +Ngram* ngram = NULL; + +extern "C" { + +void srilm_init(const char* fname, int order) { + cerr << "Loading " << order << "-gram LM: " << fname << endl; + File file(fname, "r", 0); + assert(file); + ngram = new Ngram(vocab, order); + ngram->read(file, false); + cerr << "Done\n"; +} + +int srilm_getvoc(const char* word) { + return vocab.getIndex((VocabString)word); +} + +float srilm_wordprob(int w, int* context) { + return (float)ngram->wordProb(w, (VocabIndex*)context); +} + +} + diff --git a/contrib/lmserver/srilm.h b/contrib/lmserver/srilm.h new file mode 100644 index 000000000..d9b00ef92 --- /dev/null +++ b/contrib/lmserver/srilm.h @@ -0,0 +1,8 @@ +#ifndef lmserver_srilm_h +#define lmserver_srilm_h + +void srilm_init(const char* fname, int order); +int srilm_getvoc(const char* word); +float srilm_wordprob(int, int*); + +#endif diff --git a/contrib/lmserver/stamp-h1 b/contrib/lmserver/stamp-h1 new file mode 100644 index 000000000..4547fe1b5 --- /dev/null +++ b/contrib/lmserver/stamp-h1 @@ -0,0 +1 @@ +timestamp for config.h diff --git a/contrib/lmserver/stats.h b/contrib/lmserver/stats.h new file mode 100644 index 000000000..895dfcd10 --- /dev/null +++ b/contrib/lmserver/stats.h @@ -0,0 +1,13 @@ +#ifndef lmserver_stats_h +#define lmserver_stats_h + +/* stats */ +void stats_prefix_init(void); +void stats_prefix_clear(void); +void stats_prefix_record_get(const char *key, const bool is_hit); +void stats_prefix_record_delete(const char *key); +void stats_prefix_record_set(const char *key); +/*@null@*/ +char *stats_prefix_dump(int *length); + +#endif diff --git a/contrib/lmserver/thread.c b/contrib/lmserver/thread.c new file mode 100644 index 000000000..7fc9d0463 --- /dev/null +++ b/contrib/lmserver/thread.c @@ -0,0 +1,678 @@ +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Thread management for memcached. + * + * $Id$ + */ +#include "lmserver.h" +#include <stdio.h> +#include <errno.h> +#include <stdlib.h> +#include <errno.h> + +#ifdef HAVE_MALLOC_H +#include <malloc.h> +#endif + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef USE_THREADS + +#include <pthread.h> + +#define ITEMS_PER_ALLOC 64 + +/* An item in the connection queue. */ +typedef struct conn_queue_item CQ_ITEM; +struct conn_queue_item { + int sfd; + int init_state; + int event_flags; + int read_buffer_size; + int is_udp; + CQ_ITEM *next; +}; + +/* A connection queue. */ +typedef struct conn_queue CQ; +struct conn_queue { + CQ_ITEM *head; + CQ_ITEM *tail; + pthread_mutex_t lock; + pthread_cond_t cond; +}; + +/* Lock for connection freelist */ +static pthread_mutex_t conn_lock; + +/* Lock for alternative item suffix freelist */ +static pthread_mutex_t suffix_lock; + +/* Lock for cache operations (item_*, assoc_*) */ +static pthread_mutex_t cache_lock; + +/* Lock for slab allocator operations */ +static pthread_mutex_t slabs_lock; + +/* Lock for global stats */ +static pthread_mutex_t stats_lock; + +/* Free list of CQ_ITEM structs */ +static CQ_ITEM *cqi_freelist; +static pthread_mutex_t cqi_freelist_lock; + +/* + * Each libevent instance has a wakeup pipe, which other threads + * can use to signal that they've put a new connection on its queue. + */ +typedef struct { + pthread_t thread_id; /* unique ID of this thread */ + struct event_base *base; /* libevent handle this thread uses */ + struct event notify_event; /* listen event for notify pipe */ + int notify_receive_fd; /* receiving end of notify pipe */ + int notify_send_fd; /* sending end of notify pipe */ + CQ new_conn_queue; /* queue of new connections to handle */ +} LIBEVENT_THREAD; + +static LIBEVENT_THREAD *threads; + +/* + * Number of threads that have finished setting themselves up. + */ +static int init_count = 0; +static pthread_mutex_t init_lock; +static pthread_cond_t init_cond; + + +static void thread_libevent_process(int fd, short which, void *arg); + +/* + * Initializes a connection queue. + */ +static void cq_init(CQ *cq) { + pthread_mutex_init(&cq->lock, NULL); + pthread_cond_init(&cq->cond, NULL); + cq->head = NULL; + cq->tail = NULL; +} + +/* + * Waits for work on a connection queue. + */ +static CQ_ITEM *cq_pop(CQ *cq) { + CQ_ITEM *item; + + pthread_mutex_lock(&cq->lock); + while (NULL == cq->head) + pthread_cond_wait(&cq->cond, &cq->lock); + item = cq->head; + cq->head = item->next; + if (NULL == cq->head) + cq->tail = NULL; + pthread_mutex_unlock(&cq->lock); + + return item; +} + +/* + * Looks for an item on a connection queue, but doesn't block if there isn't + * one. + * Returns the item, or NULL if no item is available + */ +static CQ_ITEM *cq_peek(CQ *cq) { + CQ_ITEM *item; + + pthread_mutex_lock(&cq->lock); + item = cq->head; + if (NULL != item) { + cq->head = item->next; + if (NULL == cq->head) + cq->tail = NULL; + } + pthread_mutex_unlock(&cq->lock); + + return item; +} + +/* + * Adds an item to a connection queue. + */ +static void cq_push(CQ *cq, CQ_ITEM *item) { + item->next = NULL; + + pthread_mutex_lock(&cq->lock); + if (NULL == cq->tail) + cq->head = item; + else + cq->tail->next = item; + cq->tail = item; + pthread_cond_signal(&cq->cond); + pthread_mutex_unlock(&cq->lock); +} + +/* + * Returns a fresh connection queue item. + */ +static CQ_ITEM *cqi_new() { + CQ_ITEM *item = NULL; + pthread_mutex_lock(&cqi_freelist_lock); + if (cqi_freelist) { + item = cqi_freelist; + cqi_freelist = item->next; + } + pthread_mutex_unlock(&cqi_freelist_lock); + + if (NULL == item) { + int i; + + /* Allocate a bunch of items at once to reduce fragmentation */ + item = malloc(sizeof(CQ_ITEM) * ITEMS_PER_ALLOC); + if (NULL == item) + return NULL; + + /* + * Link together all the new items except the first one + * (which we'll return to the caller) for placement on + * the freelist. + */ + for (i = 2; i < ITEMS_PER_ALLOC; i++) + item[i - 1].next = &item[i]; + + pthread_mutex_lock(&cqi_freelist_lock); + item[ITEMS_PER_ALLOC - 1].next = cqi_freelist; + cqi_freelist = &item[1]; + pthread_mutex_unlock(&cqi_freelist_lock); + } + + return item; +} + + +/* + * Frees a connection queue item (adds it to the freelist.) + */ +static void cqi_free(CQ_ITEM *item) { + pthread_mutex_lock(&cqi_freelist_lock); + item->next = cqi_freelist; + cqi_freelist = item; + pthread_mutex_unlock(&cqi_freelist_lock); +} + + +/* + * Creates a worker thread. + */ +static void create_worker(void *(*func)(void *), void *arg) { + pthread_t thread; + pthread_attr_t attr; + int ret; + + pthread_attr_init(&attr); + + if ((ret = pthread_create(&thread, &attr, func, arg)) != 0) { + fprintf(stderr, "Can't create thread: %s\n", + strerror(ret)); + exit(1); + } +} + + +/* + * Pulls a conn structure from the freelist, if one is available. + */ +conn *mt_conn_from_freelist() { + conn *c; + + pthread_mutex_lock(&conn_lock); + c = do_conn_from_freelist(); + pthread_mutex_unlock(&conn_lock); + + return c; +} + + +/* + * Adds a conn structure to the freelist. + * + * Returns 0 on success, 1 if the structure couldn't be added. + */ +bool mt_conn_add_to_freelist(conn *c) { + bool result; + + pthread_mutex_lock(&conn_lock); + result = do_conn_add_to_freelist(c); + pthread_mutex_unlock(&conn_lock); + + return result; +} + +/* + * Pulls a suffix buffer from the freelist, if one is available. + */ +char *mt_suffix_from_freelist() { + char *s; + + pthread_mutex_lock(&suffix_lock); + s = do_suffix_from_freelist(); + pthread_mutex_unlock(&suffix_lock); + + return s; +} + + +/* + * Adds a suffix buffer to the freelist. + * + * Returns 0 on success, 1 if the buffer couldn't be added. + */ +bool mt_suffix_add_to_freelist(char *s) { + bool result; + + pthread_mutex_lock(&suffix_lock); + result = do_suffix_add_to_freelist(s); + pthread_mutex_unlock(&suffix_lock); + + return result; +} + + +/****************************** LIBEVENT THREADS *****************************/ + +/* + * Set up a thread's information. + */ +static void setup_thread(LIBEVENT_THREAD *me) { + if (! me->base) { + me->base = event_init(); + if (! me->base) { + fprintf(stderr, "Can't allocate event base\n"); + exit(1); + } + } + + /* Listen for notifications from other threads */ + event_set(&me->notify_event, me->notify_receive_fd, + EV_READ | EV_PERSIST, thread_libevent_process, me); + event_base_set(me->base, &me->notify_event); + + if (event_add(&me->notify_event, 0) == -1) { + fprintf(stderr, "Can't monitor libevent notify pipe\n"); + exit(1); + } + + cq_init(&me->new_conn_queue); +} + + +/* + * Worker thread: main event loop + */ +static void *worker_libevent(void *arg) { + LIBEVENT_THREAD *me = arg; + + /* Any per-thread setup can happen here; thread_init() will block until + * all threads have finished initializing. + */ + + pthread_mutex_lock(&init_lock); + init_count++; + pthread_cond_signal(&init_cond); + pthread_mutex_unlock(&init_lock); + + return (void*) event_base_loop(me->base, 0); +} + + +/* + * Processes an incoming "handle a new connection" item. This is called when + * input arrives on the libevent wakeup pipe. + */ +static void thread_libevent_process(int fd, short which, void *arg) { + LIBEVENT_THREAD *me = arg; + CQ_ITEM *item; + char buf[1]; + + if (read(fd, buf, 1) != 1) + if (settings.verbose > 0) + fprintf(stderr, "Can't read from libevent pipe\n"); + + item = cq_peek(&me->new_conn_queue); + + if (NULL != item) { + conn *c = conn_new(item->sfd, item->init_state, item->event_flags, + item->read_buffer_size, item->is_udp, me->base); + if (c == NULL) { + if (item->is_udp) { + fprintf(stderr, "Can't listen for events on UDP socket\n"); + exit(1); + } else { + if (settings.verbose > 0) { + fprintf(stderr, "Can't listen for events on fd %d\n", + item->sfd); + } + close(item->sfd); + } + } + cqi_free(item); + } +} + +/* Which thread we assigned a connection to most recently. */ +static int last_thread = -1; + +/* + * Dispatches a new connection to another thread. This is only ever called + * from the main thread, either during initialization (for UDP) or because + * of an incoming connection. + */ +void dispatch_conn_new(int sfd, int init_state, int event_flags, + int read_buffer_size, int is_udp) { + CQ_ITEM *item = cqi_new(); + int thread = (last_thread + 1) % settings.num_threads; + + last_thread = thread; + + item->sfd = sfd; + item->init_state = init_state; + item->event_flags = event_flags; + item->read_buffer_size = read_buffer_size; + item->is_udp = is_udp; + + cq_push(&threads[thread].new_conn_queue, item); + + MEMCACHED_CONN_DISPATCH(sfd, threads[thread].thread_id); + if (write(threads[thread].notify_send_fd, "", 1) != 1) { + perror("Writing to thread notify pipe"); + } +} + +/* + * Returns true if this is the thread that listens for new TCP connections. + */ +int mt_is_listen_thread() { + return pthread_self() == threads[0].thread_id; +} + +/********************************* ITEM ACCESS *******************************/ + +/* + * Walks through the list of deletes that have been deferred because the items + * were locked down at the tmie. + */ +void mt_run_deferred_deletes() { + pthread_mutex_lock(&cache_lock); + do_run_deferred_deletes(); + pthread_mutex_unlock(&cache_lock); +} + +/* + * Allocates a new item. + */ +item *mt_item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes) { + item *it; + pthread_mutex_lock(&cache_lock); + it = do_item_alloc(key, nkey, flags, exptime, nbytes); + pthread_mutex_unlock(&cache_lock); + return it; +} + +/* + * Returns an item if it hasn't been marked as expired or deleted, + * lazy-expiring as needed. + */ +item *mt_item_get_notedeleted(const char *key, const size_t nkey, bool *delete_locked) { + item *it; + pthread_mutex_lock(&cache_lock); + it = do_item_get_notedeleted(key, nkey, delete_locked); + pthread_mutex_unlock(&cache_lock); + return it; +} + +/* + * Links an item into the LRU and hashtable. + */ +int mt_item_link(item *item) { + int ret; + + pthread_mutex_lock(&cache_lock); + ret = do_item_link(item); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/* + * Decrements the reference count on an item and adds it to the freelist if + * needed. + */ +void mt_item_remove(item *item) { + pthread_mutex_lock(&cache_lock); + do_item_remove(item); + pthread_mutex_unlock(&cache_lock); +} + +/* + * Replaces one item with another in the hashtable. + */ +int mt_item_replace(item *old, item *new) { + int ret; + + pthread_mutex_lock(&cache_lock); + ret = do_item_replace(old, new); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/* + * Unlinks an item from the LRU and hashtable. + */ +void mt_item_unlink(item *item) { + pthread_mutex_lock(&cache_lock); + do_item_unlink(item); + pthread_mutex_unlock(&cache_lock); +} + +/* + * Moves an item to the back of the LRU queue. + */ +void mt_item_update(item *item) { + pthread_mutex_lock(&cache_lock); + do_item_update(item); + pthread_mutex_unlock(&cache_lock); +} + +/* + * Adds an item to the deferred-delete list so it can be reaped later. + */ +char *mt_defer_delete(item *item, time_t exptime) { + char *ret; + + pthread_mutex_lock(&cache_lock); + ret = do_defer_delete(item, exptime); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/* + * Does arithmetic on a numeric item value. + */ +char *mt_add_delta(conn *c, item *item, int incr, const int64_t delta, + char *buf) { + char *ret; + + pthread_mutex_lock(&cache_lock); + ret = do_add_delta(c, item, incr, delta, buf); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/* + * Stores an item in the cache (high level, obeys set/add/replace semantics) + */ +int mt_store_item(item *item, int comm) { + int ret; + + pthread_mutex_lock(&cache_lock); + ret = do_store_item(item, comm); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/* + * Flushes expired items after a flush_all call + */ +void mt_item_flush_expired() { + pthread_mutex_lock(&cache_lock); + do_item_flush_expired(); + pthread_mutex_unlock(&cache_lock); +} + +/* + * Dumps part of the cache + */ +char *mt_item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int *bytes) { + char *ret; + + pthread_mutex_lock(&cache_lock); + ret = do_item_cachedump(slabs_clsid, limit, bytes); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/* + * Dumps statistics about slab classes + */ +char *mt_item_stats(int *bytes) { + char *ret; + + pthread_mutex_lock(&cache_lock); + ret = do_item_stats(bytes); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/* + * Dumps a list of objects of each size in 32-byte increments + */ +char *mt_item_stats_sizes(int *bytes) { + char *ret; + + pthread_mutex_lock(&cache_lock); + ret = do_item_stats_sizes(bytes); + pthread_mutex_unlock(&cache_lock); + return ret; +} + +/****************************** HASHTABLE MODULE *****************************/ + +void mt_assoc_move_next_bucket() { + pthread_mutex_lock(&cache_lock); + do_assoc_move_next_bucket(); + pthread_mutex_unlock(&cache_lock); +} + +/******************************* SLAB ALLOCATOR ******************************/ + +void *mt_slabs_alloc(size_t size, unsigned int id) { + void *ret; + + pthread_mutex_lock(&slabs_lock); + ret = do_slabs_alloc(size, id); + pthread_mutex_unlock(&slabs_lock); + return ret; +} + +void mt_slabs_free(void *ptr, size_t size, unsigned int id) { + pthread_mutex_lock(&slabs_lock); + do_slabs_free(ptr, size, id); + pthread_mutex_unlock(&slabs_lock); +} + +char *mt_slabs_stats(int *buflen) { + char *ret; + + pthread_mutex_lock(&slabs_lock); + ret = do_slabs_stats(buflen); + pthread_mutex_unlock(&slabs_lock); + return ret; +} + +#ifdef ALLOW_SLABS_REASSIGN +int mt_slabs_reassign(unsigned char srcid, unsigned char dstid) { + int ret; + + pthread_mutex_lock(&slabs_lock); + ret = do_slabs_reassign(srcid, dstid); + pthread_mutex_unlock(&slabs_lock); + return ret; +} +#endif + +/******************************* GLOBAL STATS ******************************/ + +void mt_stats_lock() { + pthread_mutex_lock(&stats_lock); +} + +void mt_stats_unlock() { + pthread_mutex_unlock(&stats_lock); +} + +/* + * Initializes the thread subsystem, creating various worker threads. + * + * nthreads Number of event handler threads to spawn + * main_base Event base for main thread + */ +void thread_init(int nthreads, struct event_base *main_base) { + int i; + + pthread_mutex_init(&cache_lock, NULL); + pthread_mutex_init(&conn_lock, NULL); + pthread_mutex_init(&slabs_lock, NULL); + pthread_mutex_init(&stats_lock, NULL); + + pthread_mutex_init(&init_lock, NULL); + pthread_cond_init(&init_cond, NULL); + + pthread_mutex_init(&cqi_freelist_lock, NULL); + cqi_freelist = NULL; + + threads = malloc(sizeof(LIBEVENT_THREAD) * nthreads); + if (! threads) { + perror("Can't allocate thread descriptors"); + exit(1); + } + + threads[0].base = main_base; + threads[0].thread_id = pthread_self(); + + for (i = 0; i < nthreads; i++) { + int fds[2]; + if (pipe(fds)) { + perror("Can't create notify pipe"); + exit(1); + } + + threads[i].notify_receive_fd = fds[0]; + threads[i].notify_send_fd = fds[1]; + + setup_thread(&threads[i]); + } + + /* Create threads after we've done all the libevent setup. */ + for (i = 1; i < nthreads; i++) { + create_worker(worker_libevent, &threads[i]); + } + + /* Wait for all the threads to set themselves up before returning. */ + pthread_mutex_lock(&init_lock); + init_count++; /* main thread */ + while (init_count < nthreads) { + pthread_cond_wait(&init_cond, &init_lock); + } + pthread_mutex_unlock(&init_lock); +} + +#endif diff --git a/contrib/reranking/data/README b/contrib/reranking/data/README new file mode 100644 index 000000000..59b20b32d --- /dev/null +++ b/contrib/reranking/data/README @@ -0,0 +1,5 @@ + +sample usage: + +../src/nbest -input-file nbest.small -output-file nbest.1best 1 -sort -weights weights + diff --git a/contrib/reranking/data/nbest.small b/contrib/reranking/data/nbest.small new file mode 100644 index 000000000..0fcbc44ce --- /dev/null +++ b/contrib/reranking/data/nbest.small @@ -0,0 +1,7 @@ +0 ||| Once a major milestone in the Balkans ||| d: 0 -0.608213 0 0 -0.512647 0 0 lm: -35.7187 tm: -3.97053 -17.5137 -3.24082 -15.8638 2.99969 w: -7 ||| -3.92049 +0 ||| Once a crucial period in the Balkans ||| d: 0 -0.944329 0 0 -1.06468 0 0 lm: -37.5341 tm: -4.27619 -19.441 -3.81074 -14.767 3.99959 w: -7 ||| -4.00353 +1 ||| Since the world is focused on Iraq , North Korea and a possible crisis with Iran on nuclear weapons , Kosovo is somewhat unnoticed . ||| d: -6 -5.80589 -0.65383 -1.29291 -6.19413 -0.0861354 -0.993748 lm: -112.868 tm: -42.7841 -61.6487 -16.5351 -23.8061 21.9977 w: -25 ||| -13.0796 +2 ||| The public will soon turn its attention back to that province during a decision regarding his fate . ||| d: -8 -4.61691 0 -3.62979 -4.85916 0 -4.43407 lm: -81.3478 tm: -46.0407 -63.79 -23.7663 -25.175 14.9984 w: -18 ||| -12.1226 +2 ||| The public will soon be able to turn its attention back into this province during a decision on his fate . ||| d: -8 -5.53064 0 -3.51999 -3.26708 0 -4.44003 lm: -84.7939 tm: -36.2621 -66.32 -21.0804 -33.9136 13.9985 w: -21 ||| -12.1227 +2 ||| The public will soon turn his attention to them at a decision on his destiny . ||| d: -8 -5.3448 0 -2.65118 -4.35949 0 -3.95447 lm: -67.451 tm: -54.851 -89.0503 -17.9389 -22.9488 12.9986 w: -16 ||| -12.1234 +2 ||| The public will soon turn his attention to them at a decision on his destiny . ||| d: -8 -5.3448 0 -2.65118 -4.35949 0 -3.95447 lm: -67.451 tm: -54.851 -89.0503 -17.9389 -22.9488 12.9986 w: -16 ||| -12.1234 diff --git a/contrib/reranking/data/weights b/contrib/reranking/data/weights new file mode 100644 index 000000000..c6b6c1ac0 --- /dev/null +++ b/contrib/reranking/data/weights @@ -0,0 +1,11 @@ +0 +1 2 3 +4 +5 +6 +7 +8 +9 +10 +11 +12 13 diff --git a/contrib/reranking/src/Hypo.cpp b/contrib/reranking/src/Hypo.cpp new file mode 100644 index 000000000..0ceb21abd --- /dev/null +++ b/contrib/reranking/src/Hypo.cpp @@ -0,0 +1,59 @@ +/* + * nbest: tool to process moses n-best lists + * + * File: Hypo.cpp + * basic functions to process one hypothesis + * + * Created by Holger Schwenk, University of Le Mans, 05/16/2008 + * + */ + + +#include "Hypo.h" +#include <iostream> + +//const char* NBEST_DELIM = "|||"; + +Hypo::Hypo() +{ + //cerr << "Hypo: constructor called" << endl; +} + +Hypo::~Hypo() +{ + //cerr << "Hypo: destructor called" << endl; +} + +void Hypo::Write(ofstream &outf) +{ + outf << id << NBEST_DELIM2 << trg << NBEST_DELIM2; + for (vector<float>::iterator i = f.begin(); i != f.end(); i++) + outf << (*i) << " "; + outf << NBEST_DELIM << " " << s << endl; + +} + +float Hypo::CalcGlobal(Weights &w) +{ + //cerr << " HYP: calc global" << endl; + int sz=w.val.size(); + if (sz<f.size()) { + cerr << " - NOTE: padding weight vector with " << f.size()-sz << " zeros" << endl; + w.val.resize(f.size()); + } + + s=0; + for (int i=0; i<f.size(); i++) { + //cerr << "i=" << i << ", " << w.val[i] << ", " << f[i] << endl; + s+=w.val[i]*f[i]; + } + //cerr << "s=" << s << endl; + return s; +} + +// this is actually a "greater than" since we want to sort in descending order +bool Hypo::operator< (const Hypo &h2) const +{ + return (this->s > h2.s); +} + diff --git a/contrib/reranking/src/Hypo.h b/contrib/reranking/src/Hypo.h new file mode 100644 index 000000000..a85410289 --- /dev/null +++ b/contrib/reranking/src/Hypo.h @@ -0,0 +1,44 @@ +/* + * nbest: tool to process moses n-best lists + * + * File: Hypo.h + * basic functions to process one hypothesis + * + * Created by Holger Schwenk, University of Le Mans, 05/16/2008 + * + */ + + +#ifndef _HYPO_H_ +#define _HYPO_H_ + +using namespace std; + +#include <iostream> +#include <fstream> +#include <string> +#include <vector> + +#include "Tools.h" + +#define NBEST_DELIM "|||" +#define NBEST_DELIM2 " ||| " + +class Hypo +{ + int id; + string trg; // translation + vector<float> f; // feature function scores + float s; // global score + // segmentation +public: + Hypo(); + Hypo(int p_id,string &p_trg, vector<float> &p_f, float p_s) : id(p_id),trg(p_trg),f(p_f),s(p_s) {}; + ~Hypo(); + float CalcGlobal(Weights&); + void Write(ofstream&); + bool operator< (const Hypo&) const; + // bool CompareLikelihoods (const Hypo&, const Hypo&) const; +}; + +#endif diff --git a/contrib/reranking/src/Main.cpp b/contrib/reranking/src/Main.cpp new file mode 100644 index 000000000..a91be8e73 --- /dev/null +++ b/contrib/reranking/src/Main.cpp @@ -0,0 +1,98 @@ +/* + * nbest: tool to process moses n-best lists + * + * File: Main.cpp + * command line interface + * + * Created by Holger Schwenk, University of Le Mans, 05/16/2008 + * + */ + +#include <iostream> +#include <fstream> +#include "ParameterNBest.h" +#include "NBest.h" +#include "Tools.h" + +#include "Util.h" // from Moses + + +using namespace std; + +int main (int argc, char *argv[]) +{ + // parse parameters + ParameterNBest *parameter = new ParameterNBest(); + if (!parameter->LoadParam(argc, argv)) { + parameter->Explain(); + delete parameter; + return 1; + } + + // read input + ifstream inpf; + PARAM_VEC p=parameter->GetParam("input-file"); + if (p.size()<1 || p.size()>2) Error("The option -input-file requires one or two arguments"); + int in_n=p.size()>1 ? Scan<int>(p[1]) : 0; + cout << "NBest version 0.1, written by Holger.Schwenk@lium.univ-lemans.fr" << endl + << " - reading input from file '" << p[0] << "'"; + if (in_n>0) cout << " (limited to the first " << in_n << " hypothesis)"; + cout << endl; + inpf.open(p[0].c_str()); + if (inpf.fail()) { + perror ("ERROR"); + exit(1); + } + + // open output + ofstream outf; + p=parameter->GetParam("output-file"); + if (p.size()<1 || p.size()>2) Error("The option -output-file requires one or two arguments"); + int out_n=p.size()>1 ? Scan<int>(p[1]) : 0; + cout << " - writing output to file '" << p[0] << "'"; + if (out_n>0) cout << " (limited to the first " << out_n << " hypothesis)"; + cout << endl; + outf.open(p[0].c_str()); + if (outf.fail()) { + perror ("ERROR"); + exit(1); + } + + // eventually read weights + Weights w; + int do_calc=false; + if (parameter->isParamSpecified("weights")) { + p=parameter->GetParam("weights"); + if (p.size()<1) Error("The option -weights requires one argument"); + cout << " - reading weights from file '" << p[0] << "'"; + int n=w.Read(p[0].c_str()); + cout << " (found " << n << " values)" << endl; + do_calc=true; + cout << " - recalculating global scores" << endl; + } + + // shall we sort ? + bool do_sort = parameter->isParamSpecified("sort"); + if (do_sort) cout << " - sorting global scores" << endl; + + // main loop + int nb_sent=0, nb_nbest=0; + while (!inpf.eof()) { + NBest nbest(inpf, in_n); + + if (do_calc) nbest.CalcGlobal(w); + if (do_sort) nbest.Sort(); + nbest.Write(outf, out_n); + + nb_sent++; + nb_nbest+=nbest.NbNBest(); + } + inpf.close(); + outf.close(); + + // display final statistics + cout << " - processed " << nb_nbest << " n-best hypotheses in " << nb_sent << " sentences" + << " (average " << (float) nb_nbest/nb_sent << ")" << endl; + + return 0; +} diff --git a/contrib/reranking/src/Makefile b/contrib/reranking/src/Makefile new file mode 100644 index 000000000..7b0ec6945 --- /dev/null +++ b/contrib/reranking/src/Makefile @@ -0,0 +1,18 @@ + +# where to find include files and libraries from Moses +MOSES_INC=../../moses/src +LIB_DIR=../../moses/src/ + +LIBS=-lmoses -lz +OBJS=Main.o NBest.o Hypo.o Tools.o ParameterNBest.o + +CFLAGS=-I$(MOSES_INC) + +nbest-tool: $(OBJS) + $(CXX) -o nbest $(OBJS) -L$(LIB_DIR) $(LIBS) + +%.o: %.cpp + $(CXX) $(CFLAGS) -o $@ -c $< + +clean: + -rm $(OBJS) nbest diff --git a/contrib/reranking/src/NBest.cpp b/contrib/reranking/src/NBest.cpp new file mode 100644 index 000000000..24a0f60c3 --- /dev/null +++ b/contrib/reranking/src/NBest.cpp @@ -0,0 +1,131 @@ +/* + * nbest: tool to process moses n-best lists + * + * File: NBest.cpp + * basic functions on n-best lists + * + * Created by Holger Schwenk, University of Le Mans, 05/16/2008 + * + */ + + +#include "NBest.h" + +#include "Util.h" // from Moses + +#include <sstream> +#include <algorithm> + +//NBest::NBest() { +//cerr << "NBEST: constructor called" << endl; +//} + + +bool NBest::ParseLine(ifstream &inpf, const int n) +{ + static string line; // used internally to buffer an input line + static int prev_id=-1; // used to detect a change of the n-best ID + int id; + vector<float> f; + float s; + int pos=0, epos; + vector<string> blocks; + + + if (line.empty()) { + getline(inpf,line); + if (inpf.eof()) return false; + } + + // split line into blocks + //cerr << "PARSE line: " << line << endl; + while ((epos=line.find(NBEST_DELIM,pos))!=string::npos) { + blocks.push_back(line.substr(pos,epos-pos)); + // cerr << " block: " << blocks.back() << endl; + pos=epos+strlen(NBEST_DELIM); + } + blocks.push_back(line.substr(pos,line.size())); + // cerr << " block: " << blocks.back() << endl; + + if (blocks.size()<4) { + cerr << line << endl; + Error("can't parse the above line"); + } + + // parse ID + id=Scan<int>(blocks[0]); + if (prev_id>=0 && id!=prev_id) { + prev_id=id; // new nbest list has started + return false; + } + prev_id=id; + //cerr << "same ID " << id << endl; + + if (n>0 && nbest.size() >= n) { + //cerr << "skipped" << endl; + line.clear(); + return true; // skip parsing of unused hypos + } + + // parse feature function scores + //cerr << "PARSE features: '" << blocks[2] << "' size: " << blocks[2].size() << endl; + pos=blocks[2].find_first_not_of(' '); + while (pos<blocks[2].size() && (epos=blocks[2].find(" ",pos))!=string::npos) { + string feat=blocks[2].substr(pos,epos-pos); + //cerr << " feat: '" << feat << "', pos: " << pos << ", " << epos << endl; + if (feat.find(":",0)!=string::npos) { + //cerr << " name: " << feat << endl; + } else { + f.push_back(Scan<float>(feat)); + //cerr << " value: " << f.back() << endl; + } + pos=epos+1; + } + + // eventually parse segmentation + if (blocks.size()>4) { + Error("parsing segmentation not yet supported"); + } + + nbest.push_back(Hypo(id, blocks[1], f, Scan<float>(blocks[3]))); + + line.clear(); // force read of new line + + return true; +} + + +NBest::NBest(ifstream &inpf, const int n) +{ + //cerr << "NBEST: constructor with file called" << endl; + while (ParseLine(inpf,n)); + //cerr << "NBEST: found " << nbest.size() << " lines" << endl; +} + + +NBest::~NBest() +{ + //cerr << "NBEST: destructor called" << endl; +} + +void NBest::Write(ofstream &outf, int n) +{ + if (n<1 || n>nbest.size()) n=nbest.size(); + for (int i=0; i<n; i++) nbest[i].Write(outf); +} + + +float NBest::CalcGlobal(Weights &w) +{ + //cerr << "NBEST: calc global of size " << nbest.size() << endl; + for (vector<Hypo>::iterator i = nbest.begin(); i != nbest.end(); i++) { + (*i).CalcGlobal(w); + } +} + + +void NBest::Sort() +{ + sort(nbest.begin(),nbest.end()); +} + diff --git a/contrib/reranking/src/NBest.h b/contrib/reranking/src/NBest.h new file mode 100644 index 000000000..9a4aa9447 --- /dev/null +++ b/contrib/reranking/src/NBest.h @@ -0,0 +1,44 @@ +/* + * nbest: tool to process moses n-best lists + * + * File: NBest.h + * basic functions on n-best lists + * + * Created by Holger Schwenk, University of Le Mans, 05/16/2008 + * + */ + + +#ifndef _NBEST_H_ +#define _NBEST_H_ + +using namespace std; + +#include <iostream> +#include <fstream> +#include <string> +#include <vector> + +#include "Tools.h" +#include "Hypo.h" + +class NBest +{ + int id; + string src; + vector<Hypo> nbest; + bool ParseLine(ifstream &inpf, const int n); +public: + NBest(ifstream&, const int=0); + ~NBest(); + int NbNBest() { + return nbest.size(); + }; + float CalcGlobal(Weights&); + void Sort(); // largest values first + void Write(ofstream&, int=0); +}; + +void Error(char *msg); + +#endif diff --git a/contrib/reranking/src/ParameterNBest.cpp b/contrib/reranking/src/ParameterNBest.cpp new file mode 100644 index 000000000..005f3890c --- /dev/null +++ b/contrib/reranking/src/ParameterNBest.cpp @@ -0,0 +1,337 @@ +// $Id: $ + +/*********************************************************************** +nbest - tool to process Moses n-best list +Copyright (C) 2008 Holger Schwenk, University of Le Mans, France + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <iostream> +#include <iterator> +#include <fstream> +#include <sstream> +#include <algorithm> +#include "ParameterNBest.h" +#include "Tools.h" + +#include "Util.h" // from Moses +#include "InputFileStream.h" +#include "UserMessage.h" + +using namespace std; + +/** define allowed parameters */ +ParameterNBest::ParameterNBest() +{ + AddParam("input-file", "i", "file name of the input n-best list"); + AddParam("output-file", "o", "file name of the output n-best list"); + AddParam("recalc", "r", "recalc global scores"); + AddParam("weights", "w", "coefficients of the feature functions"); + AddParam("sort", "s", "sort n-best list according to the global scores"); + AddParam("lexical", "l", "report number of lexically different hypothesis"); +} + +ParameterNBest::~ParameterNBest() +{ +} + +/** initialize a parameter, sub of constructor */ +void ParameterNBest::AddParam(const string ¶mName, const string &description) +{ + m_valid[paramName] = true; + m_description[paramName] = description; +} + +/** initialize a parameter (including abbreviation), sub of constructor */ +void ParameterNBest::AddParam(const string ¶mName, const string &abbrevName, const string &description) +{ + m_valid[paramName] = true; + m_valid[abbrevName] = true; + m_abbreviation[paramName] = abbrevName; + m_description[paramName] = description; +} + +/** print descriptions of all parameters */ +void ParameterNBest::Explain() +{ + cerr << "Usage:" << endl; + for(PARAM_STRING::const_iterator iterParam = m_description.begin(); iterParam != m_description.end(); iterParam++) { + const string paramName = iterParam->first; + const string paramDescription = iterParam->second; + cerr << "\t-" << paramName; + PARAM_STRING::const_iterator iterAbbr = m_abbreviation.find( paramName ); + if ( iterAbbr != m_abbreviation.end() ) + cerr << " (" << iterAbbr->second << ")"; + cerr << ": " << paramDescription << endl; + } +} + +/** check whether an item on the command line is a switch or a value + * \param token token on the command line to checked **/ + +bool ParameterNBest::isOption(const char* token) +{ + if (! token) return false; + std::string tokenString(token); + size_t length = tokenString.size(); + if (length > 0 && tokenString.substr(0,1) != "-") return false; + if (length > 1 && tokenString.substr(1,1).find_first_not_of("0123456789") == 0) return true; + return false; +} + +/** load all parameters from the configuration file and the command line switches */ +bool ParameterNBest::LoadParam(const string &filePath) +{ + const char *argv[] = {"executable", "-f", filePath.c_str() }; + return LoadParam(3, (char**) argv); +} + +/** load all parameters from the configuration file and the command line switches */ +bool ParameterNBest::LoadParam(int argc, char* argv[]) +{ + // config file (-f) arg mandatory + string configPath; + /* + if ( (configPath = FindParam("-f", argc, argv)) == "" + && (configPath = FindParam("-config", argc, argv)) == "") + { + PrintCredit(); + + UserMessage::Add("No configuration file was specified. Use -config or -f"); + return false; + } + else + { + if (!ReadConfigFile(configPath)) + { + UserMessage::Add("Could not read "+configPath); + return false; + } + } + */ + + // overwrite parameters with values from switches + for(PARAM_STRING::const_iterator iterParam = m_description.begin(); iterParam != m_description.end(); iterParam++) { + const string paramName = iterParam->first; + OverwriteParam("-" + paramName, paramName, argc, argv); + } + + // ... also shortcuts + for(PARAM_STRING::const_iterator iterParam = m_abbreviation.begin(); iterParam != m_abbreviation.end(); iterParam++) { + const string paramName = iterParam->first; + const string paramShortName = iterParam->second; + OverwriteParam("-" + paramShortName, paramName, argc, argv); + } + + // logging of parameters that were set in either config or switch + int verbose = 1; + if (m_setting.find("verbose") != m_setting.end() && + m_setting["verbose"].size() > 0) + verbose = Scan<int>(m_setting["verbose"][0]); + if (verbose >= 1) { // only if verbose + TRACE_ERR( "Defined parameters (per moses.ini or switch):" << endl); + for(PARAM_MAP::const_iterator iterParam = m_setting.begin() ; iterParam != m_setting.end(); iterParam++) { + TRACE_ERR( "\t" << iterParam->first << ": "); + for ( size_t i = 0; i < iterParam->second.size(); i++ ) + TRACE_ERR( iterParam->second[i] << " "); + TRACE_ERR( endl); + } + } + + // check for illegal parameters + bool noErrorFlag = true; + for (int i = 0 ; i < argc ; i++) { + if (isOption(argv[i])) { + string paramSwitch = (string) argv[i]; + string paramName = paramSwitch.substr(1); + if (m_valid.find(paramName) == m_valid.end()) { + UserMessage::Add("illegal switch: " + paramSwitch); + noErrorFlag = false; + } + } + } + + // check if parameters make sense + return Validate() && noErrorFlag; +} + +/** check that parameter settings make sense */ +bool ParameterNBest::Validate() +{ + bool noErrorFlag = true; + + // required parameters + if (m_setting["input-file"].size() == 0) { + UserMessage::Add("No input-file"); + noErrorFlag = false; + } + + if (m_setting["output-file"].size() == 0) { + UserMessage::Add("No output-file"); + noErrorFlag = false; + } + + if (m_setting["recalc"].size() > 0 && m_setting["weights"].size()==0) { + UserMessage::Add("you need to spezify weight when recalculating global scores"); + noErrorFlag = false; + } + + + return noErrorFlag; +} + +/** check whether a file exists */ +bool ParameterNBest::FilesExist(const string ¶mName, size_t tokenizeIndex,std::vector<std::string> const& extensions) +{ + typedef std::vector<std::string> StringVec; + StringVec::const_iterator iter; + + PARAM_MAP::const_iterator iterParam = m_setting.find(paramName); + if (iterParam == m_setting.end()) { + // no param. therefore nothing to check + return true; + } + const StringVec &pathVec = (*iterParam).second; + for (iter = pathVec.begin() ; iter != pathVec.end() ; ++iter) { + StringVec vec = Tokenize(*iter); + if (tokenizeIndex >= vec.size()) { + stringstream errorMsg(""); + errorMsg << "Expected at least " << (tokenizeIndex+1) << " tokens per emtry in '" + << paramName << "', but only found " + << vec.size(); + UserMessage::Add(errorMsg.str()); + return false; + } + const string &pathStr = vec[tokenizeIndex]; + + bool fileFound=0; + for(size_t i=0; i<extensions.size() && !fileFound; ++i) { + fileFound|=FileExists(pathStr + extensions[i]); + } + if(!fileFound) { + stringstream errorMsg(""); + errorMsg << "File " << pathStr << " does not exist"; + UserMessage::Add(errorMsg.str()); + return false; + } + } + return true; +} + +/** look for a switch in arg, update parameter */ +// TODO arg parsing like this does not belong in the library, it belongs +// in moses-cmd +string ParameterNBest::FindParam(const string ¶mSwitch, int argc, char* argv[]) +{ + for (int i = 0 ; i < argc ; i++) { + if (string(argv[i]) == paramSwitch) { + if (i+1 < argc) { + return argv[i+1]; + } else { + stringstream errorMsg(""); + errorMsg << "Option " << paramSwitch << " requires a parameter!"; + UserMessage::Add(errorMsg.str()); + // TODO return some sort of error, not the empty string + } + } + } + return ""; +} + +/** update parameter settings with command line switches + * \param paramSwitch (potentially short) name of switch + * \param paramName full name of parameter + * \param argc number of arguments on command line + * \param argv values of paramters on command line */ +void ParameterNBest::OverwriteParam(const string ¶mSwitch, const string ¶mName, int argc, char* argv[]) +{ + int startPos = -1; + for (int i = 0 ; i < argc ; i++) { + if (string(argv[i]) == paramSwitch) { + startPos = i+1; + break; + } + } + if (startPos < 0) + return; + + int index = 0; + m_setting[paramName]; // defines the parameter, important for boolean switches + while (startPos < argc && (!isOption(argv[startPos]))) { + if (m_setting[paramName].size() > (size_t)index) + m_setting[paramName][index] = argv[startPos]; + else + m_setting[paramName].push_back(argv[startPos]); + index++; + startPos++; + } +} + + +/** read parameters from a configuration file */ +bool ParameterNBest::ReadConfigFile( string filePath ) +{ + InputFileStream inFile(filePath); + string line, paramName; + while(getline(inFile, line)) { + // comments + size_t comPos = line.find_first_of("#"); + if (comPos != string::npos) + line = line.substr(0, comPos); + // trim leading and trailing spaces/tabs + line = Trim(line); + + if (line[0]=='[') { + // new parameter + for (size_t currPos = 0 ; currPos < line.size() ; currPos++) { + if (line[currPos] == ']') { + paramName = line.substr(1, currPos - 1); + break; + } + } + } else if (line != "") { + // add value to parameter + m_setting[paramName].push_back(line); + } + } + return true; +} + + +void ParameterNBest::PrintCredit() +{ + cerr << "NBest - A tool to process Moses n-best lists" << endl + << "Copyright (C) 2008 Holger Schwenk" << endl << endl + + << "This library is free software; you can redistribute it and/or" << endl + << "modify it under the terms of the GNU Lesser General Public" << endl + << "License as published by the Free Software Foundation; either" << endl + << "version 2.1 of the License, or (at your option) any later version." << endl << endl + + << "This library is distributed in the hope that it will be useful," << endl + << "but WITHOUT ANY WARRANTY; without even the implied warranty of" << endl + << "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU" << endl + << "Lesser General Public License for more details." << endl << endl + + << "You should have received a copy of the GNU Lesser General Public" << endl + << "License along with this library; if not, write to the Free Software" << endl + << "Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA" << endl << endl + << "***********************************************************************" << endl << endl + << "Built on " << __DATE__ << endl << endl + + << "Written by Holger Schwenk, Holger.Schwenk@lium.univ-lemans.fr" << endl << endl; +} + diff --git a/contrib/reranking/src/ParameterNBest.h b/contrib/reranking/src/ParameterNBest.h new file mode 100644 index 000000000..bc554d4b9 --- /dev/null +++ b/contrib/reranking/src/ParameterNBest.h @@ -0,0 +1,76 @@ +// $Id: $ + +/*********************************************************************** +nbest - tool to process Moses n-best list +Copyright (C) 2008 Holger Schwenk, University of Le Mans, France + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#ifndef _PARAMETER_NBEST_H_ +#define _PARAMETER_NBEST_H_ + +#include <string> +#include <map> +#include <vector> +#include "TypeDef.h" + +typedef std::vector<std::string> PARAM_VEC; +typedef std::map<std::string, PARAM_VEC > PARAM_MAP; +typedef std::map<std::string, bool> PARAM_BOOL; +typedef std::map<std::string, std::string > PARAM_STRING; + +/** Handles parameter values set in config file or on command line. + * Process raw parameter data (names and values as strings) for StaticData + * to parse; to get useful values, see StaticData. */ +class ParameterNBest +{ +protected: + PARAM_MAP m_setting; + PARAM_BOOL m_valid; + PARAM_STRING m_abbreviation; + PARAM_STRING m_description; + + std::string FindParam(const std::string ¶mSwitch, int argc, char* argv[]); + void OverwriteParam(const std::string ¶mSwitch, const std::string ¶mName, int argc, char* argv[]); + bool ReadConfigFile( std::string filePath ); + bool FilesExist(const std::string ¶mName, size_t tokenizeIndex,std::vector<std::string> const& fileExtension=std::vector<std::string>(1,"")); + bool isOption(const char* token); + bool Validate(); + + void AddParam(const std::string ¶mName, const std::string &description); + void AddParam(const std::string ¶mName, const std::string &abbrevName, const std::string &description); + + void PrintCredit(); + +public: + ParameterNBest(); + ~ParameterNBest(); + bool LoadParam(int argc, char* argv[]); + bool LoadParam(const std::string &filePath); + void Explain(); + + /** return a vector of strings holding the whitespace-delimited values on the ini-file line corresponding to the given parameter name */ + const PARAM_VEC &GetParam(const std::string ¶mName) { + return m_setting[paramName]; + } + /** check if parameter is defined (either in moses.ini or as switch) */ + bool isParamSpecified(const std::string ¶mName) { + return m_setting.find( paramName ) != m_setting.end(); + } + +}; + +#endif diff --git a/contrib/reranking/src/Tools.cpp b/contrib/reranking/src/Tools.cpp new file mode 100644 index 000000000..8312c3370 --- /dev/null +++ b/contrib/reranking/src/Tools.cpp @@ -0,0 +1,29 @@ +/* + * nbest: tool to process moses n-best lists + * + * File: Tools.cpp + * basic utility functions + * + * Created by Holger Schwenk, University of Le Mans, 05/16/2008 + * + */ + +#include "Tools.h" + +int Weights::Read(const char *fname) +{ + ifstream inpf; + + inpf.open(fname); + if (inpf.fail()) { + perror ("ERROR"); + exit(1); + } + + float f; + while (inpf >> f) val.push_back(f); + + inpf.close(); + return val.size(); +} + diff --git a/contrib/reranking/src/Tools.h b/contrib/reranking/src/Tools.h new file mode 100644 index 000000000..eb71746b0 --- /dev/null +++ b/contrib/reranking/src/Tools.h @@ -0,0 +1,73 @@ +/* + * nbest: tool to process moses n-best lists + * + * File: Tools.cpp + * basic utility functions + * + * Created by Holger Schwenk, University of Le Mans, 05/16/2008 + * + */ + + +#ifndef _TOOLS_H_ +#define _TOOLS_H_ + +using namespace std; + +#include <iostream> +#include <fstream> +#include <vector> + +class Weights +{ + vector<float> val; +public: + Weights() {}; + ~Weights() {}; + int Read(const char *); + friend class Hypo; +}; + +//****************************************************** + +/* +template<typename T> +inline T Scan(const std::string &input) +{ + std::stringstream stream(input); + T ret; + stream >> ret; + return ret; +} +*/ + +//****************************************************** + +inline void Error (char *msg) +{ + cerr << "ERROR: " << msg << endl; + exit(1); +} + +//****************************************************** +// From Moses code: + + +/* + * Outputting debugging/verbose information to stderr. + * Use TRACE_ENABLE flag to redirect tracing output into oblivion + * so that you can output your own ad-hoc debugging info. + * However, if you use stderr diretly, please delete calls to it once + * you finished debugging so that it won't clutter up. + * Also use TRACE_ENABLE to turn off output of any debugging info + * when compiling for a gui front-end so that running gui won't generate + * output on command line + * */ +#ifdef TRACE_ENABLE +#define TRACE_ERR(str) std::cerr << str +#else +#define TRACE_ERR(str) {} +#endif + +#endif + diff --git a/contrib/server/Jamfile b/contrib/server/Jamfile new file mode 100644 index 000000000..eecec2b98 --- /dev/null +++ b/contrib/server/Jamfile @@ -0,0 +1,29 @@ +#If you get compilation errors here, make sure you have xmlrpc-c installed properly. . . + +import option ; + +if [ option.get "with-xmlrpc-c" ] +{ + with-xmlrpc-c = [ option.get "with-xmlrpc-c" ] ; + xmlrpc-lib = <search>$(with-xmlrpc-c)/lib ; + xmlrpc-include = <include>$(with-xmlrpc-c)/include ; + build-moses-server = "yes" ; +} else { + if [ SHELL $(TOP)"/jam-files/test.sh -include xmlrpc-c/base.hpp -E" ] = 0 { + build-moses-server = "yes" ; + } +} + +if $(build-moses-server) = "yes" +{ + xmlrpc-deps = ; + #This is the libraries in reverse order + for xmllib in xmlrpc_xmltok xmlrpc_xmlparse xmlrpc_util xmlrpc xmlrpc_abyss xmlrpc_server xmlrpc_server_abyss xmlrpc++ xmlrpc_server++ xmlrpc_server_abyss++ { + lib $(xmllib) : : <use>$(xmlrpc-deps) $(xmlrpc-lib) : : $(xmlrpc-include) ; + xmlrpc-deps = $(xmlrpc-deps) $(xmllib) ; + } + + exe mosesserver : mosesserver.cpp ../moses/src//moses ../OnDiskPt/src//OnDiskPt $(xmlrpc-deps) ; +} else { + alias mosesserver ; +} diff --git a/contrib/server/Makefile.am b/contrib/server/Makefile.am new file mode 100644 index 000000000..925a8c40f --- /dev/null +++ b/contrib/server/Makefile.am @@ -0,0 +1,5 @@ +bin_PROGRAMS = mosesserver +mosesserver_SOURCES = mosesserver.cpp +mosesserver_CPPFLAGS = -W -Wall -I$(top_srcdir)/moses/src $(XMLRPC_C_CPPFLAGS) $(BOOST_CPPFLAGS) +mosesserver_LDADD = -L$(top_srcdir)/moses/src -lmoses -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt $(top_srcdir)/util/libkenutil.la $(top_srcdir)/lm/libkenlm.la $(BOOST_THREAD_LDFLAGS) $(XMLRPC_C_LIBS) $(BOOST_THREAD_LIBS) +mosesserver_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a diff --git a/contrib/server/client.perl b/contrib/server/client.perl new file mode 100755 index 000000000..eca5e388f --- /dev/null +++ b/contrib/server/client.perl @@ -0,0 +1,30 @@ +#!/usr/bin/env perl + +# +# Sample client for mosesserver, illustrating allignment info and +# report all factors +# + +use Encode; +use XMLRPC::Lite; +use utf8; + +$url = "http://localhost:8080/RPC2"; +$proxy = XMLRPC::Lite->proxy($url); + +$text = "il a souhaité que la présidence trace à nice le chemin pour l' avenir ."; + +# Work-around for XMLRPC::Lite bug +$encoded = SOAP::Data->type(string => Encode::encode("utf8",$text)); + +my %param = ("text" => $encoded, "align" => "true", "report-all-factors" => "true"); +$result = $proxy->call("translate",\%param)->result; +print $result->{'text'} . "\n"; +if ($result->{'align'}) { + print "Phrase alignments: \n"; + $aligns = $result->{'align'}; + foreach my $align (@$aligns) { + print $align->{'tgt-start'} . "," . $align->{'src-start'} . "," + . $align->{'src-end'} . "\n"; + } +} diff --git a/contrib/server/mosesserver.cpp b/contrib/server/mosesserver.cpp new file mode 100644 index 000000000..c3459a59e --- /dev/null +++ b/contrib/server/mosesserver.cpp @@ -0,0 +1,381 @@ +#include "util/check.hh" +#include <stdexcept> +#include <iostream> + +#include <xmlrpc-c/base.hpp> +#include <xmlrpc-c/registry.hpp> +#include <xmlrpc-c/server_abyss.hpp> + +#include "Hypothesis.h" +#include "Manager.h" +#include "StaticData.h" +#include "PhraseDictionaryDynSuffixArray.h" +#include "TranslationSystem.h" +#include "LMList.h" +#ifdef LM_ORLM +# include "LanguageModelORLM.h" +#endif + +using namespace Moses; +using namespace std; + +typedef std::map<std::string, xmlrpc_c::value> params_t; + +/** Find out which translation system to use */ +const TranslationSystem& getTranslationSystem(params_t params) +{ + string system_id = TranslationSystem::DEFAULT; + params_t::const_iterator pi = params.find("system"); + if (pi != params.end()) { + system_id = xmlrpc_c::value_string(pi->second); + } + VERBOSE(1, "Using translation system " << system_id << endl;) + return StaticData::Instance().GetTranslationSystem(system_id); +} + +class Updater: public xmlrpc_c::method +{ +public: + Updater() { + // signature and help strings are documentation -- the client + // can query this information with a system.methodSignature and + // system.methodHelp RPC. + this->_signature = "S:S"; + this->_help = "Updates stuff"; + } + void + execute(xmlrpc_c::paramList const& paramList, + xmlrpc_c::value * const retvalP) { + const params_t params = paramList.getStruct(0); + breakOutParams(params); + const TranslationSystem& system = getTranslationSystem(params); + const PhraseDictionaryFeature* pdf = system.GetPhraseDictionaries()[0]; + PhraseDictionaryDynSuffixArray* pdsa = (PhraseDictionaryDynSuffixArray*) pdf->GetDictionary(); + cerr << "Inserting into address " << pdsa << endl; + pdsa->insertSnt(source_, target_, alignment_); + if(add2ORLM_) { + updateORLM(); + } + cerr << "Done inserting\n"; + //PhraseDictionary* pdsa = (PhraseDictionary*) pdf->GetDictionary(*dummy); + map<string, xmlrpc_c::value> retData; + //*retvalP = xmlrpc_c::value_struct(retData); + pdf = 0; + pdsa = 0; + *retvalP = xmlrpc_c::value_string("Phrase table updated"); + } + string source_, target_, alignment_; + bool bounded_, add2ORLM_; + void updateORLM() { +#ifdef LM_ORLM + vector<string> vl; + map<vector<string>, int> ngSet; + LMList lms = StaticData::Instance().GetLMList(); // get LM + LMList::const_iterator lmIter = lms.begin(); + const LanguageModel* lm = *lmIter; + /* currently assumes a single LM that is a ORLM */ +#ifdef WITH_THREADS + boost::shared_ptr<LanguageModelORLM> orlm; + orlm = boost::dynamic_pointer_cast<LanguageModelORLM>(lm->GetLMImplementation()); +#else + LanguageModelORLM* orlm; + orlm = (LanguageModelORLM*)lm->GetLMImplementation(); +#endif + if(orlm == 0) { + cerr << "WARNING: Unable to add target sentence to ORLM\n"; + return; + } + // break out new ngrams from sentence + const int ngOrder(orlm->GetNGramOrder()); + const std::string sBOS = orlm->GetSentenceStart()->GetString(); + const std::string sEOS = orlm->GetSentenceEnd()->GetString(); + Utils::splitToStr(target_, vl, " "); + // insert BOS and EOS + vl.insert(vl.begin(), sBOS); + vl.insert(vl.end(), sEOS); + for(int j=0; j < vl.size(); ++j) { + int i = (j<ngOrder) ? 0 : j-ngOrder+1; + for(int t=j; t >= i; --t) { + vector<string> ngVec; + for(int s=t; s<=j; ++s) { + ngVec.push_back(vl[s]); + //cerr << vl[s] << " "; + } + ngSet[ngVec]++; + //cerr << endl; + } + } + // insert into LM in order from 1grams up (for LM well-formedness) + cerr << "Inserting " << ngSet.size() << " ngrams into ORLM...\n"; + for(int i=1; i <= ngOrder; ++i) { + iterate(ngSet, it) { + if(it->first.size() == i) + orlm->UpdateORLM(it->first, it->second); + } + } +#endif + } + void breakOutParams(const params_t& params) { + params_t::const_iterator si = params.find("source"); + if(si == params.end()) + throw xmlrpc_c::fault("Missing source sentence", xmlrpc_c::fault::CODE_PARSE); + source_ = xmlrpc_c::value_string(si->second); + cerr << "source = " << source_ << endl; + si = params.find("target"); + if(si == params.end()) + throw xmlrpc_c::fault("Missing target sentence", xmlrpc_c::fault::CODE_PARSE); + target_ = xmlrpc_c::value_string(si->second); + cerr << "target = " << target_ << endl; + si = params.find("alignment"); + if(si == params.end()) + throw xmlrpc_c::fault("Missing alignment", xmlrpc_c::fault::CODE_PARSE); + alignment_ = xmlrpc_c::value_string(si->second); + cerr << "alignment = " << alignment_ << endl; + si = params.find("bounded"); + bounded_ = (si != params.end()); + si = params.find("updateORLM"); + add2ORLM_ = (si != params.end()); + } +}; + +class Translator : public xmlrpc_c::method +{ +public: + Translator() { + // signature and help strings are documentation -- the client + // can query this information with a system.methodSignature and + // system.methodHelp RPC. + this->_signature = "S:S"; + this->_help = "Does translation"; + } + + void + execute(xmlrpc_c::paramList const& paramList, + xmlrpc_c::value * const retvalP) { + + const params_t params = paramList.getStruct(0); + paramList.verifyEnd(1); + params_t::const_iterator si = params.find("text"); + if (si == params.end()) { + throw xmlrpc_c::fault( + "Missing source text", + xmlrpc_c::fault::CODE_PARSE); + } + const string source( + (xmlrpc_c::value_string(si->second))); + + cerr << "Input: " << source << endl; + si = params.find("align"); + bool addAlignInfo = (si != params.end()); + si = params.find("sg"); + bool addGraphInfo = (si != params.end()); + si = params.find("topt"); + bool addTopts = (si != params.end()); + si = params.find("report-all-factors"); + bool reportAllFactors = (si != params.end()); + + const StaticData &staticData = StaticData::Instance(); + + if (addGraphInfo) { + (const_cast<StaticData&>(staticData)).SetOutputSearchGraph(true); + } + + const TranslationSystem& system = getTranslationSystem(params); + + Sentence sentence; + const vector<FactorType> &inputFactorOrder = + staticData.GetInputFactorOrder(); + stringstream in(source + "\n"); + sentence.Read(in,inputFactorOrder); + Manager manager(sentence,staticData.GetSearchAlgorithm(), &system); + manager.ProcessSentence(); + const Hypothesis* hypo = manager.GetBestHypothesis(); + + vector<xmlrpc_c::value> alignInfo; + stringstream out, graphInfo, transCollOpts; + outputHypo(out,hypo,addAlignInfo,alignInfo,reportAllFactors); + + map<string, xmlrpc_c::value> retData; + pair<string, xmlrpc_c::value> + text("text", xmlrpc_c::value_string(out.str())); + cerr << "Output: " << out.str() << endl; + if (addAlignInfo) { + retData.insert(pair<string, xmlrpc_c::value>("align", xmlrpc_c::value_array(alignInfo))); + } + retData.insert(text); + + if(addGraphInfo) { + insertGraphInfo(manager,retData); + (const_cast<StaticData&>(staticData)).SetOutputSearchGraph(false); + } + if (addTopts) { + insertTranslationOptions(manager,retData); + } + *retvalP = xmlrpc_c::value_struct(retData); + } + + void outputHypo(ostream& out, const Hypothesis* hypo, bool addAlignmentInfo, vector<xmlrpc_c::value>& alignInfo, bool reportAllFactors = false) { + if (hypo->GetPrevHypo() != NULL) { + outputHypo(out,hypo->GetPrevHypo(),addAlignmentInfo, alignInfo, reportAllFactors); + Phrase p = hypo->GetCurrTargetPhrase(); + if(reportAllFactors) { + out << p << " "; + } else { + for (size_t pos = 0 ; pos < p.GetSize() ; pos++) { + const Factor *factor = p.GetFactor(pos, 0); + out << *factor << " "; + } + } + + if (addAlignmentInfo) { + /** + * Add the alignment info to the array. This is in target order and consists of + * (tgt-start, src-start, src-end) triples. + **/ + map<string, xmlrpc_c::value> phraseAlignInfo; + phraseAlignInfo["tgt-start"] = xmlrpc_c::value_int(hypo->GetCurrTargetWordsRange().GetStartPos()); + phraseAlignInfo["src-start"] = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetStartPos()); + phraseAlignInfo["src-end"] = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetEndPos()); + alignInfo.push_back(xmlrpc_c::value_struct(phraseAlignInfo)); + } + } + } + + void insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData) { + vector<xmlrpc_c::value> searchGraphXml; + vector<SearchGraphNode> searchGraph; + manager.GetSearchGraph(searchGraph); + for (vector<SearchGraphNode>::const_iterator i = searchGraph.begin(); i != searchGraph.end(); ++i) { + map<string, xmlrpc_c::value> searchGraphXmlNode; + searchGraphXmlNode["forward"] = xmlrpc_c::value_double(i->forward); + searchGraphXmlNode["fscore"] = xmlrpc_c::value_double(i->fscore); + const Hypothesis* hypo = i->hypo; + searchGraphXmlNode["hyp"] = xmlrpc_c::value_int(hypo->GetId()); + searchGraphXmlNode["stack"] = xmlrpc_c::value_int(hypo->GetWordsBitmap().GetNumWordsCovered()); + if (hypo->GetId() != 0) { + const Hypothesis *prevHypo = hypo->GetPrevHypo(); + searchGraphXmlNode["back"] = xmlrpc_c::value_int(prevHypo->GetId()); + searchGraphXmlNode["score"] = xmlrpc_c::value_double(hypo->GetScore()); + searchGraphXmlNode["transition"] = xmlrpc_c::value_double(hypo->GetScore() - prevHypo->GetScore()); + if (i->recombinationHypo) { + searchGraphXmlNode["recombined"] = xmlrpc_c::value_int(i->recombinationHypo->GetId()); + } + searchGraphXmlNode["cover-start"] = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetStartPos()); + searchGraphXmlNode["cover-end"] = xmlrpc_c::value_int(hypo->GetCurrSourceWordsRange().GetEndPos()); + searchGraphXmlNode["out"] = + xmlrpc_c::value_string(hypo->GetCurrTargetPhrase().GetStringRep(StaticData::Instance().GetOutputFactorOrder())); + } + searchGraphXml.push_back(xmlrpc_c::value_struct(searchGraphXmlNode)); + } + retData.insert(pair<string, xmlrpc_c::value>("sg", xmlrpc_c::value_array(searchGraphXml))); + } + + void insertTranslationOptions(Manager& manager, map<string, xmlrpc_c::value>& retData) { + const TranslationOptionCollection* toptsColl = manager.getSntTranslationOptions(); + vector<xmlrpc_c::value> toptsXml; + for (size_t startPos = 0 ; startPos < toptsColl->GetSize() ; ++startPos) { + size_t maxSize = toptsColl->GetSize() - startPos; + size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength(); + maxSize = std::min(maxSize, maxSizePhrase); + + for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) { + WordsRange range(startPos,endPos); + const TranslationOptionList& fullList = toptsColl->GetTranslationOptionList(range); + for (size_t i = 0; i < fullList.size(); i++) { + const TranslationOption* topt = fullList.Get(i); + map<string, xmlrpc_c::value> toptXml; + toptXml["phrase"] = xmlrpc_c::value_string(topt->GetTargetPhrase(). + GetStringRep(StaticData::Instance().GetOutputFactorOrder())); + toptXml["fscore"] = xmlrpc_c::value_double(topt->GetFutureScore()); + toptXml["start"] = xmlrpc_c::value_int(startPos); + toptXml["end"] = xmlrpc_c::value_int(endPos); + vector<xmlrpc_c::value> scoresXml; + ScoreComponentCollection scores = topt->GetScoreBreakdown(); + for (size_t j = 0; j < scores.size(); ++j) { + scoresXml.push_back(xmlrpc_c::value_double(scores[j])); + } + toptXml["scores"] = xmlrpc_c::value_array(scoresXml); + toptsXml.push_back(xmlrpc_c::value_struct(toptXml)); + } + } + } + retData.insert(pair<string, xmlrpc_c::value>("topt", xmlrpc_c::value_array(toptsXml))); + } + + + +}; + + +int main(int argc, char** argv) +{ + + //Extract port and log, send other args to moses + char** mosesargv = new char*[argc+2]; + int mosesargc = 0; + int port = 8080; + const char* logfile = "/dev/null"; + bool isSerial = false; + + for (int i = 0; i < argc; ++i) { + if (!strcmp(argv[i],"--server-port")) { + ++i; + if (i >= argc) { + cerr << "Error: Missing argument to --server-port" << endl; + exit(1); + } else { + port = atoi(argv[i]); + } + } else if (!strcmp(argv[i],"--server-log")) { + ++i; + if (i >= argc) { + cerr << "Error: Missing argument to --server-log" << endl; + exit(1); + } else { + logfile = argv[i]; + } + } else if (!strcmp(argv[i], "--serial")) { + cerr << "Running single-threaded server" << endl; + isSerial = true; + } else { + mosesargv[mosesargc] = new char[strlen(argv[i])+1]; + strcpy(mosesargv[mosesargc],argv[i]); + ++mosesargc; + } + } + + Parameter* params = new Parameter(); + if (!params->LoadParam(mosesargc,mosesargv)) { + params->Explain(); + exit(1); + } + if (!StaticData::LoadDataStatic(params)) { + exit(1); + } + + xmlrpc_c::registry myRegistry; + + xmlrpc_c::methodPtr const translator(new Translator); + xmlrpc_c::methodPtr const updater(new Updater); + + myRegistry.addMethod("translate", translator); + myRegistry.addMethod("updater", updater); + + xmlrpc_c::serverAbyss myAbyssServer( + myRegistry, + port, // TCP port on which to listen + logfile + ); + + cerr << "Listening on port " << port << endl; + if (isSerial) { + while(1) { + myAbyssServer.runOnce(); + } + } else { + myAbyssServer.run(); + } + // xmlrpc_c::serverAbyss.run() never returns + CHECK(false); + return 0; +} diff --git a/contrib/server/sgclient.perl b/contrib/server/sgclient.perl new file mode 100755 index 000000000..9a23ad8f4 --- /dev/null +++ b/contrib/server/sgclient.perl @@ -0,0 +1,52 @@ +#!/usr/bin/env perl + +# +# Sample client to print out the search graph +# + +use Encode; +use XMLRPC::Lite; +use utf8; + +$url = "http://localhost:8086/RPC2"; +$proxy = XMLRPC::Lite->proxy($url); + +$text = "il a souhaité que la présidence trace à nice le chemin pour l' avenir ."; +#$text = "je ne sais pas"; + +# Work-around for XMLRPC::Lite bug +$encoded = SOAP::Data->type(string => Encode::encode("utf8",$text)); + +#my %param = ("text" => $encoded ); +my %param = ("text" => $encoded , "sg" => "true", "topt" => "true"); +#my %param = ("text" => $encoded , "topt" => "true"); +die "translation failed" unless $result = $proxy->call("translate",\%param)->result; +print $result->{'text'} . "\n"; +exit; + +if ($result->{'sg'}) { + print "Search graph: \n"; + $sg = $result->{'sg'}; + foreach my $sgn (@$sg) { + foreach my $key (keys %$sgn) { + my $value = $sgn->{$key}; + print "$key=$value "; + } + print "\n"; + } +} + +if ($result->{'topt'}) { + print "Translation options: \n"; + $sg = $result->{'topt'}; + foreach my $sgn (@$sg) { + foreach my $key (keys %$sgn) { + my $value = $sgn->{$key}; + if (ref($value) eq 'ARRAY') { + $value = join(",", @$value); + } + print "$key=$value "; + } + print "\n"; + } +} diff --git a/contrib/sigtest-filter/Makefile b/contrib/sigtest-filter/Makefile new file mode 100644 index 000000000..ddefc907b --- /dev/null +++ b/contrib/sigtest-filter/Makefile @@ -0,0 +1,10 @@ +SALMDIR=/Users/hieuhoang/workspace/salm +FLAVOR?=o32 +INC=-I$(SALMDIR)/Src/Shared -I$(SALMDIR)/Src/SuffixArrayApplications -I$(SALMDIR)/Src/SuffixArrayApplications/SuffixArraySearch +OBJS=$(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArrayApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArraySearchApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_String.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_IDVocabulary.$(FLAVOR) + +all: filter-pt + +filter-pt: filter-pt.cpp + ./check-install $(SALMDIR) + $(CXX) -O6 $(INC) $(OBJS) -o filter-pt filter-pt.cpp diff --git a/contrib/sigtest-filter/README.txt b/contrib/sigtest-filter/README.txt new file mode 100644 index 000000000..b21129b89 --- /dev/null +++ b/contrib/sigtest-filter/README.txt @@ -0,0 +1,42 @@ +Re-implementation of Johnson et al. (2007)'s phrasetable filtering strategy. + +This implementation relies on Joy Zhang's SALM Suffix Array toolkit. It is +available here: + + http://projectile.sv.cmu.edu/research/public/tools/salm/salm.htm + +--Chris Dyer <redpony@umd.edu> + +BUILD INSTRUCTIONS +--------------------------------- + +1. Download and build SALM. + +2. make SALMDIR=/path/to/SALM + + +USAGE INSTRUCTIONS +--------------------------------- + +1. Using the SALM/Bin/Linux/Index/IndexSA.O32, create a suffix array index + of the source and target sides of your training bitext. + +2. cat phrase-table.txt | ./filter-pt -e TARG.suffix -f SOURCE.suffix \ + -l <FILTER-VALUE> + + FILTER-VALUE is the -log prob threshold described in Johnson et al. + (2007)'s paper. It may be either 'a+e', 'a-e', or a positive real + value. 'a+e' is a good setting- it filters out <1,1,1> phrase pairs. + I also recommend using -n 30, which filteres out all but the top + 30 phrase pairs, sorted by P(e|f). This was used in the paper. + +3. Run with no options to see more use-cases. + + +REFERENCES +--------------------------------- + +H. Johnson, J. Martin, G. Foster and R. Kuhn. (2007) Improving Translation + Quality by Discarding Most of the Phrasetable. In Proceedings of the 2007 + Joint Conference on Empirical Methods in Natural Language Processing and + Computational Natural Language Learning (EMNLP-CoNLL), pp. 967-975. diff --git a/contrib/sigtest-filter/WIN32_functions.cpp b/contrib/sigtest-filter/WIN32_functions.cpp new file mode 100644 index 000000000..60ddd340c --- /dev/null +++ b/contrib/sigtest-filter/WIN32_functions.cpp @@ -0,0 +1,231 @@ +// XGetopt.cpp Version 1.2
+//
+// Author: Hans Dietrich
+// hdietrich2@hotmail.com
+//
+// Description:
+// XGetopt.cpp implements getopt(), a function to parse command lines.
+//
+// History
+// Version 1.2 - 2003 May 17
+// - Added Unicode support
+//
+// Version 1.1 - 2002 March 10
+// - Added example to XGetopt.cpp module header
+//
+// This software is released into the public domain.
+// You are free to use it in any way you like.
+//
+// This software is provided "as is" with no expressed
+// or implied warranty. I accept no liability for any
+// damage or loss of business that this software may cause.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+
+///////////////////////////////////////////////////////////////////////////////
+// if you are using precompiled headers then include this line:
+///////////////////////////////////////////////////////////////////////////////
+
+
+///////////////////////////////////////////////////////////////////////////////
+// if you are not using precompiled headers then include these lines:
+//#include <windows.h>
+//#include <stdio.h>
+//#include <tchar.h>
+///////////////////////////////////////////////////////////////////////////////
+
+
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+#include "WIN32_functions.h"
+
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// X G e t o p t . c p p
+//
+//
+// NAME
+// getopt -- parse command line options
+//
+// SYNOPSIS
+// int getopt(int argc, char *argv[], char *optstring)
+//
+// extern char *optarg;
+// extern int optind;
+//
+// DESCRIPTION
+// The getopt() function parses the command line arguments. Its
+// arguments argc and argv are the argument count and array as
+// passed into the application on program invocation. In the case
+// of Visual C++ programs, argc and argv are available via the
+// variables __argc and __argv (double underscores), respectively.
+// getopt returns the next option letter in argv that matches a
+// letter in optstring. (Note: Unicode programs should use
+// __targv instead of __argv. Also, all character and string
+// literals should be enclosed in ( ) ).
+//
+// optstring is a string of recognized option letters; if a letter
+// is followed by a colon, the option is expected to have an argument
+// that may or may not be separated from it by white space. optarg
+// is set to point to the start of the option argument on return from
+// getopt.
+//
+// Option letters may be combined, e.g., "-ab" is equivalent to
+// "-a -b". Option letters are case sensitive.
+//
+// getopt places in the external variable optind the argv index
+// of the next argument to be processed. optind is initialized
+// to 0 before the first call to getopt.
+//
+// When all options have been processed (i.e., up to the first
+// non-option argument), getopt returns EOF, optarg will point
+// to the argument, and optind will be set to the argv index of
+// the argument. If there are no non-option arguments, optarg
+// will be set to NULL.
+//
+// The special option "--" may be used to delimit the end of the
+// options; EOF will be returned, and "--" (and everything after it)
+// will be skipped.
+//
+// RETURN VALUE
+// For option letters contained in the string optstring, getopt
+// will return the option letter. getopt returns a question mark (?)
+// when it encounters an option letter not included in optstring.
+// EOF is returned when processing is finished.
+//
+// BUGS
+// 1) Long options are not supported.
+// 2) The GNU double-colon extension is not supported.
+// 3) The environment variable POSIXLY_CORRECT is not supported.
+// 4) The + syntax is not supported.
+// 5) The automatic permutation of arguments is not supported.
+// 6) This implementation of getopt() returns EOF if an error is
+// encountered, instead of -1 as the latest standard requires.
+//
+// EXAMPLE
+// BOOL CMyApp::ProcessCommandLine(int argc, char *argv[])
+// {
+// int c;
+//
+// while ((c = getopt(argc, argv, ("aBn:"))) != EOF)
+// {
+// switch (c)
+// {
+// case ('a'):
+// TRACE(("option a\n"));
+// //
+// // set some flag here
+// //
+// break;
+//
+// case ('B'):
+// TRACE( ("option B\n"));
+// //
+// // set some other flag here
+// //
+// break;
+//
+// case ('n'):
+// TRACE(("option n: value=%d\n"), atoi(optarg));
+// //
+// // do something with value here
+// //
+// break;
+//
+// case ('?'):
+// TRACE(("ERROR: illegal option %s\n"), argv[optind-1]);
+// return FALSE;
+// break;
+//
+// default:
+// TRACE(("WARNING: no handler for option %c\n"), c);
+// return FALSE;
+// break;
+// }
+// }
+// //
+// // check for non-option args here
+// //
+// return TRUE;
+// }
+//
+///////////////////////////////////////////////////////////////////////////////
+
+char *optarg; // global argument pointer
+int optind = 0; // global argv index
+
+int getopt(int argc, char *argv[], char *optstring)
+{
+ static char *next = NULL;
+ if (optind == 0)
+ next = NULL;
+
+ optarg = NULL;
+
+ if (next == NULL || *next =='\0') {
+ if (optind == 0)
+ optind++;
+
+ if (optind >= argc || argv[optind][0] != ('-') || argv[optind][1] == ('\0')) {
+ optarg = NULL;
+ if (optind < argc)
+ optarg = argv[optind];
+ return EOF;
+ }
+
+ if (strcmp(argv[optind], "--") == 0) {
+ optind++;
+ optarg = NULL;
+ if (optind < argc)
+ optarg = argv[optind];
+ return EOF;
+ }
+
+ next = argv[optind];
+ next++; // skip past -
+ optind++;
+ }
+
+ char c = *next++;
+ char *cp = strchr(optstring, c);
+
+ if (cp == NULL || c == (':'))
+ return ('?');
+
+ cp++;
+ if (*cp == (':')) {
+ if (*next != ('\0')) {
+ optarg = next;
+ next = NULL;
+ } else if (optind < argc) {
+ optarg = argv[optind];
+ optind++;
+ } else {
+ return ('?');
+ }
+ }
+
+ return c;
+}
+
+// for an overview, see
+// W. Press, S. Teukolsky and W. Vetterling. (1992) Numerical Recipes in C. Chapter 6.1.
+double lgamma(int x)
+{
+ // size_t xx=(size_t)x; xx--; size_t sum=1; while (xx) { sum *= xx--; } return log((double)(sum));
+ if (x <= 2) {
+ return 0.0;
+ }
+ static double coefs[6] = {76.18009172947146, -86.50532032941677, 24.01409824083091, -1.231739572450155, 0.1208650973866179e-2, -0.5395239384953e-5};
+ double tmp=(double)x+5.5;
+ tmp -= (((double)x)+0.5)*log(tmp);
+ double y=(double)x;
+ double sum = 1.000000000190015;
+ for (size_t j=0; j<6; ++j) {
+ sum += coefs[j]/++y;
+ }
+ return -tmp+log(2.5066282746310005*sum/(double)x);
+}
\ No newline at end of file diff --git a/contrib/sigtest-filter/WIN32_functions.h b/contrib/sigtest-filter/WIN32_functions.h new file mode 100644 index 000000000..6a719392e --- /dev/null +++ b/contrib/sigtest-filter/WIN32_functions.h @@ -0,0 +1,24 @@ +// XGetopt.h Version 1.2
+//
+// Author: Hans Dietrich
+// hdietrich2@hotmail.com
+//
+// This software is released into the public domain.
+// You are free to use it in any way you like.
+//
+// This software is provided "as is" with no expressed
+// or implied warranty. I accept no liability for any
+// damage or loss of business that this software may cause.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef XGETOPT_H
+#define XGETOPT_H
+
+extern int optind, opterr;
+extern char *optarg;
+
+int getopt(int argc, char *argv[], char *optstring);
+double lgamma(int x);
+
+#endif //XGETOPT_H
diff --git a/contrib/sigtest-filter/check-install b/contrib/sigtest-filter/check-install new file mode 100755 index 000000000..ba4f431e0 --- /dev/null +++ b/contrib/sigtest-filter/check-install @@ -0,0 +1,5 @@ +#!/usr/bin/perl -w +use strict; +my $path = shift @ARGV; +die "Can't find SALM installation path: $path\nPlease use:\n\n make SALMDIR=/path/to/SALM\n\n" unless (-d $path); +exit 0; diff --git a/contrib/sigtest-filter/filter-pt.cpp b/contrib/sigtest-filter/filter-pt.cpp new file mode 100644 index 000000000..b0828ae33 --- /dev/null +++ b/contrib/sigtest-filter/filter-pt.cpp @@ -0,0 +1,382 @@ + +#include <cstring> +#include <cassert> +#include <cstdio> +#include <cstdlib> +#include <algorithm> + +#include "_SuffixArraySearchApplicationBase.h" + +#include <vector> +#include <iostream> +#include <set> + +#ifdef WIN32 +#include "WIN32_functions.h" +#else +#include <unistd.h> +#endif + +typedef std::set<TextLenType> SentIdSet; +typedef std::map<std::string, SentIdSet> PhraseSetMap; + +#undef min + +// constants +const size_t MINIMUM_SIZE_TO_KEEP = 10000; // reduce this to improve memory usage, +// increase for speed +const std::string SEPARATOR = " ||| "; + +const double ALPHA_PLUS_EPS = -1000.0; // dummy value +const double ALPHA_MINUS_EPS = -2000.0; // dummy value + +// configuration params +int pfe_filter_limit = 0; // 0 = don't filter anything based on P(f|e) +bool print_cooc_counts = false; // add cooc counts to phrase table? +bool print_neglog_significance = false; // add -log(p) to phrase table? +double sig_filter_limit = 0; // keep phrase pairs with -log(sig) > sig_filter_limit +// higher = filter-more +bool pef_filter_only = false; // only filter based on pef + +// globals +PhraseSetMap esets; +double p_111 = 0.0; // alpha +size_t nremoved_sigfilter = 0; +size_t nremoved_pfefilter = 0; + +C_SuffixArraySearchApplicationBase e_sa; +C_SuffixArraySearchApplicationBase f_sa; +int num_lines; + +void usage() +{ + std::cerr << "\nFilter phrase table using significance testing as described\n" + << "in H. Johnson, et al. (2007) Improving Translation Quality\n" + << "by Discarding Most of the Phrasetable. EMNLP 2007.\n" + << "\nUsage:\n" + << "\n filter-pt -e english.suf-arr -f french.suf-arr\n" + << " [-c] [-p] [-l threshold] [-n num] < PHRASE-TABLE > FILTERED-PHRASE-TABLE\n\n" + << " [-l threshold] >0.0, a+e, or a-e: keep values that have a -log significance > this\n" + << " [-n num ] 0, 1...: 0=no filtering, >0 sort by P(e|f) and keep the top num elements\n" + << " [-c ] add the cooccurence counts to the phrase table\n" + << " [-p ] add -log(significance) to the phrasetable\n\n"; + exit(1); +} + +struct PTEntry { + PTEntry(const std::string& str, int index); + std::string f_phrase; + std::string e_phrase; + std::string extra; + std::string scores; + float pfe; + int cf; + int ce; + int cfe; + float nlog_pte; + void set_cooc_stats(int _cef, int _cf, int _ce, float nlp) { + cfe = _cef; + cf = _cf; + ce = _ce; + nlog_pte = nlp; + } + +}; + +PTEntry::PTEntry(const std::string& str, int index) : + cf(0), ce(0), cfe(0), nlog_pte(0.0) +{ + size_t pos = 0; + std::string::size_type nextPos = str.find(SEPARATOR, pos); + this->f_phrase = str.substr(pos,nextPos); + + pos = nextPos + SEPARATOR.size(); + nextPos = str.find(SEPARATOR, pos); + this->e_phrase = str.substr(pos,nextPos-pos); + + pos = nextPos + SEPARATOR.size(); + nextPos = str.find(SEPARATOR, pos); + this->scores = str.substr(pos,nextPos-pos); + + pos = nextPos + SEPARATOR.size(); + this->extra = str.substr(pos); + + int c = 0; + std::string::iterator i=scores.begin(); + if (index > 0) { + for (; i != scores.end(); ++i) { + if ((*i) == ' ') { + c++; + if (c == index) break; + } + } + } + if (i != scores.end()) { + ++i; + } + char f[24]; + char *fp=f; + while (i != scores.end() && *i != ' ') { + *fp++=*i++; + } + *fp++=0; + + this->pfe = atof(f); + + // std::cerr << "L: " << f_phrase << " ::: " << e_phrase << " ::: " << scores << " ::: " << pfe << std::endl; + // std::cerr << "X: " << extra << "\n"; +} + +struct PfeComparer { + bool operator()(const PTEntry* a, const PTEntry* b) const { + return a->pfe > b->pfe; + } +}; + +struct NlogSigThresholder { + NlogSigThresholder(float threshold) : t(threshold) {} + float t; + bool operator()(const PTEntry* a) const { + if (a->nlog_pte < t) { + delete a; + return true; + } else return false; + } +}; + +std::ostream& operator << (std::ostream& os, const PTEntry& pp) +{ + os << pp.f_phrase << " ||| " << pp.e_phrase; + os << " ||| " << pp.scores; + if (pp.extra.size()>0) os << " ||| " << pp.extra; + if (print_cooc_counts) os << " ||| " << pp.cfe << " " << pp.cf << " " << pp.ce; + if (print_neglog_significance) os << " ||| " << pp.nlog_pte; + return os; +} + +void print(int a, int b, int c, int d, float p) +{ + std::cerr << a << "\t" << b << "\t P=" << p << "\n" + << c << "\t" << d << "\t xf=" << (double)(b)*(double)(c)/(double)(a+1)/(double)(d+1) << "\n\n"; +} + +// 2x2 (one-sided) Fisher's exact test +// see B. Moore. (2004) On Log Likelihood and the Significance of Rare Events +double fisher_exact(int cfe, int ce, int cf) +{ + assert(cfe <= ce); + assert(cfe <= cf); + + int a = cfe; + int b = (cf - cfe); + int c = (ce - cfe); + int d = (num_lines - ce - cf + cfe); + int n = a + b + c + d; + + double cp = exp(lgamma(1+a+c) + lgamma(1+b+d) + lgamma(1+a+b) + lgamma(1+c+d) - lgamma(1+n) - lgamma(1+a) - lgamma(1+b) - lgamma(1+c) - lgamma(1+d)); + double total_p = 0.0; + int tc = std::min(b,c); + for (int i=0; i<=tc; i++) { + total_p += cp; +// double lg = lgamma(1+a+c) + lgamma(1+b+d) + lgamma(1+a+b) + lgamma(1+c+d) - lgamma(1+n) - lgamma(1+a) - lgamma(1+b) - lgamma(1+c) - lgamma(1+d); double cp = exp(lg); +// print(a,b,c,d,cp); + double coef = (double)(b)*(double)(c)/(double)(a+1)/(double)(d+1); + cp *= coef; + ++a; + --c; + ++d; + --b; + } + return total_p; +} + +// input: unordered list of translation options for a single source phrase +void compute_cooc_stats_and_filter(std::vector<PTEntry*>& options) +{ + if (pfe_filter_limit>0 && options.size() > pfe_filter_limit) { + nremoved_pfefilter += (options.size() - pfe_filter_limit); + std::nth_element(options.begin(), options.begin()+pfe_filter_limit, options.end(), PfeComparer()); + for (std::vector<PTEntry*>::iterator i=options.begin()+pfe_filter_limit; i != options.end(); ++i) + delete *i; + options.erase(options.begin()+pfe_filter_limit,options.end()); + } + if (pef_filter_only) return; + + SentIdSet fset; + vector<S_SimplePhraseLocationElement> locations; + //std::cerr << "Looking up f-phrase: " << options.front()->f_phrase << "\n"; + + locations = f_sa.locateExactPhraseInCorpus(options.front()->f_phrase.c_str()); + if(locations.size()==0) { + cerr<<"No occurrences found!!\n"; + } + for (vector<S_SimplePhraseLocationElement>::iterator i=locations.begin(); + i != locations.end(); + ++i) { + fset.insert(i->sentIdInCorpus); + } + size_t cf = fset.size(); + for (std::vector<PTEntry*>::iterator i=options.begin(); i != options.end(); ++i) { + const std::string& e_phrase = (*i)->e_phrase; + size_t cef=0; + SentIdSet& eset = esets[(*i)->e_phrase]; + if (eset.empty()) { + //std::cerr << "Looking up e-phrase: " << e_phrase << "\n"; + vector<S_SimplePhraseLocationElement> locations = e_sa.locateExactPhraseInCorpus(e_phrase.c_str()); + for (vector<S_SimplePhraseLocationElement>::iterator i=locations.begin(); i!= locations.end(); ++i) { + TextLenType curSentId = i->sentIdInCorpus; + eset.insert(curSentId); + } + } + size_t ce=eset.size(); + if (ce < cf) { + for (SentIdSet::iterator i=eset.begin(); i != eset.end(); ++i) { + if (fset.find(*i) != fset.end()) cef++; + } + } else { + for (SentIdSet::iterator i=fset.begin(); i != fset.end(); ++i) { + if (eset.find(*i) != eset.end()) cef++; + } + } + double nlp = -log(fisher_exact(cef, cf, ce)); + (*i)->set_cooc_stats(cef, cf, ce, nlp); + if (ce < MINIMUM_SIZE_TO_KEEP) { + esets.erase(e_phrase); + } + } + std::vector<PTEntry*>::iterator new_end = + std::remove_if(options.begin(), options.end(), NlogSigThresholder(sig_filter_limit)); + nremoved_sigfilter += (options.end() - new_end); + options.erase(new_end,options.end()); +} + +int main(int argc, char * argv[]) +{ + int c; + const char* efile=0; + const char* ffile=0; + int pfe_index = 2; + while ((c = getopt(argc, argv, "cpf:e:i:n:l:")) != -1) { + switch (c) { + case 'e': + efile = optarg; + break; + case 'f': + ffile = optarg; + break; + case 'i': // index of pfe in phrase table + pfe_index = atoi(optarg); + break; + case 'n': // keep only the top n entries in phrase table sorted by p(f|e) (0=all) + pfe_filter_limit = atoi(optarg); + std::cerr << "P(f|e) filter limit: " << pfe_filter_limit << std::endl; + break; + case 'c': + print_cooc_counts = true; + break; + case 'p': + print_neglog_significance = true; + break; + case 'l': + std::cerr << "-l = " << optarg << "\n"; + if (strcmp(optarg,"a+e") == 0) { + sig_filter_limit = ALPHA_PLUS_EPS; + } else if (strcmp(optarg,"a-e") == 0) { + sig_filter_limit = ALPHA_MINUS_EPS; + } else { + char *x; + sig_filter_limit = strtod(optarg, &x); + if (sig_filter_limit < 0.0) { + std::cerr << "Filter limit (-l) must be either 'a+e', 'a-e' or a real number >= 0.0\n"; + usage(); + } + } + break; + default: + usage(); + } + } + if (sig_filter_limit == 0.0) pef_filter_only = true; + //----------------------------------------------------------------------------- + if (optind != argc || ((!efile || !ffile) && !pef_filter_only)) { + usage(); + } + + //load the indexed corpus with vocabulary(noVoc=false) and with offset(noOffset=false) + if (!pef_filter_only) { + e_sa.loadData_forSearch(efile, false, false); + f_sa.loadData_forSearch(ffile, false, false); + size_t elines = e_sa.returnTotalSentNumber(); + size_t flines = f_sa.returnTotalSentNumber(); + if (elines != flines) { + std::cerr << "Number of lines in e-corpus != number of lines in f-corpus!\n"; + usage(); + } else { + std::cerr << "Training corpus: " << elines << " lines\n"; + num_lines = elines; + } + p_111 = -log(fisher_exact(1,1,1)); + std::cerr << "\\alpha = " << p_111 << "\n"; + if (sig_filter_limit == ALPHA_MINUS_EPS) { + sig_filter_limit = p_111 - 0.001; + } else if (sig_filter_limit == ALPHA_PLUS_EPS) { + sig_filter_limit = p_111 + 0.001; + } + std::cerr << "Sig filter threshold is = " << sig_filter_limit << "\n"; + } else { + std::cerr << "Filtering using P(e|f) only. n=" << pfe_filter_limit << std::endl; + } + + char tmpString[10000]; + std::string prev = ""; + std::vector<PTEntry*> options; + size_t pt_lines = 0; + while(!cin.eof()) { + cin.getline(tmpString,10000,'\n'); + if(++pt_lines%10000==0) { + std::cerr << "."; + if(pt_lines%500000==0) std::cerr << "[n:"<<pt_lines<<"]\n"; + } + + if(strlen(tmpString)>0) { + PTEntry* pp = new PTEntry(tmpString, pfe_index); + if (prev != pp->f_phrase) { + prev = pp->f_phrase; + + if (!options.empty()) { // always true after first line + compute_cooc_stats_and_filter(options); + } + for (std::vector<PTEntry*>::iterator i=options.begin(); i != options.end(); ++i) { + std::cout << **i << std::endl; + delete *i; + } + options.clear(); + options.push_back(pp); + + } else { + options.push_back(pp); + } + // for(int i=0;i<locations.size(); i++){ + // cout<<"SentId="<<locations[i].sentIdInCorpus<<" Pos="<<(int)locations[i].posInSentInCorpus<<endl; + // } + } + } + compute_cooc_stats_and_filter(options); + for (std::vector<PTEntry*>::iterator i=options.begin(); i != options.end(); ++i) { + std::cout << **i << std::endl; + delete *i; + } + float pfefper = (100.0*(float)nremoved_pfefilter)/(float)pt_lines; + float sigfper = (100.0*(float)nremoved_sigfilter)/(float)pt_lines; + std::cerr << "\n\n------------------------------------------------------\n" + << " unfiltered phrases pairs: " << pt_lines << "\n" + << "\n" + << " P(f|e) filter [first]: " << nremoved_pfefilter << " (" << pfefper << "%)\n" + << " significance filter: " << nremoved_sigfilter << " (" << sigfper << "%)\n" + << " TOTAL FILTERED: " << (nremoved_pfefilter + nremoved_sigfilter) << " (" << (sigfper + pfefper) << "%)\n" + << "\n" + << " FILTERED phrase pairs: " << (pt_lines - nremoved_pfefilter - nremoved_sigfilter) << " (" << (100.0-sigfper - pfefper) << "%)\n" + << "------------------------------------------------------\n"; + + return 0; +} diff --git a/contrib/sigtest-filter/sigtest-filter.sln b/contrib/sigtest-filter/sigtest-filter.sln new file mode 100644 index 000000000..517b06238 --- /dev/null +++ b/contrib/sigtest-filter/sigtest-filter.sln @@ -0,0 +1,20 @@ +
+Microsoft Visual Studio Solution File, Format Version 9.00
+# Visual Studio 2005
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sigtest-filter", "sigtest-filter.vcproj", "{FA2910DF-FD9D-4E6D-A393-9F9F9E309E78}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Win32 = Debug|Win32
+ Release|Win32 = Release|Win32
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {FA2910DF-FD9D-4E6D-A393-9F9F9E309E78}.Debug|Win32.ActiveCfg = Debug|Win32
+ {FA2910DF-FD9D-4E6D-A393-9F9F9E309E78}.Debug|Win32.Build.0 = Debug|Win32
+ {FA2910DF-FD9D-4E6D-A393-9F9F9E309E78}.Release|Win32.ActiveCfg = Release|Win32
+ {FA2910DF-FD9D-4E6D-A393-9F9F9E309E78}.Release|Win32.Build.0 = Release|Win32
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/contrib/sigtest-filter/sigtest-filter.vcproj b/contrib/sigtest-filter/sigtest-filter.vcproj new file mode 100644 index 000000000..a961ac61d --- /dev/null +++ b/contrib/sigtest-filter/sigtest-filter.vcproj @@ -0,0 +1,237 @@ +<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+ ProjectType="Visual C++"
+ Version="8.00"
+ Name="sigtest-filter"
+ ProjectGUID="{FA2910DF-FD9D-4E6D-A393-9F9F9E309E78}"
+ RootNamespace="sigtestfilter"
+ Keyword="Win32Proj"
+ >
+ <Platforms>
+ <Platform
+ Name="Win32"
+ />
+ </Platforms>
+ <ToolFiles>
+ </ToolFiles>
+ <Configurations>
+ <Configuration
+ Name="Debug|Win32"
+ OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+ IntermediateDirectory="$(ConfigurationName)"
+ ConfigurationType="1"
+ UseOfMFC="2"
+ CharacterSet="1"
+ >
+ <Tool
+ Name="VCPreBuildEventTool"
+ />
+ <Tool
+ Name="VCCustomBuildTool"
+ />
+ <Tool
+ Name="VCXMLDataGeneratorTool"
+ />
+ <Tool
+ Name="VCWebServiceProxyGeneratorTool"
+ />
+ <Tool
+ Name="VCMIDLTool"
+ />
+ <Tool
+ Name="VCCLCompilerTool"
+ Optimization="0"
+ AdditionalIncludeDirectories="..\..\..\SALM\Src\SuffixArrayApplications\SuffixArraySearch;..\..\..\SALM\Src\SuffixArrayApplications;..\..\..\SALM\Src\Shared"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+ MinimalRebuild="true"
+ BasicRuntimeChecks="3"
+ RuntimeLibrary="3"
+ UsePrecompiledHeader="0"
+ WarningLevel="3"
+ Detect64BitPortabilityProblems="true"
+ DebugInformationFormat="4"
+ />
+ <Tool
+ Name="VCManagedResourceCompilerTool"
+ />
+ <Tool
+ Name="VCResourceCompilerTool"
+ />
+ <Tool
+ Name="VCPreLinkEventTool"
+ />
+ <Tool
+ Name="VCLinkerTool"
+ LinkIncremental="2"
+ GenerateDebugInformation="true"
+ SubSystem="1"
+ TargetMachine="1"
+ />
+ <Tool
+ Name="VCALinkTool"
+ />
+ <Tool
+ Name="VCManifestTool"
+ />
+ <Tool
+ Name="VCXDCMakeTool"
+ />
+ <Tool
+ Name="VCBscMakeTool"
+ />
+ <Tool
+ Name="VCFxCopTool"
+ />
+ <Tool
+ Name="VCAppVerifierTool"
+ />
+ <Tool
+ Name="VCWebDeploymentTool"
+ />
+ <Tool
+ Name="VCPostBuildEventTool"
+ />
+ </Configuration>
+ <Configuration
+ Name="Release|Win32"
+ OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+ IntermediateDirectory="$(ConfigurationName)"
+ ConfigurationType="1"
+ UseOfMFC="2"
+ CharacterSet="1"
+ WholeProgramOptimization="1"
+ >
+ <Tool
+ Name="VCPreBuildEventTool"
+ />
+ <Tool
+ Name="VCCustomBuildTool"
+ />
+ <Tool
+ Name="VCXMLDataGeneratorTool"
+ />
+ <Tool
+ Name="VCWebServiceProxyGeneratorTool"
+ />
+ <Tool
+ Name="VCMIDLTool"
+ />
+ <Tool
+ Name="VCCLCompilerTool"
+ AdditionalIncludeDirectories="..\..\..\SALM\Src\SuffixArrayApplications\SuffixArraySearch;..\..\..\SALM\Src\SuffixArrayApplications;..\..\..\SALM\Src\Shared"
+ PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+ RuntimeLibrary="2"
+ UsePrecompiledHeader="0"
+ WarningLevel="3"
+ Detect64BitPortabilityProblems="true"
+ DebugInformationFormat="3"
+ />
+ <Tool
+ Name="VCManagedResourceCompilerTool"
+ />
+ <Tool
+ Name="VCResourceCompilerTool"
+ />
+ <Tool
+ Name="VCPreLinkEventTool"
+ />
+ <Tool
+ Name="VCLinkerTool"
+ LinkIncremental="1"
+ GenerateDebugInformation="true"
+ SubSystem="1"
+ OptimizeReferences="2"
+ EnableCOMDATFolding="2"
+ TargetMachine="1"
+ />
+ <Tool
+ Name="VCALinkTool"
+ />
+ <Tool
+ Name="VCManifestTool"
+ />
+ <Tool
+ Name="VCXDCMakeTool"
+ />
+ <Tool
+ Name="VCBscMakeTool"
+ />
+ <Tool
+ Name="VCFxCopTool"
+ />
+ <Tool
+ Name="VCAppVerifierTool"
+ />
+ <Tool
+ Name="VCWebDeploymentTool"
+ />
+ <Tool
+ Name="VCPostBuildEventTool"
+ />
+ </Configuration>
+ </Configurations>
+ <References>
+ </References>
+ <Files>
+ <Filter
+ Name="Source Files"
+ Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+ UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+ >
+ <File
+ RelativePath="..\..\..\SALM\Src\Shared\_IDVocabulary.cpp"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\SALM\Src\Shared\_String.cpp"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\SALM\Src\SuffixArrayApplications\_SuffixArrayApplicationBase.cpp"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\SALM\Src\SuffixArrayApplications\SuffixArraySearch\_SuffixArraySearchApplicationBase.cpp"
+ >
+ </File>
+ <File
+ RelativePath=".\filter-pt.cpp"
+ >
+ </File>
+ <File
+ RelativePath=".\WIN32_functions.cpp"
+ >
+ </File>
+ </Filter>
+ <Filter
+ Name="Header Files"
+ Filter="h;hpp;hxx;hm;inl;inc;xsd"
+ UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+ >
+ <File
+ RelativePath="..\..\..\SALM\Src\Shared\_IDVocabulary.h"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\SALM\Src\SuffixArrayApplications\_SuffixArrayApplicationBase.h"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\SALM\Src\SuffixArrayApplications\SuffixArraySearch\_SuffixArraySearchApplicationBase.h"
+ >
+ </File>
+ <File
+ RelativePath=".\WIN32_functions.h"
+ >
+ </File>
+ </Filter>
+ <Filter
+ Name="Resource Files"
+ Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
+ UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
+ >
+ </Filter>
+ </Files>
+ <Globals>
+ </Globals>
+</VisualStudioProject>
diff --git a/contrib/synlm/hhmm/LICENSE b/contrib/synlm/hhmm/LICENSE new file mode 100644 index 000000000..25b5b7097 --- /dev/null +++ b/contrib/synlm/hhmm/LICENSE @@ -0,0 +1,695 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. + + +MOSES EXCEPTION TO THE GPL + +Certain source files distributed by ModelBlocks developers are +subject to the following clarification and special exception to the GPL, but +only where ModelBlocks developers have expressly included in the particular source file's header +the words "ModelBlocks developers designate this particular file as subject to the "Moses" +exception as provided by ModelBlocks developers in the LICENSE file that accompanies this code." + + Linking this library statically or dynamically with other modules is making + a combined work based on this library. Thus, the terms and conditions of + the GNU General Public License cover the whole combination. + + As a special exception, the copyright holders of this library give you + permission to compile and link this library with Moses modules to produce an + executable, and to copy and distribute the resulting executable under the terms + of the Moses license, provided that you also meet the terms and conditions of that license. + If you modify this library, you may extend this exception to your version of + the library, but you are not obligated to do so. If you do not wish to do + so, delete this exception statement from your version. diff --git a/contrib/synlm/hhmm/rvtl/include/nl-archetypeset.h b/contrib/synlm/hhmm/rvtl/include/nl-archetypeset.h new file mode 100644 index 000000000..914e85e92 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-archetypeset.h @@ -0,0 +1,180 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_ARCHETYPESET_ +#define _NL_ARCHETYPESET_ + +#include <boost/thread/thread.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/bind.hpp> +#include "nl-minheap.h" + +//////////////////////////////////////////////////////////////////////////////// + +template <class S, class T> +class Scored : public T { + public: + S scr; + Scored<S,T> ( ) : T() , scr() { } + Scored<S,T> ( S s ) : T() , scr(s) { } + Scored<S,T> ( S s, T& t ) : T(t), scr(s) { } + S& setScore() { return scr; } + S getScore() const { return scr; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template<char* SD1,class I,char* SD2,class T,char* SD3> +class Numbered : public T { + private: + // Data members... + I i; + public: + // Constructor / destructor methods... + Numbered<SD1,I,SD2,T,SD3> ( ) { } + Numbered<SD1,I,SD2,T,SD3> ( char* ps ) { ps>>*this>>"\0"; } + Numbered<SD1,I,SD2,T,SD3> ( const I& iA, const T& tA ) : T(tA), i(iA) { } + // Specification methods... + I& setNumber ( ) { return i; } + T& setT ( ) { return *this; } + // Extraction methods... + const I& getNumber ( ) const { return i; } + const T& getT ( ) const { return *this; } + // Input / output methods... + friend ostream& operator<< ( ostream& os, const Numbered<SD1,I,SD2,T,SD3>& rv ) { return os<<SD1<<rv.i<<SD2<<rv.getT()<<SD3; } + friend String& operator<< ( String& str, const Numbered<SD1,I,SD2,T,SD3>& rv ) { return str<<SD1<<rv.i<<SD2<<rv.getT()<<SD3; } + friend pair<StringInput,Numbered<SD1,I,SD2,T,SD3>*> operator>> ( StringInput ps, Numbered<SD1,I,SD2,T,SD3>& rv ) { return pair<StringInput,Numbered<SD1,I,SD2,T,SD3>*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,Numbered<SD1,I,SD2,T,SD3>*> delimbuff, const char* psPostDelim ) { + return ( (SD3[0]=='\0') ? delimbuff.first>>SD1>>delimbuff.second->i>>SD2>>delimbuff.second->setT()>>psPostDelim + : delimbuff.first>>SD1>>delimbuff.second->i>>SD2>>delimbuff.second->setT()>>SD3>>psPostDelim ); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template<class V> +class ArchetypeSet : public multimap<typename V::ElementType,Numbered<psX,int,psBar,V,psX> > { + private: + // Static data members... + static const int FIRST_INDEX_TO_CHECK = 0; + typedef Numbered<psX,int,psBar,V,psX> NV; + typedef multimap<typename V::ElementType,NV> MapType; + // Data members... + MinHeap<Scored<typename V::ElementType,pair<int,SafePtr<const NV> > > > hsivCalc; + public: + ArchetypeSet<V> ( ) { } + ArchetypeSet<V> ( const ArchetypeSet<V>& aa ) : MapType(aa) { cerr<<"\nCOPY!!!!\n\n"; } + ArchetypeSet<V>& operator= ( const ArchetypeSet<V>& aa ) { cerr<<"\nCOPY2!!!!\n\n"; MapType::operator=(aa); return *this; } + // Specification methods... + void add ( const V& ); + // Extraction methods... + bool isEmpty ( ) const { return MapType::empty(); } + pair<typename V::ElementType,int> getDistanceOfNearest ( const V& ) const; + void dump ( ) { for(typename MapType::const_iterator ii=MapType::begin(); ii!=MapType::end(); ii++) cerr<<ii->second<<"\n"; } +}; + +//////////////////// +template<class V> +void ArchetypeSet<V>::add ( const V& v ) { + //cerr<<"adding "<<v.get(FIRST_INDEX_TO_CHECK)<<" "<<MapType::size()<<" "<<v<<"\n"; + MapType::insert ( pair<typename V::ElementType,NV>(v.get(FIRST_INDEX_TO_CHECK),NV(MapType::size()+1,v) ) ); + ////cerr<<"adding "<<v.second.get(1)<<" ln"<<MapType::lower_bound(v.second.get(1))->second.lineNum.toInt()<<"\n"; +} + +//////////////////// +template<class V> +pair<typename V::ElementType,int> ArchetypeSet<V>::getDistanceOfNearest ( const V& v ) const { + //const Scored<typename V::ElementType,pair<int,SafePtr<const V> > > sipvDummy ( DBL_MAX ); + //MinHeap<Scored<typename V::ElementType,pair<int,SafePtr<const V> > > > hsiv ( MapType::size()+1, sipvDummy ); + MinHeap<Scored<typename V::ElementType,pair<int,SafePtr<const NV> > > >& hsiv = + const_cast<MinHeap<Scored<typename V::ElementType,pair<int,SafePtr<const NV> > > >&> ( hsivCalc ); + hsiv.clear(); + + typename MapType::const_iterator iUpper = MapType::upper_bound(v.get(FIRST_INDEX_TO_CHECK)); + typename MapType::const_iterator iLower = iUpper; if(iLower!=MapType::begin())iLower--; + ////cerr<<"seeking "<<v.get(0)<<" (upper=ln"<<(&iUpper->second)<<" "<<((iUpper!=MapType::end())?iUpper->first:-1)<<", lower=ln"<<&iLower->second<<" "<<iLower->first<<")\n"; + int iNext = 0; + if ( iUpper!=MapType::end() ) { + hsiv.set(iNext).first = FIRST_INDEX_TO_CHECK; + hsiv.set(iNext).second = SafePtr<const NV> ( iUpper->second ); + typename V::ElementType d = v.getMarginalDistance ( hsiv.get(iNext).first, hsiv.get(iNext).second.getRef() ); + hsiv.set(iNext).setScore() = d; + //hsiv.set(iNext).setScore() = v.getMarginalDistance ( hsiv.getMin().first, iUpper->second.second ); + ////int j = + hsiv.fixDecr(iNext); + ////cerr<<" adding ln"<<&hsiv.get(j).second.getRef()<<" marg-dist="<<d<<" new-score="<<double(hsiv.get(j).getScore())<<" new-pos="<<j<<"\n"; + iNext++; + ////for(int i=0;i<iNext;i++) cerr<<" "<<i<<": ln"<<hsiv.get(i).second.getRef().lineNum.toInt()<<" new-score="<<double(hsiv.get(i).getScore())<<"\n"; + } + hsiv.set(iNext).first = FIRST_INDEX_TO_CHECK; + hsiv.set(iNext).second = SafePtr<const NV> ( iLower->second ); + typename V::ElementType d = v.getMarginalDistance ( hsiv.get(iNext).first, hsiv.get(iNext).second.getRef() ); + hsiv.set(iNext).setScore() = d; + //hsiv.set(iNext).setScore() = v.getMarginalDistance ( hsiv.getMin().first, iLower->second.second ); + ////int j = + hsiv.fixDecr(iNext); + ////cerr<<" adding ln"<<&hsiv.get(j).second.getRef()<<" marg-dist="<<d<<" new-score="<<double(hsiv.get(j).getScore())<<" new-pos="<<j<<"\n"; + iNext++; + ////for(int i=0;i<iNext;i++) cerr<<" "<<i<<": ln"<<hsiv.get(i).second.getRef().lineNum.toInt()<<" new-score="<<double(hsiv.get(i).getScore())<<"\n"; + while ( hsiv.getMin().first < V::SIZE-1 ) { + typename V::ElementType d = v.getMarginalDistance ( ++hsiv.setMin().first, hsiv.getMin().second.getRef() ); + hsiv.setMin().setScore() += d; + ////cerr<<" matching ln"<<&hsiv.getMin().second.getRef()<<" i="<<hsiv.setMin().first<<" marg-dist="<<d<<" new-score="<<hsiv.getMin().getScore(); + ////int j = + hsiv.fixIncr(0); + ////cerr<<" new-pos="<<j<<"\n"; + ////if(j!=0) for(int i=0;i<iNext;i++) cerr<<" "<<i<<": ln"<<hsiv.get(i).second.getRef().lineNum.toInt()<<" new-score="<<double(hsiv.get(i).getScore())<<"\n"; + if ( iUpper!=MapType::end() && &hsiv.getMin().second.getRef() == &iUpper->second ) { + iUpper++; + if ( iUpper!=MapType::end() ) { + hsiv.set(iNext).first = FIRST_INDEX_TO_CHECK; + hsiv.set(iNext).second = SafePtr<const NV> ( iUpper->second ); + typename V::ElementType d = v.getMarginalDistance ( hsiv.get(iNext).first, hsiv.get(iNext).second.getRef() ); + hsiv.set(iNext).setScore() = d; + ////int j = + hsiv.fixDecr(iNext); + ////cerr<<" adding ln"<<&hsiv.get(j).second.getRef()<<" marg-dist="<<d<<" new-score="<<double(hsiv.get(j).getScore())<<" new-pos="<<j<<"\n"; + iNext++; + ////for(int i=0;i<iNext;i++) cerr<<" "<<i<<": ln"<<hsiv.get(i).second.getRef().lineNum.toInt()<<" new-score="<<double(hsiv.get(i).getScore())<<"\n"; + } + } + if ( iLower!=MapType::begin() && &hsiv.getMin().second.getRef() == &iLower->second ) { + iLower--; + hsiv.set(iNext).first = FIRST_INDEX_TO_CHECK; + hsiv.set(iNext).second = SafePtr<const NV> ( iLower->second ); + typename V::ElementType d = v.getMarginalDistance ( hsiv.get(iNext).first, hsiv.get(iNext).second.getRef() ); + hsiv.set(iNext).setScore() = d; + ////int j = + hsiv.fixDecr(iNext); + ////cerr<<" adding ln"<<&hsiv.get(j).second.getRef()<<" marg-dist="<<d<<" new-score="<<double(hsiv.get(j).getScore())<<" new-pos="<<j<<"\n"; + iNext++; + ////for(int i=0;i<iNext;i++) cerr<<" "<<i<<": ln"<<hsiv.get(i).second.getRef().lineNum.toInt()<<" new-score="<<double(hsiv.get(i).getScore())<<"\n"; + } + } + return pair<typename V::ElementType,int> ( hsiv.getMin().getScore(), hsiv.getMin().second.getRef().getNumber() ); +} + + +//////////////////////////////////////////////////////////////////////////////// + +#endif //_NL_ARCHITYPESET_ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-array.h b/contrib/synlm/hhmm/rvtl/include/nl-array.h new file mode 100644 index 000000000..0dfb74b44 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-array.h @@ -0,0 +1,163 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + + +#ifndef _NL_ARRAY__ +#define _NL_ARRAY__ + +#include <cassert> +#include <iostream> +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// + +template<class T> +class Array { + private: + + T* pt; + unsigned int iNextOpen; + unsigned int iCapacity; + + static const T tDummy; + + public: + + class const_iterator; + class iterator : public pair<Array<T>*,int> { + typedef pair<Array<T>*,int> parent; + public: + iterator ( Array<T>* pat, int i ) : parent(pat,i) { } + bool operator== ( const iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator== ( const const_iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator!= ( const iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + bool operator!= ( const const_iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + T* operator-> ( ) { return &parent::first->set(parent::second); } + iterator& operator++ ( ) { ++parent::second; return *this; } + iterator next ( ) { return iterator ( parent::first, parent::second+1 ); } + }; + class const_iterator : public pair<const Array<T>*,int> { + typedef pair<const Array<T>*,int> parent; + public: + const_iterator ( const iterator& it ) : parent(it) { } + const_iterator ( const Array<T>* pat, int i ) : parent(pat,i) { } + bool operator== ( const iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator== ( const const_iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator!= ( const iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + bool operator!= ( const const_iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + const T* operator-> ( ) { return &parent::first->get(parent::second); } + const_iterator& operator++ ( ) { ++parent::second; return *this; } + const_iterator next ( ) { return const_iterator ( parent::first, parent::second+1 ); } + }; + + // Constructor / destructor methods... + Array ( ) : pt(new T[1]), iNextOpen(0), iCapacity(1) { } + Array ( unsigned int i ) : pt(new T[i]), iNextOpen(0), iCapacity(i) { } + Array ( unsigned int i, const T& t ) : pt(new T[i]), iNextOpen(0), iCapacity(i) { for(unsigned int j=0; j<iCapacity; j++) pt[j]=t; } + Array ( const Array<T>& at ) : pt(new T[at.iCapacity]), iNextOpen(at.iNextOpen), iCapacity(at.iCapacity) + { for(unsigned int i=0; i<iNextOpen; i++) pt[i]=at.pt[i]; } + ~Array() { delete[] pt; } + + // Specification methods... + Array<T>& operator= ( const Array<T>& at ) { delete[] pt; pt=new T[at.iCapacity]; iNextOpen=at.iNextOpen; iCapacity=at.iCapacity; + for(unsigned int i=0; i<iNextOpen; i++) pt[i]=at.pt[i]; return *this; } + void clear ( ) { iNextOpen=0; } + void clearEnd ( unsigned int i ) { iNextOpen=i; } + T& add ( ) { return operator[](iNextOpen); } + void addReserve ( unsigned int i ) { ensureCapacity(iNextOpen+i); } + Array<T>& ensureCapacity ( unsigned int i ) { + if ( i>=iCapacity ) { + T* ptTmp = pt; + iCapacity = (i<iCapacity*2) ? iCapacity*2 : i+1; + pt=new T[iCapacity]; + for ( unsigned int j=0; j<iNextOpen; j++ ) + pt[j]=ptTmp[j]; + delete[] ptTmp; + } + return *this; + } + + // Iterator methods... + iterator begin ( ) { return iterator(this,0); } + const_iterator& begin ( ) const { return const_iterator(this,0); } + iterator end ( ) { return iterator(this,iNextOpen); } + const_iterator& end ( ) const { return const_iterator(this,iNextOpen); } + + // Indexing methods... + const T& operator[](unsigned int i) const { return get(i); } + T& operator[](unsigned int i) { return set(i); } + const T& get(unsigned int i) const { if (i>=iCapacity) return tDummy; /*assert(i<iCapacity);*/ return pt[i]; } + T& set(unsigned int i) { ensureCapacity(i); if(i>=iNextOpen)iNextOpen=i+1; return pt[i]; } + + // Aliasing methods... + T* c_array ( ) { return pt; } // NOTE: now it's risky! + const T* c_array ( ) const { return pt; } + + // Extraction methods... + unsigned int size() const { return iNextOpen; } + unsigned int getSize() const { return iNextOpen; } + bool operator== ( const Array<T>& a ) const { + if ( size() != a.size() ) return false; + for ( unsigned int i=0; i<a.size(); i++ ) + if ( get(i) != a.get(i) ) return false; + return true; + } + bool operator!= ( const Array<T>& a ) const { return !(*this==a); } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const Array<T>& at ) { for(unsigned int i=0;i<at.getSize();i++) os<<at.get(i)<<"\n"; return os; } +}; + +template<class T> const T Array<T>::tDummy = T(); + + +//////////////////////////////////////////////////////////////////////////////// + +template<class T> +class SubArray { + private: + T* pat; + unsigned int length; + public: + // Constructor / destructor methods... + SubArray ( ) : pat(NULL), length(0) { } + SubArray ( Array<T>& at ) : pat(&at.set(0)), length(at.size()) { } + SubArray ( Array<T>& at, unsigned int i ) : pat(&at.set(i)), length(at.size()-i) { } + SubArray ( Array<T>& at, unsigned int i, unsigned int j ) : pat(&at.set(i)), length(j-i) { } + SubArray ( SubArray<T>& at, unsigned int i ) : pat(&at.set(i)), length(at.size()-i) { } + SubArray ( SubArray<T>& at, unsigned int i, unsigned int j ) : pat(&at.set(i)), length(j-i) { } + // Indexing methods... + const T& operator[] ( unsigned int i ) const { return get(i); } + T& operator[] ( unsigned int i ) { return set(i); } + const T& get ( unsigned int i ) const { assert(i<length); return pat[i]; } + T& set ( unsigned int i ) { assert(i<length); return pat[i]; } + // Extraction methods... + unsigned int size() const { return length; } + unsigned int getSize() const { return length; } +}; + + +//////////////////////////////////////////////////////////////////////////////// + +#endif //_NL_ARRAY__ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-beam.h b/contrib/synlm/hhmm/rvtl/include/nl-beam.h new file mode 100644 index 000000000..398babe21 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-beam.h @@ -0,0 +1,164 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_BEAM__ +#define _NL_BEAM__ + +#include "nl-heap.h" +#include "nl-hash.h" +//#include <boost/thread/thread.hpp> +//#include <boost/thread/mutex.hpp> +#include <tr1/unordered_map> + +//////////////////////////////////////////////////////////////////////////////// + +/* +template <class R> +class SafePtr { + private: + R* pr; + static R rDummy; + public: + SafePtr<R> ( ) : pr(NULL) { } + SafePtr<R> ( R& r ) : pr(&r) { } + bool operator== ( const SafePtr<R>& spr ) const { return(pr==spr.pr); } + bool operator!= ( const SafePtr<R>& spr ) const { return(!(pr==spr.pr)); } + R& set ( ) { assert(pr); return (pr!=NULL) ? *pr : rDummy; } + const R& get ( ) const { return (pr!=NULL) ? *pr : rDummy; } +}; +template <class R> +R SafePtr<R>::rDummy = R(); + +template <class S, class R> +class ScoredPtr : public SafePtr<R> { + public: + S scr; + ScoredPtr<S,R> ( ) : SafePtr<R>() , scr() { } + ScoredPtr<S,R> ( S s, R& r ) : SafePtr<R>(r), scr(s) { } + S& setScore() { return scr; } + S getScore() const { return scr; } +}; +*/ + +//////////////////////////////////////////////////////////////////////////////// + +template <class S, class C> +class ScoredIter : public C::iterator { + private: + //static C cDummy; + S s; + public: + ScoredIter<S,C> ( ) : C::iterator(0,0), s() { } + ScoredIter<S,C> ( S s1, const typename C::iterator& i1 ) : C::iterator(i1), s(s1) { } + //ScoredIter<S,C> ( ) : C::iterator(cDummy.end()), s() { } + S& setScore() { return s; } + S getScore() const { return s; } +}; +//template <class S, class C> C ScoredIter<S,C>::cDummy; + +//////////////////////////////////////////////////////////////////////////////// + +template <class S,class K,class D> +class Beam { + public: + typedef std::pair<int,D> ID; + typedef std::pair<K,std::pair<int,D> > KID; + typedef std::tr1::unordered_multimap<K,ID,SimpleHashFn<K>,SimpleHashEqual<K> > BeamMap; + typedef MinHeap<ScoredIter<S,BeamMap> > BeamHeap; + private: + BeamMap mkid; + BeamHeap hspkid; + public: + // Constructor methods... + Beam<S,K,D> ( int i ) : mkid(2*i), hspkid(i) { for(int j=0;j<i;j++)set(j,K(),D(),S()); } + // Specification methods... + bool tryAdd ( const K&, const D&, const S& ) ; + void set ( int i, const K& k, const D& d, const S& s ) { hspkid.set(i) = ScoredIter<S,BeamMap>(s,mkid.insert(KID(k,ID(i,d)))); } + // Extraction methods... + const ScoredIter<S,BeamMap>& getMin ( ) const { return hspkid.getMin(); } + const ScoredIter<S,BeamMap>& get ( int i ) const { return hspkid.get(i); } + void sort ( SafeArray1D<Id<int>,std::pair<std::pair<K,D>,S> >& ) ; + void write(FILE *pf){ +/* for (typename BeamMap::const_iterator i = mkid.begin(); i != mkid.end(); i++){ + i->first.write(pf); + fprintf(pf, " %d ", i->second.first); +// i->second.second.write(pf); + fprintf(pf, "\n"); + } +*/ + for(int i=0; i<hspkid.getSize(); i++){ + fprintf(pf, "%d ", hspkid.get(i).getScore().toInt()); + hspkid.get(i)->first.write(pf); + fprintf(pf, "\n"); + } + } +}; + +template <class S,class K,class D> +bool Beam<S,K,D>::tryAdd ( const K& k, const D& d, const S& s ) { + // If score good enough to get into beam... + if ( s > hspkid.getMin().getScore() ) { + typename BeamMap::const_iterator i = mkid.find(k); + // If key in beam already... + if ( i != mkid.end() ) { + // If same key in beam now has better score... + if ( s > hspkid.get(i->second.first).getScore() ) { + // Update score (and data associated with that score)... + hspkid.set(i->second.first).setScore() = s; + hspkid.set(i->second.first)->second.second = d; + // Update heap... + int iStart = i->second.first; int iDeeper = hspkid.fixIncr(iStart); + // Fix pointers in hash... + for ( int j = iDeeper+1; j>=iStart+1; j/=2 ) hspkid.set(j-1)->second.first = j-1; + } + } + // If x not in beam already, add... + else { + // Remove min from map (via pointer in heap)... + mkid.erase ( hspkid.getMin() ); + // Insert new entry at min... + set(0,k,d,s); + // Update heap... + int iStart = 0; int iDeeper = hspkid.fixIncr(iStart); + // Fix pointers in hash... + for ( int j = iDeeper+1; j>=iStart+1; j/=2 ) hspkid.set(j-1)->second.first = j-1; + } + } + return ( LogProb() != hspkid.getMin().getScore() ); // true = beam full, false = beam still has gaps +} + +template <class S,class K,class D> +void Beam<S,K,D>::sort ( SafeArray1D<Id<int>,std::pair<std::pair<K,D>,S> >& akdsOut ) { + for ( int i=0; i<hspkid.getSize(); i++ ) { + akdsOut.set(hspkid.getSize()-i-1).first.first = hspkid.getMin()->first; // copy min key to output key. + akdsOut.set(hspkid.getSize()-i-1).first.second = hspkid.getMin()->second.second; // copy min dat to output dat. + akdsOut.set(hspkid.getSize()-i-1).second = hspkid.getMin().getScore(); // copy min scr to output scr. + hspkid.setMin().setScore() = LogProb(1); // get min out of the way. + hspkid.fixIncr(0); // repair heap. + } +} + +//////////////////////////////////////////////////////////////////////////////// + + +#endif //_NL_BEAM__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-const.h b/contrib/synlm/hhmm/rvtl/include/nl-const.h new file mode 100644 index 000000000..3d6023f39 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-const.h @@ -0,0 +1,125 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __NL_CONST_H_ +#define __NL_CONST_H_ + +//#include <getopt.h> + +/////////////////////////////////////////////////////////////////////////////// +// type defs... + +typedef short int16 ; +typedef int16 Sample ; +//typedef int Mag ; +//typedef int Gam ; + +/////////////////////////////////////////////////////////////////////////////// +// Misc consts... + +int max(int i,int j) {return (i>j)?i:j;} +int min(int i,int j) {return (i<j)?i:j;} + +inline size_t rotLeft (const size_t& n, const size_t& i) { return (n << i) | (n >> (sizeof(size_t) - i)); } +inline size_t rotRight(const size_t& n, const size_t& i) { return (n >> i) | (n << (sizeof(size_t) - i)); } + +//inline float abs ( float a ) { return (a>=0)?a:-a; } +/* + +/////////////////////////////////////////////////////////////////////////////// +// Basic phone recognition consts... +static const int NUM_SAMPLES_PER_FRAME = 512; +#ifdef OLD_Q +static const int FRAME_RATE_IN_SAMPLES = 160; //// 80 +#else +static const int FRAME_RATE_IN_SAMPLES = 256; //// 160; //// 80 +#endif + +static const int LOG_NUM_FREQUENCIES = 8; +static const int LOG_NUM_QUEFRENCIES = LOG_NUM_FREQUENCIES; +static const int NUM_FREQUENCIES = 1<<LOG_NUM_FREQUENCIES; +static const int NUM_QUEFRENCIES = 1<<LOG_NUM_QUEFRENCIES; + +/////////////////////////////////////////////////////////////////////////////// +// Output format globals +static bool OUTPUT_QUIET = false; + +/////////////////////////////////////////////////////////////////////////////// +// H/O consts... + +static int LOG_MAX_SIGNS = 13; // NOTE: bit limit: LOG_MAX_SIGNS + 3*LOG_MAX_ENTS < 31 +static int MAX_SIGNS = 1<<LOG_MAX_SIGNS; +static int MAX_IVS = 100; + +/////////////////////////////////////////////////////////////////////////////// +// H sign recognition consts... + +static double INSERT_PENALTY = 1.0; // MULTIPLICATIVE +static int MAX_FANOUT = 150; +static const int MAX_BOOLS = 2; +static const int MAX_TRUTHVALS = 3; + +/////////////////////////////////////////////////////////////////////////////// +// H sem recognition consts... + +static int LOG_MAX_ENTS = 6; +static int MAX_ENTS = 1<<LOG_MAX_ENTS; +static int MAX_CONTEXTS = 100; +static int MAX_RELNS = 100; +static int MAX_CATS = 1000; + +/////////////////////////////////////////////////////////////////////////////// +// Reader consts... + +static int MAX_READER_FIELDS = 50; //62442; //20; +static int LENGTH_READER_FIELDS = 1024; //512; //256; + +/////////////////////////////////////////////////////////////////////////////// +// HMM consts... + +//static const int BEAM_WIDTH = 4095; +static int BEAM_WIDTH = 63; //255; +//static const int BEAM_WIDTH = 1023; + +/////////////////////////////////////////////////////////////////////////////// + +static const int NUM_MFCC_FILTERS = 40; +static const int NUM_CEPSTRUM = 13; +static const int WEIGHT_SIZE = 8; +static const int MFCC_SIZE = 3 * NUM_CEPSTRUM; +static const float MIN_FREQUENCY = 0; //130.0; +static const float MAX_FREQUENCY = 8000.0; //Max allowed freq in signal is 16000Hz +static const int MEAN_SIZE = (WEIGHT_SIZE * MFCC_SIZE); +//Use a diagonal matrix for now +//static const int COVARIANCE_SIZE = (MEAN_SIZE * MFCC_SIZE); +static const int COVARIANCE_SIZE = MEAN_SIZE; +static const int MAX_NUM_FRAMES = 10000; +static const float PREEMPASIZE_FACTOR = 0.97; +static const int NUM_FFT_POINTS = NUM_SAMPLES_PER_FRAME; +static const int SAMPLING_RATE = 16000; + +static const bool DEBUG_MODE = false; + +*/ + +#endif /*__NL_CONST_H_*/ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-cpt.h b/contrib/synlm/hhmm/rvtl/include/nl-cpt.h new file mode 100644 index 000000000..67a1a1021 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-cpt.h @@ -0,0 +1,1332 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_CPT__ +#define _NL_CPT__ + +//#include <stdlib.h> +//#include <vector> +//#include <string> +//#include <cassert> +//using namespace std; +//#include "nl-string.h" +//#include "nl-safeids.h" +//#include "nl-stringindex.h" +#include "nl-randvar.h" +//#include "nl-probmodel.h" +#include "nl-hash.h" +//#include <tr1/unordered_map> +//using namespace tr1; + +#include <netinet/in.h> + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// psNil +// +//////////////////////////////////////////////////////////////////////////////// + +char psNil[] = ""; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Unit +// +//////////////////////////////////////////////////////////////////////////////// + +class Unit { + public: + void write(FILE* pf)const{} + size_t getHashKey ( ) const { return 0; } + bool operator== ( const Unit& u ) const { return true; } + bool operator< ( const Unit& u ) const { return false; } + friend ostream& operator<< ( ostream& os, const Unit& u ) { return os; } + friend String& operator<< ( String& str, const Unit& u ) { return str; } + friend IStream operator>> ( pair<IStream,Unit*> si_m, const char* psD ) { return si_m.first; } + + // OBSOLETE! + friend pair<StringInput,Unit*> operator>> ( StringInput si, Unit& m ) { return pair<StringInput,Unit*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,Unit*> si_m, const char* psD ) { return si_m.first; } +}; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// MapKeyXD +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////// +template<class X1> +class MapKey1D { + private: + X1 x1; + public: + const X1& getX1() const { return x1; } + size_t getHashKey ( ) const { return x1.getHashKey(); } + MapKey1D ( ) { } + MapKey1D ( const X1& a1 ) { x1=a1; } + bool operator== ( const MapKey1D<X1>& h ) const { return(x1==h.x1); } + bool operator< ( const MapKey1D<X1>& h ) const { return(x1< h.x1); } + friend ostream& operator<< ( ostream& os, const MapKey1D<X1>& k ) { return os<<k.x1; } + friend String& operator<< ( String& str, const MapKey1D<X1>& k ) { return str<<k.x1; } + friend IStream operator>> ( pair<IStream,MapKey1D<X1>*> is_m, const char* psD ) { + //MapKey1D<X1>& m = *is_m.second; + return is_m.first>>is_m.second->x1>>psD; + } + + // OBSOLETE! + friend pair<StringInput,MapKey1D<X1>*> operator>> ( StringInput si, MapKey1D<X1>& m ) { + return pair<StringInput,MapKey1D<X1>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,MapKey1D<X1>*> si_m, const char* psD ) { + MapKey1D<X1>& m = *si_m.second; + return si_m.first>>m.x1>>psD; + } +}; + +//////////////////// +template<class X1, class X2> +class MapKey2D { + private: + X1 x1; + X2 x2; + public: + const X1& getX1() const { return x1; } + const X2& getX2() const { return x2; } + size_t getHashKey ( ) const { size_t k=rotLeft(x1.getHashKey(),3); k^=x2.getHashKey(); return k; } + MapKey2D ( ) { } + MapKey2D ( const X1& a1, const X2& a2 ) { x1=a1; x2=a2; } + bool operator== ( const MapKey2D<X1,X2>& h ) const { return(x1==h.x1 && x2==h.x2); } + bool operator< ( const MapKey2D<X1,X2>& h ) const { + return ( (x1<h.x1) || + (x1==h.x1 && x2<h.x2) ); + } + friend ostream& operator<< ( ostream& os, const MapKey2D<X1,X2>& k ) { return os<<k.x1<<" "<<k.x2; } + friend String& operator<< ( String& str, const MapKey2D<X1,X2>& k ) { return str<<k.x1<<" "<<k.x2; } + friend IStream operator>> ( pair<IStream,MapKey2D<X1,X2>*> is_m, const char* psD ) { + MapKey2D<X1,X2>& m = *is_m.second; IStream is2; + IStream is=is_m.first>>m.x1>>" "; + while((is2=is>>" ")!=IStream())is=is2; + return is>>m.x2>>psD; + } + + // OBSOLETE! + friend pair<StringInput,MapKey2D<X1,X2>*> operator>> ( StringInput si, MapKey2D<X1,X2>& m ) { + return pair<StringInput,MapKey2D<X1,X2>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,MapKey2D<X1,X2>*> si_m, const char* psD ) { + MapKey2D<X1,X2>& m = *si_m.second; StringInput si2; + StringInput si=si_m.first>>m.x1>>" "; + while((si2=si>>" ")!=NULL)si=si2; + return si>>m.x2>>psD; + } +}; + +//////////////////// +template<class X1, class X2, class X3> +class MapKey3D { + private: + X1 x1; + X2 x2; + X3 x3; + public: + const X1& getX1() const { return x1; } + const X2& getX2() const { return x2; } + const X3& getX3() const { return x3; } + size_t getHashKey ( ) const { size_t k=rotLeft(x1.getHashKey(),3); k=rotLeft(k^x2.getHashKey(),3); k^=x3.getHashKey(); return k; } + MapKey3D ( ) { } + MapKey3D ( const X1& a1, const X2& a2, const X3& a3 ) { x1=a1; x2=a2; x3=a3; } + bool operator== ( const MapKey3D<X1,X2,X3>& h ) const { return(x1==h.x1 && x2==h.x2 && x3==h.x3); } + bool operator< ( const MapKey3D<X1,X2,X3>& h ) const { + return ( (x1<h.x1) || + (x1==h.x1 && x2<h.x2) || + (x1==h.x1 && x2==h.x2 && x3<h.x3) ); + } + friend ostream& operator<< ( ostream& os, const MapKey3D<X1,X2,X3>& k ) { return os<<k.x1<<" "<<k.x2<<" "<<k.x3; } + friend String& operator<< ( String& str, const MapKey3D<X1,X2,X3>& k ) { return str<<k.x1<<" "<<k.x2<<" "<<k.x3; } + friend IStream operator>> ( pair<IStream,MapKey3D<X1,X2,X3>*> is_m, const char* psD ) { + MapKey3D<X1,X2,X3>& m = *is_m.second; IStream is2; + IStream is=is_m.first>>m.x1>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x2>>" "; + while((is2=is>>" ")!=IStream())is=is2; + return is>>m.x3>>psD; + } + + // OBSOLETE! + friend pair<StringInput,MapKey3D<X1,X2,X3>*> operator>> ( StringInput si, MapKey3D<X1,X2,X3>& m ) { + return pair<StringInput,MapKey3D<X1,X2,X3>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,MapKey3D<X1,X2,X3>*> si_m, const char* psD ) { + MapKey3D<X1,X2,X3>& m = *si_m.second; StringInput si2; + StringInput si=si_m.first>>m.x1>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x2>>" "; + while((si2=si>>" ")!=NULL)si=si2; + return si>>m.x3>>psD; + } +}; + +//////////////////// +template<class X1, class X2, class X3, class X4> +class MapKey4D { + private: + X1 x1; + X2 x2; + X3 x3; + X4 x4; + public: + const X1& getX1() const { return x1; } + const X2& getX2() const { return x2; } + const X3& getX3() const { return x3; } + const X4& getX4() const { return x4; } + size_t getHashKey ( ) const { size_t k=rotLeft(x1.getHashKey(),3); k=rotLeft(k^x2.getHashKey(),3); k=rotLeft(k^x3.getHashKey(),3); k^=x4.getHashKey(); return k; } + MapKey4D ( ) { } + MapKey4D ( const X1& a1, const X2& a2, const X3& a3, const X4& a4 ) { x1=a1; x2=a2; x3=a3; x4=a4; } + bool operator== ( const MapKey4D<X1,X2,X3,X4>& h ) const { return(x1==h.x1 && x2==h.x2 && x3==h.x3 && x4==h.x4); } + bool operator< ( const MapKey4D<X1,X2,X3,X4>& h ) const { + return ( (x1<h.x1) || + (x1==h.x1 && x2<h.x2) || + (x1==h.x1 && x2==h.x2 && x3<h.x3) || + (x1==h.x1 && x2==h.x2 && x3==h.x3 && x4<h.x4) ); + } + friend ostream& operator<< ( ostream& os, const MapKey4D<X1,X2,X3,X4>& k ) { return os<<k.x1<<" "<<k.x2<<" "<<k.x3<<" "<<k.x4; } + friend String& operator<< ( String& str, const MapKey4D<X1,X2,X3,X4>& k ) { return str<<k.x1<<" "<<k.x2<<" "<<k.x3<<" "<<k.x4; } + friend IStream operator>> ( pair<IStream,MapKey4D<X1,X2,X3,X4>*> is_m, const char* psD ) { + MapKey4D<X1,X2,X3,X4>& m = *is_m.second; IStream is2; + IStream is=is_m.first>>m.x1>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x2>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x3>>" "; + while((is2=is>>" ")!=IStream())is=is2; + return is>>m.x4>>psD; + } + + // OBSOLETE! + friend pair<StringInput,MapKey4D<X1,X2,X3,X4>*> operator>> ( StringInput si, MapKey4D<X1,X2,X3,X4>& m ) { + return pair<StringInput,MapKey4D<X1,X2,X3,X4>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,MapKey4D<X1,X2,X3,X4>*> si_m, const char* psD ) { + MapKey4D<X1,X2,X3,X4>& m = *si_m.second; StringInput si2; + StringInput si=si_m.first>>m.x1>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x2>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x3>>" "; + while((si2=si>>" ")!=NULL)si=si2; + return si>>m.x4>>psD; + } +}; + +//////////////////// +template<class X1, class X2, class X3, class X4, class X5> +class MapKey5D { + private: + X1 x1; + X2 x2; + X3 x3; + X4 x4; + X5 x5; + public: + const X1& getX1() const { return x1; } + const X2& getX2() const { return x2; } + const X3& getX3() const { return x3; } + const X4& getX4() const { return x4; } + const X5& getX5() const { return x5; } + size_t getHashKey ( ) const { size_t k=rotLeft(x1.getHashKey(),3); k=rotLeft(k^x2.getHashKey(),3); + k=rotLeft(k^x3.getHashKey(),3); k=rotLeft(k^x4.getHashKey(),3); k^=x5.getHashKey(); return k; } + MapKey5D ( ) { } + MapKey5D ( const X1& a1, const X2& a2, const X3& a3, const X4& a4, const X5& a5 ) { x1=a1; x2=a2; x3=a3; x4=a4; x5=a5; } + bool operator== ( const MapKey5D<X1,X2,X3,X4,X5>& h ) const { return(x1==h.x1 && x2==h.x2 && x3==h.x3 && x4==h.x4 && x5==h.x5); } + bool operator< ( const MapKey5D<X1,X2,X3,X4,X5>& h ) const { + return ( (x1<h.x1) || + (x1==h.x1 && x2<h.x2) || + (x1==h.x1 && x2==h.x2 && x3<h.x3) || + (x1==h.x1 && x2==h.x2 && x3==h.x3 && x4<h.x4) || + (x1==h.x1 && x2==h.x2 && x3==h.x3 && x4==h.x4 && x5<h.x5) ); + } + friend ostream& operator<< ( ostream& os, const MapKey5D<X1,X2,X3,X4,X5>& k ) { return os<<k.x1<<" "<<k.x2<<" "<<k.x3<<" "<<k.x4<<" "<<k.x5; } + friend String& operator<< ( String& str, const MapKey5D<X1,X2,X3,X4,X5>& k ) { return str<<k.x1<<" "<<k.x2<<" "<<k.x3<<" "<<k.x4<<" "<<k.x5; } + friend IStream operator>> ( pair<IStream,MapKey5D<X1,X2,X3,X4,X5>*> is_m, const char* psD ) { + MapKey5D<X1,X2,X3,X4,X5>& m = *is_m.second; IStream is2; + IStream is=is_m.first>>m.x1>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x2>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x3>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x4>>" "; + while((is2=is>>" ")!=IStream())is=is2; + return is>>m.x5>>psD; + } + + // OBSOLETE! + friend pair<StringInput,MapKey5D<X1,X2,X3,X4,X5>*> operator>> ( StringInput si, MapKey5D<X1,X2,X3,X4,X5>& m ) { + return pair<StringInput,MapKey5D<X1,X2,X3,X4,X5>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,MapKey5D<X1,X2,X3,X4,X5>*> si_m, const char* psD ) { + MapKey5D<X1,X2,X3,X4,X5>& m = *si_m.second; StringInput si2; + StringInput si=si_m.first>>m.x1>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x2>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x3>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x4>>" "; + while((si2=si>>" ")!=NULL)si=si2; + return si>>m.x5>>psD; + } +}; + +//////////////////// +template<class X1, class X2, class X3, class X4, class X5, class X6> +class MapKey6D { + private: + X1 x1; + X2 x2; + X3 x3; + X4 x4; + X5 x5; + X6 x6; + public: + const X1& getX1() const { return x1; } + const X2& getX2() const { return x2; } + const X3& getX3() const { return x3; } + const X4& getX4() const { return x4; } + const X5& getX5() const { return x5; } + const X5& getX6() const { return x6; } + size_t getHashKey ( ) const { size_t k=rotLeft(x1.getHashKey(),3); k=rotLeft(k^x2.getHashKey(),3); + k=rotLeft(k^x3.getHashKey(),3); k=rotLeft(k^x4.getHashKey(),3); k=rotLeft(k^x5.getHashKey(),3); k^=x6.getHashKey(); return k; } + MapKey6D ( ) { } + MapKey6D ( const X1& a1, const X2& a2, const X3& a3, const X4& a4, const X5& a5, const X6& a6 ) { x1=a1; x2=a2; x3=a3; x4=a4; x5=a5; x6=a6; } + bool operator== ( const MapKey6D<X1,X2,X3,X4,X5,X6>& h ) const { return(x1==h.x1 && x2==h.x2 && x3==h.x3 && x4==h.x4 && x5==h.x5 && x6==h.x6); } + bool operator< ( const MapKey6D<X1,X2,X3,X4,X5,X6>& h ) const { + return ( (x1<h.x1) || + (x1==h.x1 && x2<h.x2) || + (x1==h.x1 && x2==h.x2 && x3<h.x3) || + (x1==h.x1 && x2==h.x2 && x3==h.x3 && x4<h.x4) || + (x1==h.x1 && x2==h.x2 && x3==h.x3 && x4==h.x4 && x5<h.x5) || + (x1==h.x1 && x2==h.x2 && x3==h.x3 && x4==h.x4 && x5==h.x5 && x6<h.x6) ); + } + friend ostream& operator<< ( ostream& os, const MapKey6D<X1,X2,X3,X4,X5,X6>& k ) { return os<<k.x1<<" "<<k.x2<<" "<<k.x3<<" "<<k.x4<<" "<<k.x5<<" "<<k.x6; } + friend String& operator<< ( String& str, const MapKey6D<X1,X2,X3,X4,X5,X6>& k ) { return str<<k.x1<<" "<<k.x2<<" "<<k.x3<<" "<<k.x4<<" "<<k.x5<<" "<<k.x6; } + friend IStream operator>> ( pair<IStream,MapKey6D<X1,X2,X3,X4,X5,X6>*> is_m, const char* psD ) { + MapKey6D<X1,X2,X3,X4,X5,X6>& m = *is_m.second; IStream is2; + IStream is=is_m.first>>m.x1>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x2>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x3>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x4>>" "; + while((is2=is>>" ")!=IStream())is=is2; + is=is>>m.x5>>" "; + while((is2=is>>" ")!=IStream())is=is2; + return is>>m.x6>>psD; + } + + // OBSOLETE! + friend pair<StringInput,MapKey6D<X1,X2,X3,X4,X5,X6>*> operator>> ( StringInput si, MapKey6D<X1,X2,X3,X4,X5,X6>& m ) { + return pair<StringInput,MapKey6D<X1,X2,X3,X4,X5,X6>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,MapKey6D<X1,X2,X3,X4,X5,X6>*> si_m, const char* psD ) { + MapKey6D<X1,X2,X3,X4,X5,X6>& m = *si_m.second; StringInput si2; + StringInput si=si_m.first>>m.x1>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x2>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x3>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x4>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>m.x5>>" "; + while((si2=si>>" ")!=NULL)si=si2; + return si>>m.x6>>psD; + } +}; + +// Declare random access conditional probability tables (un-iteratable) +#include "nl-racpt.h" + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// Multimap CPTXDModel +// +//////////////////////////////////////////////////////////////////////////////// + +/* +template<class II,class V> +class BaseIterVal : public V { + public: + static const int NUM_ITERS = 1; + II iter; + BaseIterVal ( ) : V(), iter(typename II::first_type(0,0),typename II::second_type(0,0)) { } + BaseIterVal ( II ii ) : V((ii.first!=ii.second)?ii.first->first:V()), iter(ii) { } + BaseIterVal ( V v ) : V(v), iter(typename II::first_type(0,0),typename II::second_type(0,0)) { } + BaseIterVal& operator++ ( int ) { V& v=*this; iter.first++; if(iter.first!=iter.second)v=iter.first->first; return *this; } +}; +*/ + +//////////////////////////////////////////////////////////////////////////////// + + +template<class X, class Y> +class SimpleMap : public map<X,Y> { + private: + typedef map<X,Y> OrigMap; + static const Y yDummy; + + public: + // Constructor / destructor methods... + SimpleMap ( ) : OrigMap() { } + SimpleMap ( int i ) : OrigMap() { } + SimpleMap (const SimpleMap& s) : OrigMap(s) { } + // Specification methods... + Y& set ( const X& x ) { return OrigMap::operator[](x); } + // Extraction methods... + const Y& get ( const X& x ) const { return (OrigMap::end()!=OrigMap::find(x)) ? OrigMap::find(x)->second : yDummy; } + bool contains ( const X& x ) const { return (OrigMap::end()!=OrigMap::find(x)); } + friend ostream& operator<< ( ostream& os, const SimpleMap<X,Y>& h ) { + for ( typename SimpleMap<X,Y>::const_iterator it=h.begin(); it!=h.end(); it++ ) + os<<((it==h.begin())?"":",")<<it->first<<":"<<it->second; + return os; + } +}; +template<class X, class Y> const Y SimpleMap<X,Y>::yDummy = Y(); + + +//////////////////////////////////////////////////////////////////////////////// + +#define MAP_CONTAINER SimpleMap +//#define MAP_CONTAINER SimpleHash + +template<class Y, class K, class P, const char* psLbl=psNil> +class GenericCPTModel : public MAP_CONTAINER<K,MAP_CONTAINER<Y,P> > { + private: + typedef MAP_CONTAINER<K,MAP_CONTAINER<Y,P> > HKYP; + typedef typename MAP_CONTAINER<Y,P>::const_iterator IYP; + //HKYP h; + + public: + typedef Y RandVarType; + typedef Y RVType; +/* typedef BaseIterVal<std::pair<IYP,IYP>,Y> IterVal; */ + typedef MAP_CONTAINER<Y,P> distribution; + +/* bool setFirst ( IterVal& ikyp, const K& k ) const { */ +/* const MAP_CONTAINER<Y,P>& hyp = MAP_CONTAINER<K,MAP_CONTAINER<Y,P> >::get(k); */ +/* ikyp = std::pair<IYP,IYP>(hyp.begin(),hyp.end()); */ +/* return ( ikyp.iter.first != ikyp.iter.second ); */ +/* } */ +/* bool setNext ( IterVal& ikyp, const K& k ) const { */ +/* if ( ikyp.iter.first != ikyp.iter.second ) ikyp++; */ +/* return ( ikyp.iter.first != ikyp.iter.second ); */ +/* } */ + bool contains ( const Y& y, const K& k ) const { + return ( MAP_CONTAINER<K,MAP_CONTAINER<Y,P> >::get(k).contains(y) ); + } + bool contains ( const K& k ) const { + return ( MAP_CONTAINER<K,MAP_CONTAINER<Y,P> >::contains(k) ); + } +/* P getProb ( const IterVal& ikyp, const K& k ) const { */ +/* if ( ikyp.iter.first == ikyp.iter.second ) { cerr<<"ERROR: no iterator to fix probability: "<<k<<endl; return P(); } */ +/* return ( ikyp.iter.first->second ); */ +/* } */ + P getProb ( const Y& y, const K& k ) const { + return MAP_CONTAINER<K,MAP_CONTAINER<Y,P> >::get(k).get(y); + } + const MAP_CONTAINER<Y,P>& getDist ( const K& k ) const { + return MAP_CONTAINER<K,MAP_CONTAINER<Y,P> >::get(k); + } + P& setProb ( const Y& y, const K& k ) { + return MAP_CONTAINER<K,MAP_CONTAINER<Y,P> >::set(k).set(y); + } + void normalize ( ) { + for ( typename HKYP::iterator ik=HKYP::begin(); ik!=HKYP::end(); ik++ ) { + K k=ik->first; + P p=P(); + for ( typename distribution::iterator itd = ik->second.begin(); itd != ik->second.end(); ++itd ) + p += itd->second; + if (p!=P()) + for ( typename distribution::iterator itd = ik->second.begin(); itd != ik->second.end(); ++itd ) + itd->second /= p; + } + } +/* void transmit ( int tSockfd, const char* psId ) const { */ +/* for ( typename HKYP::const_iterator ik=HKYP::begin(); ik!=HKYP::end(); ik++ ) { */ +/* K k=ik->first; */ +/* IterVal y; */ +/* // For each non-zero probability in model... */ +/* for ( bool b=setFirst(y,k); b; b=setNext(y,k) ) { */ +/* //if ( getProb(y,k) != P() ) { */ +/* String str(1000); */ +/* str<<psId<<" "<<k<<" : "<<y<<" = "<<getProb(y,k).toDouble()<<"\n"; */ +/* if ( send(tSockfd,str.c_array(),str.size()-1,MSG_EOR) != int(str.size()-1) ) */ +/* {cerr<<"ERROR writing to socket\n";exit(0);} */ +/* //} */ +/* } */ +/* } */ +/* } */ + void dump ( ostream& os, const char* psId ) const { + for ( typename HKYP::const_iterator ik=HKYP::begin(); ik!=HKYP::end(); ik++ ) { + K k=ik->first; + const distribution& dist = ik->second; + for ( typename distribution::const_iterator itd = dist.begin(); itd != dist.end(); ++itd ) { + const Y& y = itd->first; + os<<psId<<" "<<k<<" : "<<y<<" = "<<getProb(y,k).toDouble()<<"\n"; + } + } + } +/* void subsume ( GenericCPTModel<Y,K,P>& m ) { */ +/* for ( typename HKYP::const_iterator ik=m.HKYP::begin(); ik!=m.HKYP::end(); ik++ ) { */ +/* K k=ik->first; */ +/* IterVal y; */ +/* for ( bool b=m.setFirst(y,k); b; b=m.setNext(y,k) ) */ +/* setProb(y,k) = m.getProb(y,k); */ +/* } */ +/* } */ + void clear ( ) { MAP_CONTAINER<K,MAP_CONTAINER<Y,P> >::clear(); } + + //friend pair<IStream,GenericCPTModel<Y,K,P>*> operator>> ( IStream is, GenericCPTModel<Y,K,P>& m ) { + // return pair<IStream,GenericCPTModel<Y,K,P>*>(is,&m); } + friend IStream operator>> ( pair<IStream,GenericCPTModel<Y,K,P,psLbl>*> is_m, const char* psD ) { + Y y; K k; IStream is,is1; GenericCPTModel<Y,K,P,psLbl>& m = *is_m.second; + is=is_m.first; + if ( is==IStream() ) return is; + is=is>>psLbl; + while((is1=is>>" ")!=IStream())is=is1; + ////cerr<<"reading k...\n"; + is=is>>k>>" "; + ////cerr<<" ...k='"<<k<<"'\n"; + while((is1=is>>" ")!=IStream())is=is1; + is=is>>": "; + while((is1=is>>" ")!=IStream())is=is1; + ////cerr<<"reading y...\n"; + is=is>>y>>" "; + ////cerr<<" ...y='"<<y<<"'\n"; + while((is1=is>>" ")!=IStream())is=is1; + is=is>>"= "; + while((is1=is>>" ")!=IStream())is=is1; + ////cerr<<"reading pr...\n"; + return (is!=IStream()) ? is>>m.setProb(y,k)>>psD : is; + } + + // OBSOLETE! + friend pair<StringInput,GenericCPTModel<Y,K,P,psLbl>*> operator>> ( StringInput si, GenericCPTModel<Y,K,P,psLbl>& m ) { + return pair<StringInput,GenericCPTModel<Y,K,P,psLbl>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,GenericCPTModel<Y,K,P,psLbl>*> si_m, const char* psD ) { + Y y; K k; StringInput si,si2; GenericCPTModel<Y,K,P,psLbl>& m = *si_m.second; + si=si_m.first; + if ( si==NULL ) return si; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>k>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>": "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>y>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>"= "; + while((si2=si>>" ")!=NULL)si=si2; + return (si!=NULL) ? si>>m.setProb(y,k)>>psD : si; + } +}; + + +//////////////////////////////////////////////////////////////////////////////// + +//////////////////// +template<class Y, class P, const char* psLbl=psNil> +class CPT1DModel : public GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl> { + typedef GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl> ParentType; + typedef CPT1DModel<Y,P,psLbl> ThisType; + public: +/* typedef typename GenericCPTModel<Y,MapKey1D<Unit>,P>::IterVal IterVal; */ + +/* bool setFirst ( IterVal& ixyp ) const { */ +/* return GenericCPTModel<Y,MapKey1D<Unit>,P>::setFirst ( ixyp, MapKey1D<Unit>(Unit()) ); */ +/* } */ +/* bool setNext ( IterVal& ixyp ) const { */ +/* return GenericCPTModel<Y,MapKey1D<Unit>,P>::setNext ( ixyp, MapKey1D<Unit>(Unit()) ); */ +/* } */ + bool contains ( const Y& y ) const { + return GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl>::contains ( y, MapKey1D<Unit>(Unit()) ); + } +/* P getProb ( const IterVal& ixyp ) const { */ +/* return GenericCPTModel<Y,MapKey1D<Unit>,P>::getProb ( ixyp, MapKey1D<Unit>(Unit()) ); */ +/* } */ + P getProb ( const Y& y ) const { + return GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl>::getProb ( y, MapKey1D<Unit>(Unit()) ); + } + const typename GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl>::distribution& getDist ( ) const { + return GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl>::get ( MapKey1D<Unit>(Unit()) ); + } + P& setProb ( const Y& y ) { + return GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl>::setProb ( y, MapKey1D<Unit>(Unit()) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 3==aps.size() ) { + GenericCPTModel<Y,MapKey1D<Unit>,P,psLbl>::setProb ( Y(aps[1]), MapKey1D<Unit>(Unit()) ) = atof(aps[2]); + return true; + } + return false; + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + + +//////////////////// +template<class Y, class X1, class P, const char* psLbl=psNil> +class CPT2DModel : public GenericCPTModel<Y,MapKey1D<X1>,P,psLbl> { + typedef GenericCPTModel<Y,MapKey1D<X1>,P,psLbl> ParentType; + typedef CPT2DModel<Y,X1,P,psLbl> ThisType; + public: +/* typedef typename GenericCPTModel<Y,MapKey1D<X1>,P>::IterVal IterVal; */ + +/* // This stuff only for deterministic 'Determ' models... */ +/* typedef X1 Dep1Type; */ +/* typedef P ProbType; */ +/* bool hasDeterm ( const X1& x1 ) { IterVal y; bool b=setFirst(y,x1); return b; } */ +/* Y getDeterm ( const X1& x1 ) { IterVal y; bool b=setFirst(y,x1); if(!b)cerr<<"ERROR: determ case missing: "<<x1.getString()<<endl; return y; } */ + +/* bool setFirst ( IterVal& ixyp, const X1& x1 ) const { */ +/* return GenericCPTModel<Y,MapKey1D<X1>,P>::setFirst ( ixyp, MapKey1D<X1>(x1) ); */ +/* } */ +/* bool setNext ( IterVal& ixyp, const X1& x1 ) const { */ +/* return GenericCPTModel<Y,MapKey1D<X1>,P>::setNext ( ixyp, MapKey1D<X1>(x1) ); */ +/* } */ + bool contains ( const Y& y, const X1& x1 ) const { + return GenericCPTModel<Y,MapKey1D<X1>,P,psLbl>::contains ( y, MapKey1D<X1>(x1) ); + } + bool contains ( const X1& x1 ) const { + return GenericCPTModel<Y,MapKey1D<X1>,P,psLbl>::contains ( MapKey1D<X1>(x1) ); + } +/* P getProb ( const IterVal& ixyp, const X1& x1 ) const { */ +/* return GenericCPTModel<Y,MapKey1D<X1>,P>::getProb ( ixyp, MapKey1D<X1>(x1) ); */ +/* } */ + P getProb ( const Y& y, const X1& x1 ) const { + return GenericCPTModel<Y,MapKey1D<X1>,P,psLbl>::getProb ( y, MapKey1D<X1>(x1) ); + } + const typename GenericCPTModel<Y,MapKey1D<X1>,P,psLbl>::distribution& getDist ( const X1& x1 ) const { + return GenericCPTModel<Y,MapKey1D<X1>,P,psLbl>::get ( MapKey1D<X1>(x1) ); + } + P& setProb ( const Y& y, const X1& x1 ) { + return GenericCPTModel<Y,MapKey1D<X1>,P,psLbl>::setProb ( y, MapKey1D<X1>(x1) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 4==aps.size() ) { + GenericCPTModel<Y,MapKey1D<X1>,P,psLbl>::setProb ( Y(aps[2]), MapKey1D<X1>(aps[1]) ) = atof(aps[3]); + return true; + } + return false; + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + + +//////////////////// +template<class Y, class X1, class X2, class P, const char* psLbl=psNil> +class CPT3DModel : public GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl> { + typedef GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl> ParentType; + typedef CPT3DModel<Y,X1,X2,P,psLbl> ThisType; + public: +/* typedef typename GenericCPTModel<Y,MapKey2D<X1,X2>,P>::IterVal IterVal; */ + +/* bool setFirst ( IterVal& ixyp, const X1& x1, const X2& x2 ) const { */ +/* return GenericCPTModel<Y,MapKey2D<X1,X2>,P>::setFirst ( ixyp, MapKey2D<X1,X2>(x1,x2) ); */ +/* } */ +/* bool setNext ( IterVal& ixyp, const X1& x1, const X2& x2 ) const { */ +/* return GenericCPTModel<Y,MapKey2D<X1,X2>,P>::setNext ( ixyp, MapKey2D<X1,X2>(x1,x2) ); */ +/* } */ + bool contains ( const Y& y, const X1& x1, const X2& x2 ) const { + return GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::contains ( y, MapKey2D<X1,X2>(x1,x2) ); + } + bool contains ( const X1& x1, const X2& x2 ) const { + return GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::contains ( MapKey2D<X1,X2>(x1,x2) ); + } +/* P getProb ( const IterVal& ixyp, const X1& x1, const X2& x2 ) const { */ +/* return GenericCPTModel<Y,MapKey2D<X1,X2>,P>::getProb ( ixyp, MapKey2D<X1,X2>(x1,x2) ); */ +/* } */ + P getProb ( const Y& y, const X1& x1, const X2& x2 ) const { + return GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::getProb ( y, MapKey2D<X1,X2>(x1,x2) ); + } + const typename GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::distribution& getDist ( const X1& x1, const X2& x2 ) const { + return GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::get ( MapKey2D<X1,X2>(x1,x2) ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2 ) { + return GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::setProb ( y, MapKey2D<X1,X2>(x1,x2) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 5==aps.size() ) { + GenericCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::setProb ( Y(aps[3]), MapKey2D<X1,X2>(aps[1],aps[2]) ) = atof(aps[4]); + return true; + } + return false; + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + + +//////////////////// +template<class Y, class X1, class X2, class X3, class P, const char* psLbl=psNil> +class CPT4DModel : public GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl> { + typedef GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl> ParentType; + typedef CPT4DModel<Y,X1,X2,X3,P,psLbl> ThisType; + public: +/* typedef typename GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::IterVal IterVal; */ + +/* bool setFirst ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3 ) const { */ +/* return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::setFirst ( ixyp, MapKey3D<X1,X2,X3>(x1,x2,x3) ); */ +/* } */ +/* bool setNext ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3 ) const { */ +/* return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::setNext ( ixyp, MapKey3D<X1,X2,X3>(x1,x2,x3) ); */ +/* } */ + bool contains ( const Y& y, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::contains ( y, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::contains ( MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } +/* P getProb ( const IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3 ) const { */ +/* return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::getProb ( ixyp, MapKey3D<X1,X2,X3>(x1,x2,x3) ); */ +/* } */ + P getProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::getProb ( y, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + const typename GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::distribution& getDist ( const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::get ( MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3 ) { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::setProb ( y, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 6==aps.size() ) { + GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::setProb ( Y(aps[4]), MapKey3D<X1,X2,X3>(aps[1],aps[2],aps[3]) ) = atof(aps[5]); + return true; + } + return false; + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + + +//////////////////// +template<class Y, class X1, class X2, class X3, class X4, class P, const char* psLbl=psNil> +class CPT5DModel : public GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl> { + typedef GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl> ParentType; + typedef CPT5DModel<Y,X1,X2,X3,X4,P,psLbl> ThisType; + public: +/* typedef typename GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::IterVal IterVal; */ + +/* bool setFirst ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { */ +/* return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::setFirst ( ixyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); */ +/* } */ +/* bool setNext ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { */ +/* return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::setNext ( ixyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); */ +/* } */ + bool contains ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::contains ( y, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::contains ( MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } +/* P getProb ( const IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { */ +/* return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::getProb ( ixyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); */ +/* } */ + P getProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::getProb ( y, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + const typename GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::distribution& getDist ( const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::get ( MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::setProb ( y, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 7==aps.size() ) { + GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::setProb ( Y(aps[5]), MapKey4D<X1,X2,X3,X4>(aps[1],aps[2],aps[3],aps[4]) ) = atof(aps[6]); + return true; + } + return false; + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + + +//////////////////// +template<class Y, class X1, class X2, class X3, class X4, class X5, class P, const char* psLbl=psNil> +class CPT6DModel : public GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl> { + typedef GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl> ParentType; + typedef CPT6DModel<Y,X1,X2,X3,X4,X5,P,psLbl> ThisType; + public: +/* typedef typename GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::IterVal IterVal; */ + +/* bool setFirst ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { */ +/* return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::setFirst ( ixyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); */ +/* } */ +/* bool setNext ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { */ +/* return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::setNext ( ixyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); */ +/* } */ + bool contains ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::contains ( y, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::contains ( MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } +/* P getProb ( const IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { */ +/* return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::getProb ( ixyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); */ +/* } */ + P getProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::getProb ( y, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + const typename GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::distribution& getDist ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::get ( MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::setProb ( y, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 8==aps.size() ) { + GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::setProb ( Y(aps[6]), MapKey5D<X1,X2,X3,X4,X5>(aps[1],aps[2],aps[3],aps[4],aps[5]) ) = atof(aps[7]); + return true; + } + return false; + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + + + + + + + + + + + + + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +/* +template<class Y, class P> +class HidVarBaseIterVal : public pair<SafePtr<const Array<pair<Y,P> > >,Id<int> > { + public: + typedef Y VAR; + static const int NUM_ITERS = 1; + operator Y ( ) const { return pair<SafePtr<const Array<pair<Y,P> > >,Id<int> >::first.getRef().get(pair<SafePtr<const Array<pair<Y,P> > >,Id<int> >::second.toInt()).first; } +}; +*/ + +template<class Y, class K, class P, const char* psLbl=psNil> +//class GenericHidVarCPTModel : public SimpleHash<K,Array<pair<Y,P> > > { +class GenericHidVarCPTModel : public SimpleHash<K,typename Y::template ArrayDistrib<P> > { + private: + //typedef SimpleHash<K,typename Y::template Distrib<P> > HKYP; + //typedef SimpleHash<K,Array<pair<Y,P> > > HKYP; + typedef SimpleHash<K,typename Y::template ArrayDistrib<P> > HKYP; + typedef int IYP; + ////HKYP h; + + public: + typedef Y RandVarType; + typedef Y RVType; + typedef typename Y::template ArrayIterator<P> IterVal; + + typename HKYP::const_iterator begin ( ) const { return HKYP::begin(); } + typename HKYP::iterator begin ( ) { return HKYP::begin(); } + typename Y::template ArrayIterator<P> begin ( const K& k ) const { + typename Y::template ArrayIterator<P> iyp; iyp.first = HKYP::get(k); iyp.second = 0; return iyp; + } + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, const K& k, int& vctr ) const { + P pr; + // If fork happens before this var, set val to first in distrib and return prob=1.0... + if ( vctr < 0 ) { + iyp.first = HKYP::get(k); + iyp.second = 0; + pr = ( iyp.first.getRef().get(0).second != P() ) ? P(1.0) : P(); + if (pr == P()){ + cerr << "ERROR: Some condition has no value! Key = '" << k << "' vctr=" << vctr << "\n"; + } + //return P(1.0); + } + // If fork happens at this var, set val to next in distrib and return prob of val (prob=0.0 if nothing is next)... + // NOTE: falling off the end of a distribution is not a model error; only failing to start a new distribution! + else if ( vctr == 0 ) { + if (iyp.second+1<iyp.first.getRef().getSize()) { iyp.second=iyp.second+1; pr = iyp.first.getRef().get(iyp.second.toInt()).second; } + else pr = P(); + } + // If fork happens after this var, retain val and return prob of val... + else { + pr = iyp.first.getRef().get(iyp.second.toInt()).second; + } + vctr--; + return pr; + } + bool contains ( const Y& y, const K& k ) const { + cerr<<"ERROR: GenericHidVarCPTModel no support for 'contains'!\n"; + return false; + } + bool contains ( const K& k ) const { + return ( HKYP::contains(k) ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp, const K& k ) const { + return iyp.first.getRef().get(iyp.second.toInt()).second; + } + const typename Y::template ArrayDistrib<P>& getDistrib ( const K& k ) const { + return HKYP::get(k); + } + + P& setProb ( const Y& y, const K& k ) { + pair<typename Y::BaseType,P>& yp = HKYP::set(k).add(); + yp.first = y; + return yp.second; + } + typename Y::template ArrayDistrib<P>& setDistrib ( const K& k ) { + return HKYP::set(k); + } + void normalize ( ) { + // NOTE: BEAR IN MIND, LOGPROBS CAN'T BE ADDED!!!!! + for ( typename HKYP::iterator ik=HKYP::begin(); ik!=HKYP::end(); ik++ ) { + P prTot; + for ( unsigned int i=0; i<ik->second.size(); i++ ) { + prTot += ik->second.get(i).second; + } + for ( unsigned int i=0; i<ik->second.size(); i++ ) { + ik->second.set(i).second /= prTot; + } + } + } + void dump ( ostream& os, const char* psId ) const { + for ( typename HKYP::const_iterator ik=HKYP::begin(); ik!=HKYP::end(); ik++ ) { + K k=ik->first; + //IterVal y; + //for ( bool b=setFirst(y,k); b; b=setNext(y,k) ) + for ( unsigned int i=0; i<ik->second.size(); i++ ) { + const pair<Y,P>& yp = ik->second.get(i); + os<<psId<<" "<<k<<" : "<<yp.first<<" = "<<yp.second.toDouble()<<"\n"; + } + } + } + void clear ( ) { HKYP::clear(); } + +// friend pair<IStream,GenericHidVarCPTModel<Y,K,P,psLbl>*> operator>> ( IStream is, GenericHidVarCPTModel<Y,K,P,psLbl>& m ) { +// return pair<IStream,GenericHidVarCPTModel<Y,K,P,psLbl>*>(is,&m); } + friend IStream operator>> ( pair<IStream,GenericHidVarCPTModel<Y,K,P,psLbl>*> is_m, const char* psD ) { + Y y; K k; IStream is,is1; GenericHidVarCPTModel<Y,K,P,psLbl>& m = *is_m.second; + is=is_m.first; + if ( is==IStream() ) return is; + is=is>>psLbl; + while((is1=is>>" ")!=IStream())is=is1; + ////cerr<<"reading k...\n"; + is=is>>k>>" "; + ////cerr<<" ...k='"<<k<<"'\n"; + while((is1=is>>" ")!=IStream())is=is1; + is=is>>": "; + while((is1=is>>" ")!=IStream())is=is1; + ////cerr<<"reading y...\n"; + is=is>>y>>" "; + ////cerr<<" ...y='"<<y<<"'\n"; + while((is1=is>>" ")!=IStream())is=is1; + is=is>>"= "; + while((is1=is>>" ")!=IStream())is=is1; + ////cerr<<"reading pr...\n"; + return (is!=IStream()) ? is>>m.setProb(y,k)>>psD : is; + } + + // OBSOLETE! + friend pair<StringInput,GenericHidVarCPTModel<Y,K,P,psLbl>*> operator>> ( StringInput si, GenericHidVarCPTModel<Y,K,P,psLbl>& m ) { + return pair<StringInput,GenericHidVarCPTModel<Y,K,P,psLbl>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,GenericHidVarCPTModel<Y,K,P,psLbl>*> si_m, const char* psD ) { + Y y; K k; StringInput si,si2; GenericHidVarCPTModel<Y,K,P,psLbl>& m = *si_m.second; + si=si_m.first; + if ( si==NULL ) return si; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>k>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>": "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>y>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>"= "; + while((si2=si>>" ")!=NULL)si=si2; + return (si!=NULL) ? si>>m.setProb(y,k)>>psD : si; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +//////////////////// +template<class Y, class P, const char* psLbl=psNil> +class HidVarCPT1DModel : public GenericHidVarCPTModel<Y,MapKey1D<Unit>,P,psLbl> { + typedef GenericHidVarCPTModel<Y,MapKey1D<Unit>,P,psLbl> ParentType; + typedef HidVarCPT1DModel<Y,P,psLbl> ThisType; + public: + HidVarCPT1DModel ( ) { } + HidVarCPT1DModel ( const Y& y ) { setProb(y)=P(1.0); } + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, int& vctr ) const { + return GenericHidVarCPTModel<Y,MapKey1D<Unit>,P,psLbl>::setIterProb ( iyp, MapKey1D<Unit>(), vctr ); + } + P& setProb ( const Y& y ) { + return GenericHidVarCPTModel<Y,MapKey1D<Unit>,P,psLbl>::setProb ( y, MapKey1D<Unit>() ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp) const { + return GenericHidVarCPTModel<Y,MapKey1D<Unit>,P,psLbl>::getProb ( iyp, MapKey1D<Unit>() ); + } + const typename Y::template ArrayDistrib<P>& getDistrib ( ) const { + return GenericHidVarCPTModel<Y,MapKey1D<Unit>,P,psLbl>::getDistrib ( MapKey1D<Unit>() ); + } + bool contains ( ) const { + return GenericHidVarCPTModel<Y,MapKey1D<Unit>,P,psLbl>::contains(MapKey1D<Unit>()); + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + +//////////////////// +template<class Y, class X1, class P, const char* psLbl=psNil> +class HidVarCPT2DModel : public GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl> { + typedef GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl> ParentType; + typedef HidVarCPT2DModel<Y,X1,P,psLbl> ThisType; + public: + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, const X1& x1, int& vctr ) const { + return GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl>::setIterProb ( iyp, MapKey1D<X1>(x1), vctr ); + } + P& setProb ( const Y& y, const X1& x1 ) { + return GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl>::setProb ( y, MapKey1D<X1>(x1) ); + } + typename Y::template ArrayDistrib<P>& setDistrib ( const X1& x1 ) { + return GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl>::setDistrib ( MapKey1D<X1>(x1) ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp, const X1& x1) const { + return GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl>::getProb ( iyp, MapKey1D<X1>(x1) ); + } + const typename Y::template ArrayDistrib<P>& getDistrib ( const X1& x1 ) const { + return GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl>::getDistrib ( MapKey1D<X1>(x1) ); + } + bool contains ( const X1& x1 ) const { + return GenericHidVarCPTModel<Y,MapKey1D<X1>,P,psLbl>::contains(MapKey1D<X1>(x1)); + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + +//////////////////// +template<class Y, class X1, class X2, class P, const char* psLbl=psNil> +class HidVarCPT3DModel : public GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl> { + typedef GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl> ParentType; + typedef HidVarCPT3DModel<Y,X1,X2,P,psLbl> ThisType; + public: + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, int& vctr ) const { + return GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::setIterProb ( iyp, MapKey2D<X1,X2>(x1,x2), vctr ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2 ) { + return GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::setProb ( y, MapKey2D<X1,X2>(x1,x2) ); + } + typename Y::template ArrayDistrib<P>& setDistrib ( const X1& x1, const X2& x2 ) { + return GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::setDistrib ( MapKey2D<X1,X2>(x1,x2) ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2 ) const { + return GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::getProb ( iyp, MapKey2D<X1,X2>(x1,x2) ); + } + const typename Y::template ArrayDistrib<P>& getDistrib ( const X1& x1, const X2& x2 ) const { + return GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::getDistrib ( MapKey2D<X1,X2>(x1,x2) ); + } + bool contains ( const X1& x1, const X2& x2 ) const { + return GenericHidVarCPTModel<Y,MapKey2D<X1,X2>,P,psLbl>::contains(MapKey2D<X1,X2>(x1,x2)); + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + +//////////////////// +template<class Y, class X1, class X2, class X3, class P, const char* psLbl=psNil> +class HidVarCPT4DModel : public GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl> { + typedef GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl> ParentType; + typedef HidVarCPT4DModel<Y,X1,X2,X3,P,psLbl> ThisType; + public: + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3, int& vctr ) const { + return GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::setIterProb ( iyp, MapKey3D<X1,X2,X3>(x1,x2,x3), vctr ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3 ) { + return GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::setProb ( y, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + typename Y::template ArrayDistrib<P>& setDistrib ( const X1& x1, const X2& x2, const X3& x3 ) { + return GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::setDistrib ( MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::getProb ( iyp, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + const typename Y::template ArrayDistrib<P>& getDistrib ( const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::getDistrib ( MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericHidVarCPTModel<Y,MapKey3D<X1,X2,X3>,P,psLbl>::contains(MapKey3D<X1,X2,X3>(x1,x2,x3)); + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + +//////////////////// +template<class Y, class X1, class X2, class X3, class X4, class P, const char* psLbl=psNil> +class HidVarCPT5DModel : public GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl> { + typedef GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl> ParentType; + typedef HidVarCPT5DModel<Y,X1,X2,X3,X4,P,psLbl> ThisType; + public: + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, int& vctr ) const { + return GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::setIterProb ( iyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4), vctr ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) { + return GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::setProb ( y, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + typename Y::template ArrayDistrib<P>& setDistrib ( const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) { + return GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::setDistrib ( MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::getProb ( iyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + const typename Y::template ArrayDistrib<P>& getDistrib ( const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::getDistrib ( MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericHidVarCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P,psLbl>::contains(MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4)); + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + +//////////////////// +template<class Y, class X1, class X2, class X3, class X4, class X5, class P, const char* psLbl=psNil> +class HidVarCPT6DModel : public GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl> { + typedef GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl> ParentType; + typedef HidVarCPT6DModel<Y,X1,X2,X3,X4,X5,P,psLbl> ThisType; + public: + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, int& vctr ) const { + return GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::setIterProb ( iyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5), vctr ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) { + return GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::setProb ( y, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + typename Y::template ArrayDistrib<P>& setDistrib ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) { + return GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::setDistrib ( MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::getProb ( iyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + const typename Y::template ArrayDistrib<P>& getDistrib ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::getDistrib ( MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericHidVarCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P,psLbl>::contains(MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5)); + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + +//////////////////// +template<class Y, class X1, class X2, class X3, class X4, class X5, class X6, class P, const char* psLbl=psNil> +class HidVarCPT7DModel : public GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl> { + typedef GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl> ParentType; + typedef HidVarCPT7DModel<Y,X1,X2,X3,X4,X5,X6,P,psLbl> ThisType; + public: + P setIterProb ( typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X6& x6, int& vctr ) const { + return GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl>::setIterProb ( iyp, MapKey6D<X1,X2,X3,X4,X5,X6>(x1,x2,x3,x4,x5,x6), vctr ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X6& x6 ) { + return GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl>::setProb ( y, MapKey6D<X1,X2,X3,X4,X5,X6>(x1,x2,x3,x4,x5,x6) ); + } + typename Y::template ArrayDistrib<P>& setDistrib ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X6& x6 ) { + return GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl>::setDistrib ( MapKey6D<X1,X2,X3,X4,X5,X6>(x1,x2,x3,x4,x5,x6) ); + } + P getProb ( const typename Y::template ArrayIterator<P>& iyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X6& x6 ) const { + return GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl>::getProb ( iyp, MapKey6D<X1,X2,X3,X4,X5,X6>(x1,x2,x3,x4,x5,x6) ); + } + const typename Y::template ArrayDistrib<P>& getDistrib ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X6& x6 ) const { + return GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl>::getDistrib ( MapKey6D<X1,X2,X3,X4,X5,X6>(x1,x2,x3,x4,x5,x6) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X6& x6 ) const { + return GenericHidVarCPTModel<Y,MapKey6D<X1,X2,X3,X4,X5,X6>,P,psLbl>::contains(MapKey6D<X1,X2,X3,X4,X5,X6>(x1,x2,x3,x4,x5,x6)); + } + friend pair<IStream,ParentType*> operator>> ( IStream is, ThisType& m ) { return pair<IStream,ParentType*>(is,&m); } +}; + + + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +template<class M1> +class SingleFactoredModel { + private: + M1 m1; + public: + const M1& getM1 ( ) const { return m1; } + M1& setM1 ( ) { return m1; } + void subsume ( SingleFactoredModel<M1>& fm ) { m1.subsume(fm.m1); } + void clear ( ) { m1.clear(); } + bool readFields ( Array<char*>& aps ) { return ( m1.readFields(aps) ); } + friend pair<StringInput,SingleFactoredModel<M1>*> operator>> ( StringInput si, SingleFactoredModel<M1>& m ) { + return pair<StringInput,SingleFactoredModel<M1>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,SingleFactoredModel<M1>*> si_m, const char* psD ) { + return si_m.first>>si_m.second->m1>>psD; } +}; + +template<class M1,class M2> +class DoubleFactoredModel { + private: + M1 m1; M2 m2; + public: + const M1& getM1 ( ) const { return m1; } + const M2& getM2 ( ) const { return m2; } + M1& setM1 ( ) { return m1; } + M2& setM2 ( ) { return m2; } + void subsume ( DoubleFactoredModel<M1,M2>& fm ) { m1.subsume(fm.m1); m2.subsume(fm.m2); } + void clear ( ) { m1.clear(); m2.clear(); } + bool readFields ( Array<char*>& aps ) { return ( m1.readFields(aps) || m2.readFields(aps) ); } + friend pair<StringInput,DoubleFactoredModel<M1,M2>*> operator>> ( StringInput si, DoubleFactoredModel<M1,M2>& m ) { + return pair<StringInput,DoubleFactoredModel<M1,M2>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,DoubleFactoredModel<M1,M2>*> si_m, const char* psD ) { + StringInput si; return ( (si=si_m.first>>si_m.second->m1>>psD)!=NULL || + (si=si_m.first>>si_m.second->m2>>psD)!=NULL ) ? si : NULL; } +}; + +template<class M1,class M2,class M3> +class TripleFactoredModel { + private: + M1 m1; M2 m2; M3 m3; + public: + const M1& getM1 ( ) const { return m1; } + const M2& getM2 ( ) const { return m2; } + const M3& getM3 ( ) const { return m3; } + M1& setM1 ( ) { return m1; } + M2& setM2 ( ) { return m2; } + M3& setM3 ( ) { return m3; } + void subsume ( TripleFactoredModel<M1,M2,M3>& fm ) { m1.subsume(fm.m1); m2.subsume(fm.m2); m3.subsume(fm.m3); } + void clear ( ) { m1.clear(); m2.clear(); m3.clear(); } + bool readFields ( Array<char*>& aps ) { return ( m1.readFields(aps) || m2.readFields(aps) || m3.readFields(aps) ); } + friend pair<StringInput,TripleFactoredModel<M1,M2,M3>*> operator>> ( StringInput si, TripleFactoredModel<M1,M2,M3>& m ) { + return pair<StringInput,TripleFactoredModel<M1,M2,M3>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,TripleFactoredModel<M1,M2,M3>*> si_m, const char* psD ) { + StringInput si; return ( (si=si_m.first>>si_m.second->m1>>psD)!=NULL || + (si=si_m.first>>si_m.second->m2>>psD)!=NULL || + (si=si_m.first>>si_m.second->m3>>psD)!=NULL ) ? si : NULL; } +}; + +template<class M1,class M2,class M3,class M4> +class QuadrupleFactoredModel { + private: + M1 m1; M2 m2; M3 m3;M4 m4; + public: + const M1& getM1 ( ) const { return m1; } + const M2& getM2 ( ) const { return m2; } + const M3& getM3 ( ) const { return m3; } + const M4& getM4 ( ) const { return m4; } + M1& setM1 ( ) { return m1; } + M2& setM2 ( ) { return m2; } + M3& setM3 ( ) { return m3; } + M4& setM4 ( ) { return m4; } + void subsume ( QuadrupleFactoredModel<M1,M2,M3,M4>& fm ) { m1.subsume(fm.m1); m2.subsume(fm.m2); m3.subsume(fm.m3); m4.subsum(fm.m4); } + void clear ( ) { m1.clear(); m2.clear(); m3.clear(); m4.clear(); } + bool readFields ( Array<char*>& aps ) { return ( m1.readFields(aps) || m2.readFields(aps) || m3.readFields(aps) || m4.readFields(aps) ); } + friend pair<StringInput,QuadrupleFactoredModel<M1,M2,M3,M4>*> operator>> ( StringInput si, QuadrupleFactoredModel<M1,M2,M3,M4>& m ) { + return pair<StringInput,QuadrupleFactoredModel<M1,M2,M3,M4>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,QuadrupleFactoredModel<M1,M2,M3,M4>*> si_m, const char* psD ) { + StringInput si; return ( (si=si_m.first>>si_m.second->m1>>psD)!=NULL || + (si=si_m.first>>si_m.second->m2>>psD)!=NULL || + (si=si_m.first>>si_m.second->m3>>psD)!=NULL || + (si=si_m.first>>si_m.second->m4>>psD)!=NULL ) ? si : NULL; } +}; + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +/* +template<class IMRV1> +class ComplexSingleIteratedModeledRV { + public: + static const int NUM_ITERS = IMRV1::NUM_ITERS; + IMRV1 iter_first; + ComplexSingleIteratedModeledRV ( ) { } + ComplexSingleIteratedModeledRV ( const ComplexSingleIteratedModeledRV& imrv ) : iter_first(imrv.iter_first) { } + ComplexSingleIteratedModeledRV ( const IMRV1& imrv1 ) : iter_first(imrv1) { } + void write ( FILE* pf ) const { iter_first.write(pf); } + friend ostream& operator<< ( ostream& os, const ComplexSingleIteratedModeledRV<IMRV1>& rv ) { return os<<rv.iter_first; } +}; + +template<char* SD1,class IMRV1,char* SD2,class IMRV2,char* SD3> +class ComplexDoubleIteratedModeledRV { + public: + static const int NUM_ITERS = IMRV1::NUM_ITERS + IMRV2::NUM_ITERS; +// typedef DelimitedJoint2DRV<SD1,typename IMRV1::VAR,SD2,typename IMRV2::VAR,SD3> VAR; + IMRV1 iter_first; + IMRV2 iter_second; + ComplexDoubleIteratedModeledRV ( ) { } + ComplexDoubleIteratedModeledRV ( const ComplexDoubleIteratedModeledRV& imrv ) : iter_first(imrv.iter_first), iter_second(imrv.iter_second) { } + ComplexDoubleIteratedModeledRV ( const IMRV1& imrv1, const IMRV2& imrv2 ) : iter_first(imrv1), iter_second(imrv2) { } +// operator VAR() { return VAR(iter_first,iter_second); } + void write ( FILE* pf ) const { iter_first.write(pf); fprintf(pf,","); iter_second.write(pf); } + friend ostream& operator<< ( ostream& os, const ComplexDoubleIteratedModeledRV<SD1,IMRV1,SD2,IMRV2,SD3>& rv ) { return os<<SD1<<rv.iter_first<<SD2<<rv.iter_second<<SD3; } +}; + +template<char* SD1,class IMRV1,char* SD2,class IMRV2,char* SD3,class IMRV3,char* SD4> +class ComplexTripleIteratedModeledRV { + public: + static const int NUM_ITERS = IMRV1::NUM_ITERS + IMRV2::NUM_ITERS + IMRV3::NUM_ITERS; + IMRV1 iter_first; + IMRV2 iter_second; + IMRV3 iter_third; + ComplexTripleIteratedModeledRV ( ) { } + ComplexTripleIteratedModeledRV ( const ComplexTripleIteratedModeledRV& imrv ) : iter_first(imrv.iter_first), iter_second(imrv.iter_second), iter_third(imrv.iter_third) { } + ComplexTripleIteratedModeledRV ( const IMRV1& imrv1, const IMRV2& imrv2, const IMRV3& imrv3 ) : iter_first(imrv1), iter_second(imrv2), iter_third(imrv3) { } + void write ( FILE* pf ) const { iter_first.write(pf); fprintf(pf,","); iter_second.write(pf); fprintf(pf,","); iter_third.write(pf); } + friend ostream& operator<< ( ostream& os, const ComplexTripleIteratedModeledRV<SD1,IMRV1,SD2,IMRV2,SD3,IMRV3,SD4>& rv ) + { return os<<SD1<<rv.iter_first<<SD2<<rv.iter_second<<SD3<<rv.iter_third<<SD4; } +}; + +template<char* SD1,class IMRV1,char* SD2,class IMRV2,char* SD3,class IMRV3,char* SD4,class IMRV4,char* SD5> +class ComplexQuadrupleIteratedModeledRV { + public: + static const int NUM_ITERS = IMRV1::NUM_ITERS + IMRV2::NUM_ITERS + IMRV3::NUM_ITERS + IMRV4::NUM_ITERS; + IMRV1 iter_first; + IMRV2 iter_second; + IMRV3 iter_third; + IMRV4 iter_fourth; + ComplexQuadrupleIteratedModeledRV ( ) { } + ComplexQuadrupleIteratedModeledRV ( const ComplexQuadrupleIteratedModeledRV& imrv ) : iter_first(imrv.iter_first), iter_second(imrv.iter_second), + iter_third(imrv.iter_third), iter_fourth(imrv.iter_fourth) { } + ComplexQuadrupleIteratedModeledRV ( const IMRV1& imrv1, const IMRV2& imrv2, const IMRV3& imrv3, const IMRV4& imrv4 ) : iter_first(imrv1), iter_second(imrv2), + iter_third(imrv3), iter_fourth(imrv4) { } + void write ( FILE* pf ) const { iter_first.write(pf); fprintf(pf,","); iter_second.write(pf); fprintf(pf,","); iter_third.write(pf); fprintf(pf,","); iter_fourth.write(pf); } + friend ostream& operator<< ( ostream& os, const ComplexQuadrupleIteratedModeledRV<SD1,IMRV1,SD2,IMRV2,SD3,IMRV3,SD4,IMRV4,SD5>& rv ) + { return os<<SD1<<rv.iter_first<<SD2<<rv.iter_second<<SD3<<rv.iter_third<<SD4<<rv.iter_fourth<<SD5; } +}; + +template<class IMRV,char* SD,int I> +class ComplexArrayIteratedModeledRV { + public: + static const int NUM_ITERS = IMRV::NUM_ITERS * I; + DelimitedStaticSafeArray<I,SD,IMRV> iter_array; + void write ( FILE* pf ) const { for(int i=0;i<I;i++) iter_array.get(i).write(pf); } + friend ostream& operator<< ( ostream& os, const ComplexArrayIteratedModeledRV<IMRV,SD,I>& rv ) { return os<<rv.iter_array; } +}; +*/ + + +#endif //_NL_CPT__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-crf.h b/contrib/synlm/hhmm/rvtl/include/nl-crf.h new file mode 100644 index 000000000..44744ad03 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-crf.h @@ -0,0 +1,359 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_CRF__ +#define _NL_CRF__ + +#include "nl-safeids.h" +#include "nl-probmodel.h" +#include <cassert> +#include <math.h> + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// CRFModeledRV<Y> +// +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2> +class CRF3DModeledRV : public Y { + + private: + + // Static data members... + static bool bModel; // whether model defined yet + static int cardGlb; // global dependencies (used in all potentials) + static int cardOff; // offset positions in site var sequence + static int cardSh; // clique shapes at each offset + static int cardCnd; // possible condition clique configs incl non-site vars in high bits + static int bitsVal; // size in bits of val part of clique config + static int bitsValSite; // size in bits of each site var in val clique config + static SafeArray5D<Id<int>,int,int,int,int,float> aaaaaPotentials; // the model +/* static SafeArray3D<int> aaaCnds; // calc features only once per frame */ + + public: + + // Static extraction methods... + static const float& getPotential ( Id<int> glb, int off, int sh, int configCnd, int configVal ) + { assert(bModel); return aaaaaPotentials.set(glb,off,sh,configCnd,configVal); } + + // Static specification methods... + static void init ( int g, int o, int s, int c, int v, int b ) + { cardGlb=g; cardOff=o; cardSh=s; cardCnd=c; bitsVal=v; bitsValSite=b; } + static float& setPotential ( Id<int> glb, int off, int sh, int configCnd, int configVal ) + { if(!bModel){aaaaaPotentials.init(cardGlb,cardOff,cardSh,cardCnd,1<<bitsVal,1.0); bModel=true;} + return aaaaaPotentials.set(glb,off,sh,configCnd,configVal); } + static void updateObservCliques ( const X1&, const X2& ) ; + + // Static input / output methods... + static bool readModelFields ( char*[], int ) ; + + // Extraction methods... + Prob getProb ( const X1&, const X2& ) const ; + + // Input / output methods... + void writeObservCliqueConfigs ( FILE*, int, const char*, const X1&, const X2&, bool ) const ; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2> bool CRF3DModeledRV<Y,X1,X2>::bModel = false; +template <class Y,class X1,class X2> int CRF3DModeledRV<Y,X1,X2>::cardGlb = 0; +template <class Y,class X1,class X2> int CRF3DModeledRV<Y,X1,X2>::cardOff = 0; +template <class Y,class X1,class X2> int CRF3DModeledRV<Y,X1,X2>::cardSh = 0; +template <class Y,class X1,class X2> int CRF3DModeledRV<Y,X1,X2>::cardCnd = 0; +template <class Y,class X1,class X2> int CRF3DModeledRV<Y,X1,X2>::bitsVal = 0; +template <class Y,class X1,class X2> int CRF3DModeledRV<Y,X1,X2>::bitsValSite = 0; +template <class Y,class X1,class X2> SafeArray5D<Id<int>,int,int,int,int,float> CRF3DModeledRV<Y,X1,X2>::aaaaaPotentials; +/* template <class Y,class X1,class X2> SafeArray3D<int> CRF3DModeledRV<Y,X1,X2>::aaaCnds; */ + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2> +Prob CRF3DModeledRV<Y,X1,X2>::getProb( const X1& x1, const X2& x2 ) const { + + SafeArray2D<int,int,int> aaCnds ( cardOff, cardSh ) ; + SafeArray2D<int,int,double> aaTrell ( cardOff, 1<<bitsVal, 0.0 ) ; + double prob = 1.0; + + // For each offset... + for ( int off=0; off<cardOff; off++ ) + // For each shape... + for ( int sh=0; sh<cardSh; sh++ ) + // Update clique config for condition... + aaCnds.set(off,sh) = Y::getCliqueConfigCnd ( x1, x2, off, sh ) ; + + // For each offset... + for ( int off=0; off<cardOff; off++ ) { + // For each shape... + for ( int sh=0; sh<cardSh; sh++ ) + // Multiply phi for feature (that is, exp lambda) into numerator... + prob *= getPotential(Y::getGlobalDependency(x1,x2),off,sh, + aaCnds.get(off,sh), + Y::getCliqueConfigVal(off,sh)); + + // If first column in trellis... + if ( 0==off ) { + // For each trellis value... + for ( int configVal=0; configVal<(1<<bitsVal); configVal++ ) { + // Add weight of each shape at current offset... + float prod=1.0; + for ( int sh=0; sh<cardSh; sh++ ) + prod *= getPotential(Y::getGlobalDependency(x1,x2),off,sh, + aaCnds.get(off,sh), + configVal) ; + aaTrell.set(off,configVal) = prod ; + } + // If subsequent column in trellis... + } else { + // For each trellis transition (overlap = all but one)... + for ( int configRghtValSite=0; configRghtValSite<(1<<bitsValSite); configRghtValSite++ ) + for ( int configValOverlap=0; configValOverlap<(1<<(bitsVal-bitsValSite)); configValOverlap++ ) { + int configRghtVal = (configValOverlap<<bitsValSite)+configRghtValSite; + // For each possible preceding trellis node... + for ( int configLeftValSite=0; configLeftValSite<(1<<bitsValSite); configLeftValSite++ ) { + int configLeftVal = (configLeftValSite<<(bitsVal-bitsValSite))+configValOverlap; + // Add product of result and previous trellis cell to current trellis cell... + aaTrell.set(off,configRghtVal) += aaTrell.get(off-1,configLeftVal) ; + } + // Multiply weight of each shape... + float prod=1.0; + for ( int sh=0; sh<cardSh; sh++ ) + prod *= getPotential(Y::getGlobalDependency(x1,x2),off,sh, + aaCnds.get(off,sh), + configRghtVal); + aaTrell.set(off,configRghtVal) *= prod; + } + } + } // END EACH OFFSET + + // Calc total prob mass: sum of all possible forward scores in trellis... + double probZ = 0.0; + for ( int i=0; i<(1<<bitsVal); i++ ) + probZ += aaTrell.get(cardOff-1,i); + // Normalize prob by total prob mass... + return prob/probZ; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2> +bool CRF3DModeledRV<Y,X1,X2>::readModelFields ( char* aps[], int numFields ) { + if ( 7==numFields ) + setPotential ( X1(string(aps[1])), // globals + atoi(aps[2]), // offsets + atoi(aps[3]), // shapes + atoi(aps[4]), // cnds + atoi(aps[5]) ) = exp(atof(aps[6])) ; // vals + else return false; + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2> +void CRF3DModeledRV<Y,X1,X2>::writeObservCliqueConfigs ( FILE* pf, int frame, const char* psMdl, + const X1& x1, const X2& x2, bool bObsVal ) const { + fprintf ( pf, "%04d> %s ", frame, psMdl ); + // For each shape (feature slope)... + for ( int sh=0; sh<cardSh; sh++ ) { + // Print clique config condition at each offset... + for ( int off=0; off<cardOff; off++ ) + fprintf ( pf, "%c", intToTetraHex(Y::getCliqueConfigCnd(x1,x2,off,sh)) ); + if (sh<cardSh-1) printf(","); // commas between shapes + } + printf(" : "); // cond/val delimiter + // Print clique config value at each offset... + if ( bObsVal ) + for ( int off=0; off<cardOff; off++ ) + fprintf ( pf, "%c", intToTetraHex(Y::getCliqueConfigVal(off,0)) ); + else fprintf ( pf, "_" ) ; + printf("\n"); +} + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// CRF4DModeledRV<Y> +// +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2,class X3> +class CRF4DModeledRV : public Y { + + private: + + // Static data members... + static bool bModel; // whether model defined yet + static int cardGlb; // global dependencies (used in all potentials) + static int cardOff; // offset positions in site var sequence + static int cardSh; // clique shapes at each offset + static int cardCnd; // possible condition clique configs incl non-site vars in high bits + static int bitsVal; // size in bits of val part of clique config + static int bitsValSite; // size in bits of each site var in val clique config + static SafeArray5D<Id<int>,int,int,int,int,float> aaaaaPotentials; // the model +/* static SafeArray3D<int> aaaCnds; // calc features only once per frame */ + + public: + + // Static extraction methods... + static const float& getPotential ( Id<int> glb, int off, int sh, int configCnd, int configVal ) + { assert(bModel); return aaaaaPotentials.set(glb,off,sh,configCnd,configVal); } + + // Static specification methods... + static void init ( int g, int o, int s, int c, int v, int b ) + { cardGlb=g; cardOff=o; cardSh=s; cardCnd=c; bitsVal=v; bitsValSite=b; } + static float& setPotential ( Id<int> glb, int off, int sh, int configCnd, int configVal ) + { if(!bModel){aaaaaPotentials.init(cardGlb,cardOff,cardSh,cardCnd,1<<bitsVal,1.0); bModel=true;} + return aaaaaPotentials.set(glb,off,sh,configCnd,configVal); } + + // Static input / output methods... + static bool readModelFields ( char*[], int ) ; + + // Extraction methods... + Prob getProb ( const X1&, const X2&, const X3& ) const ; + + // Input / output methods... + void writeObservCliqueConfigs ( FILE*, int, const char*, const X1&, const X2&, const X3&, bool ) const ; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2,class X3> bool CRF4DModeledRV<Y,X1,X2,X3>::bModel = false; +template <class Y,class X1,class X2,class X3> int CRF4DModeledRV<Y,X1,X2,X3>::cardGlb = 0; +template <class Y,class X1,class X2,class X3> int CRF4DModeledRV<Y,X1,X2,X3>::cardOff = 0; +template <class Y,class X1,class X2,class X3> int CRF4DModeledRV<Y,X1,X2,X3>::cardSh = 0; +template <class Y,class X1,class X2,class X3> int CRF4DModeledRV<Y,X1,X2,X3>::cardCnd = 0; +template <class Y,class X1,class X2,class X3> int CRF4DModeledRV<Y,X1,X2,X3>::bitsVal = 0; +template <class Y,class X1,class X2,class X3> int CRF4DModeledRV<Y,X1,X2,X3>::bitsValSite = 0; +template <class Y,class X1,class X2,class X3> SafeArray5D<Id<int>,int,int,int,int,float> + CRF4DModeledRV<Y,X1,X2,X3>::aaaaaPotentials; +/* template <class Y,class X1,class X2> SafeArray3D<int> CRF4DModeledRV<Y,X1,X2>::aaaCnds; */ + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2,class X3> +Prob CRF4DModeledRV<Y,X1,X2,X3>::getProb( const X1& x1, const X2& x2, const X3& x3 ) const { + + SafeArray2D<int,int,int> aaCnds ( cardOff, cardSh ) ; + SafeArray2D<int,int,double> aaTrell ( cardOff, 1<<bitsVal, 0.0 ) ; + double prob = 1.0; + + // For each offset... + for ( int off=0; off<cardOff; off++ ) + // For each shape... + for ( int sh=0; sh<cardSh; sh++ ) + // Update clique config for condition... + aaCnds.set(off,sh) = Y::getCliqueConfigCnd ( x1, x2, x3, off, sh ) ; + + // For each offset... + for ( int off=0; off<cardOff; off++ ) { + // For each shape... + for ( int sh=0; sh<cardSh; sh++ ) + // Multiply phi for feature (that is, exp lambda) into numerator... + prob *= getPotential(Y::getGlobalDependency(x1,x2,x3),off,sh, + aaCnds.get(off,sh), + Y::getCliqueConfigVal(off,sh)); + + // If first column in trellis... + if ( 0==off ) { + // For each trellis value... + for ( int configVal=0; configVal<(1<<bitsVal); configVal++ ) { + // Add weight of each shape at current offset... + float prod=1.0; + for ( int sh=0; sh<cardSh; sh++ ) + prod *= getPotential(Y::getGlobalDependency(x1,x2,x3),off,sh, + aaCnds.get(off,sh), + configVal) ; + aaTrell.set(off,configVal) = prod ; + } + // If subsequent column in trellis... + } else { + // For each trellis transition (overlap = all but one)... + for ( int configRghtValSite=0; configRghtValSite<(1<<bitsValSite); configRghtValSite++ ) + for ( int configValOverlap=0; configValOverlap<(1<<(bitsVal-bitsValSite)); configValOverlap++ ) { + int configRghtVal = (configValOverlap<<bitsValSite)+configRghtValSite; + // For each possible preceding trellis node... + for ( int configLeftValSite=0; configLeftValSite<(1<<bitsValSite); configLeftValSite++ ) { + int configLeftVal = (configLeftValSite<<(bitsVal-bitsValSite))+configValOverlap; + // Add product of result and previous trellis cell to current trellis cell... + aaTrell.set(off,configRghtVal) += aaTrell.get(off-1,configLeftVal) ; + } + // Multiply weight of each shape... + float prod=1.0; + for ( int sh=0; sh<cardSh; sh++ ) + prod *= getPotential(Y::getGlobalDependency(x1,x2,x3),off,sh, + aaCnds.get(off,sh), + configRghtVal); + aaTrell.set(off,configRghtVal) *= prod; + } + } + } // END EACH OFFSET + + // Calc total prob mass: sum of all possible forward scores in trellis... + double probZ = 0.0; + for ( int i=0; i<(1<<bitsVal); i++ ) + probZ += aaTrell.get(cardOff-1,i); + // Normalize prob by total prob mass... + return prob/probZ; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2,class X3> +bool CRF4DModeledRV<Y,X1,X2,X3>::readModelFields ( char* aps[], int numFields ) { + if ( 7==numFields ) + setPotential ( X1(string(aps[1])), // globals + atoi(aps[2]), // offsets + atoi(aps[3]), // shapes + atoi(aps[4]), // cnds + atoi(aps[5]) ) = exp(atof(aps[6])) ; // vals + else return false; + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2, class X3> +void CRF4DModeledRV<Y,X1,X2,X3>::writeObservCliqueConfigs ( FILE* pf, int frame, const char* psMdl, + const X1& x1, const X2& x2, + const X3& x3, bool bObsVal ) const { + fprintf ( pf, "%04d> %s ", frame, psMdl ); + // For each shape (feature slope)... + for ( int sh=0; sh<cardSh; sh++ ) { + // Print clique config condition at each offset... + for ( int off=0; off<cardOff; off++ ) + fprintf ( pf, "%c", intToTetraHex(Y::getCliqueConfigCnd(x1,x2,x3,off,sh)) ); + if (sh<cardSh-1) printf(","); // commas between shapes + } + printf(" : "); // cond/val delimiter + // Print clique config value at each offset... + if ( bObsVal ) + for ( int off=0; off<cardOff; off++ ) + fprintf ( pf, "%c", intToTetraHex(Y::getCliqueConfigVal(off,0)) ); + else fprintf ( pf, "_" ) ; + printf("\n"); +} + +#endif //_NL_CRF__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-denot.h b/contrib/synlm/hhmm/rvtl/include/nl-denot.h new file mode 100644 index 000000000..0b50663a1 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-denot.h @@ -0,0 +1,500 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_DENOT__ +#define _NL_DENOT__ + +#include "nl-safeids.h" +#include "nl-prob.h" +#include "nl-const.h" +#include "nl-randvar.h" +#include "nl-cpt.h" +#include <string> + +//////////////////////////////////////////////////////////////////////////////// +// +// VecE +// +//////////////////////////////////////////////////////////////////////////////// + +template<int N,class I,class RC=ReaderContext> +class VecE : public StaticSafeArray<N,I> { // Coindexation vector + public: + typedef I IntType; + typedef RC RCType; + static const int NUM_ENTS; + // Constructor / destructor methods... + VecE ( ) : StaticSafeArray<N,I>(-1) { } + VecE ( const string& s, RC& rc ) : StaticSafeArray<N,I>(-1) { read(s,rc); } + // Specification methods... + void unOffset ( const ReaderContext& rc ) { } + void pushOffsets ( const ReaderContext& rc ) { } + // Extraction methods... + //int getHashConst ( const int m ) const { int k=1; for(int i=0;i<NUM_ENTS;i++){k=(k<<sizeof(I))%m; } return k; } + //int getHashKey ( const int m ) const { int k=0; for(int i=0;i<NUM_ENTS;i++){k=k<<sizeof(I); k+=abs(StaticSafeArray<N,I>::get(i)); k%=m; } return k; } + size_t getHashKey ( ) const { size_t k=0; for(int i=0;i<NUM_ENTS;i++){k=rotLeft(k,3); k^=StaticSafeArray<N,I>::get(i).toInt(); } return k; } + bool operator< (const VecE<N,I,RC>& e) const { return StaticSafeArray<N,I>::operator<(e); } + bool operator==(const VecE<N,I,RC>& e) const { return StaticSafeArray<N,I>::operator==(e); } + void readercontextPush ( const ReaderContext& rc ) const { } + void readercontextPop ( const ReaderContext& rc ) const { } + // Input / output methods... + void read ( char* ps, const ReaderContext& ) ; + void write ( FILE*, ReaderContext& ) const ; + void write ( FILE* pf ) const { ReaderContext rc; write(pf,rc); } + string getString() const { ReaderContext rc; return getString(rc); } + string getString( ReaderContext& ) const; +}; +template<int N,class I,class RC> const int VecE<N,I,RC>::NUM_ENTS = N; + +//////////////////// +template<int N,class I, class RC> +void VecE<N,I,RC>::read ( char* ps, const ReaderContext& rc ) { + /* + ////fprintf(stderr,"VecE::VecE in\n"); + int i; + string::size_type j; + // Chop into individual coinds strings... + for ( i=0, j=0; s!="" && s!="," && j!=string::npos; i++, j=s.find_first_of(',',j), j=(-1==j)?j:j+1 ) + StaticSafeArray<N,I>::set(i) = s.substr ( j, s.find_first_of(',',j)-j ); + ////fprintf(stderr,"VecE::VecE out\n"); + */ + char* psT; int i=0; + for ( char* psU=strtok_r(ps,",",&psT); + psU && i<NUM_ENTS; + psU=strtok_r(NULL,",",&psT),i++ ) + StaticSafeArray<N,I>::set(i) = psU; +} + +//////////////////// +template<int N,class I, class RC> +void VecE<N,I,RC>::write ( FILE* pf, ReaderContext& rc ) const { + for(int i=0; i<NUM_ENTS; i++) + if(StaticSafeArray<N,I>::get(i) >= 0 ) { + if(i) fprintf(pf,","); StaticSafeArray<N,I>::get(i).write(pf); + } +} + +template<int N, class I, class RC> +string VecE<N,I,RC>::getString( ReaderContext& rc ) const { + string rString; + for(int i=0; i<NUM_ENTS; i++) + if(StaticSafeArray<N,I>::get(i) >= 0 ) { + if(i) rString += ","; + rString += StaticSafeArray<N,I>::get(i).getString(); + } + return rString; +} + +//////////////////////////////////////////////////////////////////////////////// +// +// VecVReaderContext +// +//////////////////////////////////////////////////////////////////////////////// + +class VecVReaderContext : public ReaderContext { + public: + map<string,int> msi; + int offset; + VecVReaderContext ( ) { offset=0; } +}; + +//////////////////////////////////////////////////////////////////////////////// +// +// VecV +// +//////////////////////////////////////////////////////////////////////////////// + +template<int N,class I,class RC=VecVReaderContext,int ND1=0,int ND2=0> +class VecV : public StaticSafeArray<N,I> { // Coindexation vector + public: + typedef I IntType; + typedef RC RCType; + static const int NUM_ENTS; + static const int NUM_ENTS_DEP1; + static const int NUM_ENTS_DEP2; + // Constructor / destructor methods... + VecV ( ) : StaticSafeArray<N,I>(-1) { } + VecV ( const string& s, VecVReaderContext& rc ) : StaticSafeArray<N,I>(-1) { read(s,rc); } + // Specification methods... + void unOffset ( const RCType& rc ) { for(int i=0; i<NUM_ENTS; i++) if(StaticSafeArray<N,I>::get(i)!=-1) StaticSafeArray<N,I>::set(i) += -rc.offset; } + void pushOffsets ( RCType& rc ) { for(typename map<string,int>::iterator i=rc.msi.begin(); i!=rc.msi.end(); i++) i->second+=NUM_ENTS; } + // Extraction methods... + //int getHashConst ( const int m ) const { int k=1; for(int i=0;i<NUM_ENTS;i++){k=(k<<sizeof(I))%m; } return k; } + //int getHashKey ( const int m ) const { int k=0; for(int i=0;i<NUM_ENTS;i++){k=k<<sizeof(I); k+=StaticSafeArray<N,I>::get(i); k%=m; } return k; } + size_t getHashKey ( ) const { size_t k=0; for(int i=0;i<NUM_ENTS;i++){k=rotLeft(k,3); k^=StaticSafeArray<N,I>::get(i).toInt(); } return k; } + bool operator< (const VecV<N,I,RC,ND1,ND2>& v) const { return StaticSafeArray<N,I>::operator<(v); } + bool operator==(const VecV<N,I,RC,ND1,ND2>& v) const { return StaticSafeArray<N,I>::operator==(v); } + void readercontextPush ( VecVReaderContext& rc ) const { rc.offset+=N; } + void readercontextPop ( VecVReaderContext& rc ) const { rc.offset-=N; } + // Input / output methods... + void read ( char*, VecVReaderContext& ) ; + void write ( FILE*, RC& ) const ; + void write ( FILE* pf ) const { RC rc; write(pf,rc); } + string getString( RC& ) const; + string getString() const { RC rc; return getString(rc); } +}; +template<int N,class I,class RC,int ND1,int ND2> const int VecV<N,I,RC,ND1,ND2>::NUM_ENTS = N; +template<int N,class I,class RC,int ND1,int ND2> const int VecV<N,I,RC,ND1,ND2>::NUM_ENTS_DEP1 = ND1; +template<int N,class I,class RC,int ND1,int ND2> const int VecV<N,I,RC,ND1,ND2>::NUM_ENTS_DEP2 = ND2; + +//////////////////// +template<int N,class I,class RC,int ND1,int ND2> +void VecV<N,I,RC,ND1,ND2>::read ( char* ps, VecVReaderContext& rc ) { + ////fprintf(stderr,"VecV::VecV in %d\n",rc.offset); + StaticSafeArray<N,string> asV; + + // Chop into individual coinds strings... + char* psT; int i=0; + for ( char* psU=strtok_r(ps,",",&psT); + psU && i<NUM_ENTS; + psU=strtok_r(NULL,",",&psT), i++ ) + asV.set(i) = psU; + +// int i; +// string::size_type j; +//// for ( i=0, j=s.find_first_of(','); j!=string::npos; i++, j=s.find_first_of(',',j+1) ) +//// for ( i=0, j=-1; i==0 || j!=string::npos; i++, j=s.find_first_of(',',j+1) ) +// for ( i=0, j=0; s!="" && j!=string::npos; i++, j=s.find_first_of(',',j), j=(-1==j)?j:j+1 ) +// asV.set(i) = s.substr ( j, s.find_first_of(',',j)-j ); + + // Going backwards... + for ( i=i-1; i>=0; i-- ) { + // Add to map if new coind... + if (rc.msi.find(asV.get(i)) == rc.msi.end()) rc.msi[asV.get(i)]=i+rc.offset; + StaticSafeArray<N,I>::set(i)=rc.msi[asV.get(i)]; + } + ////fprintf(stderr,"VecV::read out\n"); + //for(int i=0; i<NUM_ENTS; i++) fprintf(stderr," %d",int(StaticSafeArray<N,I>::get(i))); fprintf(stderr,"\n"); + //write(stderr); +} + +//////////////////// +template<int N,class I,class RC,int ND1,int ND2> +void VecV<N,I,RC,ND1,ND2>::write ( FILE* pf, RC& rc ) const { + for(int i=0; i<NUM_ENTS; i++) + ////fprintf(pf,"%sv%02d",(i)?",":"",int(StaticSafeArray<N,I>::get(i).toInt())); + if(StaticSafeArray<N,I>::get(i) >= 0 ) + fprintf(pf,"%sv%02d",(i)?",":"",(rc.offset)+int(StaticSafeArray<N,I>::get(i).toInt())); +} + +template<int N,class I,class RC,int ND1,int ND2> +string VecV<N,I,RC,ND1,ND2>::getString (RC& rc) const { + string rString; + for(int i=0; i<NUM_ENTS; i++) + if(StaticSafeArray<N,I>::get(i) >= 0 ) { + rString += (i)?",":""; + char* tmp = new char[4]; + // char tmp* = (char*)malloc(4*sizeof(char)); + sprintf(tmp, "v%02d\0",(rc.offset)+int(StaticSafeArray<N,I>::get(i).toInt())); + rString += tmp; + delete tmp; + // free(tmp); + } + return rString; +} + +//////////////////////////////////////////////////////////////////////////////// +// +// VecV Compositors +// +//////////////////////////////////////////////////////////////////////////////// + +template<class V1, class V2> +class JointVecV { //// : public StaticSafeArray<V1::NUM_ENTS+V2::NUM_ENTS,I> { + private: + V1 v1; + V2 v2; + public: + typedef typename V1::IntType IntType; + typedef typename V1::RCType RCType; + static const int NUM_ENTS; + // Constructor / destructor methods... + JointVecV ( ) { } + JointVecV ( const V1& a1, const V2& a2 ) { + ////fprintf(stderr,"iJoin "); a1.V1::write(stderr); fprintf(stderr," "); a2.V2::write(stderr); fprintf(stderr,"\n"); + for (int i=0; i<NUM_ENTS; i++) { + if ( i<V1::NUM_ENTS ) set(i) = (a1.get(i)==-1) ? IntType(-1) : (a1.get(i)<V1::NUM_ENTS) ? IntType(a1.get(i)) : a1.get(i)+V2::NUM_ENTS; + else set(i) = (a2.get(i-V1::NUM_ENTS)==-1) ? IntType(-1) : a2.get(i-V1::NUM_ENTS)+V1::NUM_ENTS; + } + ////fprintf(stderr,"oJoin "); JointVecV<V1,V2>::write(stderr); fprintf(stderr,"\n"); + } + // Specification methods... + typename V1::IntType& set(int i) { assert(i>=0); assert(i<NUM_ENTS); return ((i<V1::NUM_ENTS) ? v1.set(i) : v2.set(i-V1::NUM_ENTS) ); } + V1& setSub1() { return v1; } + V2& setSub2() { return v2; } + void unOffset ( const RCType& rc ) { setSub1().unOffset(rc); setSub2().unOffset(rc); } + void pushOffsets ( RCType& rc ) { for(typename map<string,int>::iterator i=rc.msi.begin(); i!=rc.msi.end(); i++) i->second+=NUM_ENTS; } + // Extraction methods... + //int getHashConst ( const int m ) const { return((v1.getHashConst(m) * v2.getHashConst(m))%m); } + //int getHashKey ( const int m ) const { int k=((v1.getHashKey(m)*v2.getHashConst(m))+v2.getHashKey(m))%m; return k; } + size_t getHashKey ( ) const { size_t k=rotLeft(v1.getHashKey(),3); k^=v2.getHashKey(); return k; } + bool operator==(const JointVecV<V1,V2>& vv) const { return (v1==vv.v1 && v2==vv.v2); } + bool operator<(const JointVecV<V1,V2>& vv) const { return (v1<vv.v1 || (v1==vv.v1 && v2<vv.v2)); } + void readercontextPush ( VecVReaderContext& rc ) const { rc.offset+=NUM_ENTS; } + void readercontextPop ( VecVReaderContext& rc ) const { rc.offset-=NUM_ENTS; } + const typename V1::IntType get(int i) const { assert(i>=0); assert(i<NUM_ENTS); return ((i<V1::NUM_ENTS) ? v1.get(i) : v2.get(i-V1::NUM_ENTS) ); } + const V1& getSub1() const { return v1; } + const V2& getSub2() const { return v2; } + // Input / output methods... + void write ( FILE* pf, RCType& rc ) const { v1.write(pf,rc); /*JointVecV<V1,V2>::getSub1().readercontextPush(rc);*/ fprintf(pf,";"); + v2.write(pf,rc); /*JointVecV<V1,V2>::getSub1().readercontextPop(rc);*/ } + void write ( FILE* pf ) const { RCType rc; write(pf,rc); } + string getString( RCType& rc ) const { return v1.getString(rc) + ";" + v2.getString(rc); } + string getString() const { RCType rc; return getString(rc); } +}; +template<class V1, class V2> +const int JointVecV<V1,V2>::NUM_ENTS = V1::NUM_ENTS+V2::NUM_ENTS; + +//////////////////////////////////////////////////////////// + +template<char* SD1,class V1,char* SD2,class V2,char* SD3> +class DelimitedJointVecV : public JointVecV<V1,V2> { + public: + typedef typename V1::RCType RCType; + // Constructor / destructor methods... + DelimitedJointVecV ( ) : JointVecV<V1,V2>() { } + DelimitedJointVecV ( const string& s, typename V1::RCType& rc ) : JointVecV<V1,V2>() { read(s,rc); } + // Extraction methods... + bool operator==(const DelimitedJointVecV<SD1,V1,SD2,V2,SD3>& vv) const { return JointVecV<V1,V2>::operator==(vv); } + bool operator<(const DelimitedJointVecV<SD1,V1,SD2,V2,SD3>& vv) const { return JointVecV<V1,V2>::operator<(vv); } + void readercontextPush ( typename V1::RCType& rc ) const { JointVecV<V1,V2>::getSub1().readercontextPush(rc); JointVecV<V1,V2>::getSub2().readercontextPush(rc); } + void readercontextPop ( typename V1::RCType& rc ) const { JointVecV<V1,V2>::getSub1().readercontextPop(rc); JointVecV<V1,V2>::getSub2().readercontextPop(rc); } + // Input / output methods... + void read ( char*, RCType& ) ; + void write ( FILE* pf, RCType& rc ) const { + fprintf(pf,"%s",SD1); JointVecV<V1,V2>::getSub1().write(pf,rc); /*JointVecV<V1,V2>::getSub1().readercontextPush(rc);*/ + fprintf(pf,"%s",SD2); JointVecV<V1,V2>::getSub2().write(pf,rc); /*JointVecV<V1,V2>::getSub1().readercontextPop(rc);*/ + fprintf(pf,"%s",SD3); + } + void write ( FILE* pf ) const { RCType rc; write(pf,rc); } + string getString(RCType& rc) const { + return SD1 + JointVecV<V1,V2>::getSub1().getString(rc) + SD2 + JointVecV<V1,V2>::getSub2().getString(rc) + SD3; + } + string getString() const { RCType rc; return getString(rc); } +}; + +//////////////////// +template<char* SD1,class V1,char* SD2,class V2,char* SD3> +void DelimitedJointVecV<SD1,V1,SD2,V2,SD3>::read ( char* s, RCType& rc ) { + ////fprintf(stderr,"DelimitedJointVecV::read chopping '%s' into '%s'...'%s'...'%s'\n",s.c_str(),SD1,SD2,SD3); + if(0!=strncmp(SD1,s,strlen(SD1))) fprintf(stderr,"ERR: '%s' doesn't begin with '%s'\n",s,SD1); + if(0!=strcmp(SD3,s+strlen(s)-strlen(SD3))) fprintf(stderr,"ERR: '%s' doesn't end with '%s'\n", s,SD3); + /* + if(0!=strncmp(SD1,s.c_str(),strlen(SD1))) fprintf(stderr,"ERR: '%s' doesn't begin with '%s'\n",s.c_str(),SD1); + if(0!=strcmp(SD3,s.c_str()+s.length()-strlen(SD3))) fprintf(stderr,"ERR: '%s' doesn't end with '%s'\n",s.c_str(),SD3); + string::size_type j = s.find(SD2); + if(string::npos==j) fprintf(stderr,"ERR: no '%s' found in %s\n",SD2,s.c_str()); + string s1 = s.substr ( strlen(SD1), j-strlen(SD1) ); + string s2 = s.substr ( j+strlen(SD2), int(s.length())-int(j)-strlen(SD2)-strlen(SD3) ); + */ + char* s1 = s+strlen(SD1); + char* s2 = strstr(s,SD2); if (!s2) fprintf(stderr,"WARNING: (VecV) no '%s' found in '%s' -- assuming second string empty\n",SD2,s); + s[strlen(s)-strlen(SD3)]='\0'; + if (s2) { *s2='\0'; s2+=strlen(SD2); } else { s2=s+strlen(s); } + //fprintf(stderr,"s1=%s, s2=%s\n",s1,s2); + ////fprintf(stderr,"DelimitedJointVecV::read gives `%s' and `%s'\n",s1.c_str(),s2.c_str()); + JointVecV<V1,V2>::getSub1().readercontextPush(rc); JointVecV<V1,V2>::setSub2().read ( s2, rc ); //V1::NUM_ENTS, msi ); + JointVecV<V1,V2>::getSub1().readercontextPop(rc); JointVecV<V1,V2>::setSub1().read ( s1, rc ); //0, msi ); +} + +//////////////////////////////////////////////////////////// + +char psXX[] = ""; // Null delimiter for joint RVs +template<char N> +class VecVV : public DelimitedJointVecV<psXX,VecV<N,Id<char> >,psXX,VecV<N,Id<char> >,psXX> {}; // Coindexation vector, for result of binary branch rule + +//////////////////////////////////////////////////////////////////////////////// + +template<class V1, class V2> +class ComposedVecV : public JointVecV<V1,V2> { + public: + ComposedVecV ( const V1& v1, const V2& v2 ) : JointVecV<V1,V2>() { + ////fprintf(stderr,"iComp "); v1.write(stderr); fprintf(stderr," "); v2.write(stderr); fprintf(stderr,"\n"); + for (int i=0; i<V1::NUM_ENTS+V2::NUM_ENTS; i++) { + // If in first sub-part... + if ( i<V1::NUM_ENTS ) { + // If not valid... + if ( v1.get(i)==-1 ) continue; + // If v1[i] intra-indexed... + else if ( v1.get(i)<V1::NUM_ENTS ) + JointVecV<V1,V2>::set(i) = v1.get(i); + // If v1[i] inter-indexed... + else if ( v1.get(i)<V1::NUM_ENTS+V2::NUM_ENTS ) +//{ + JointVecV<V1,V2>::set(i) = (v2.get(v1.get(i).toInt()-V1::NUM_ENTS)!=-1) ? v2.get(v1.get(i).toInt()-V1::NUM_ENTS)+V1::NUM_ENTS : -1; +//fprintf(stderr,"??????????? %d(%d,%d)%d,%d: ",i,V1::NUM_ENTS,V2::NUM_ENTS,int(v1.get(i)),int(v2.get(13))); JointVecV<V1,V2>::write(stderr); fprintf(stderr,"\n"); +//} + else + JointVecV<V1,V2>::set(i) = v1.get(i); + } + else { + // If not valid... + if ( v2.get(i-V1::NUM_ENTS)==-1 ) continue; + // If in second sub-part... + else JointVecV<V1,V2>::set(i) = v2.get(i-V1::NUM_ENTS)+V1::NUM_ENTS; + } + } + ////fprintf(stderr,"oComp "); JointVecV<V1,V2>::write(stderr); fprintf(stderr,"\n"); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template<class V1, class V2> +class CommutedVecV : public JointVecV<V1,V2> { + public: + CommutedVecV ( const JointVecV<V2,V1>& v ) : JointVecV<V1,V2>() { + ////fprintf(stderr,"iComm "); v.write(stderr); fprintf(stderr,"\n"); + // Iterate backward through new coind vector... + for ( int i=V1::NUM_ENTS+V2::NUM_ENTS-1; i>=0; i-- ) { + // If in second sub part V2 (first in v)... + if ( i>=V1::NUM_ENTS ) { + // If not valid... + if ( v.get(i-V1::NUM_ENTS)==-1 ) continue; + // If v[i-w2] intra-indexed... + else if ( v.get(i-V1::NUM_ENTS)<V2::NUM_ENTS ) + JointVecV<V1,V2>::set(i) = v.get(i-V1::NUM_ENTS)+V1::NUM_ENTS; + // If v[i-w2] cross-indexed to other sub-part (V1)... + else if ( v.get(i-V1::NUM_ENTS)<V2::NUM_ENTS+V1::NUM_ENTS ) { + if (get(v.get(i-V1::NUM_ENTS).toInt()-V2::NUM_ENTS)==-1) + JointVecV<V1,V2>::set(v.get(i-V1::NUM_ENTS).toInt()-V2::NUM_ENTS) = i; + JointVecV<V1,V2>::set(i) = get(v.get(i-V1::NUM_ENTS).toInt()-V2::NUM_ENTS); + } + // If v[i-w2] inter-indexed... + else JointVecV<V1,V2>::set(i) = v.get(i-V1::NUM_ENTS); + } + // If in first sub part V1 (second in v), and not coindexed already... + else if (JointVecV<V1,V2>::get(i)==-1) { + // If not valid... + if ( v.get(i+V2::NUM_ENTS)==-1 ) continue; + // If v[i+w1] intra-indexed... + else if ( v.get(i+V2::NUM_ENTS)<V2::NUM_ENTS+V1::NUM_ENTS ) { + if ( get(v.get(i+V2::NUM_ENTS).toInt()-V2::NUM_ENTS)==-1 ) // should only happen if self-coindexed + JointVecV<V1,V2>::set(i) = v.get(i+V2::NUM_ENTS)-V2::NUM_ENTS; + else + JointVecV<V1,V2>::set(i) = get(v.get(i+V2::NUM_ENTS).toInt()-V2::NUM_ENTS); + } + // If v[i+w1] inter-indexed... + else JointVecV<V1,V2>::set(i) = v.get(i+V2::NUM_ENTS); + } + //fprintf(stderr,"??????????? %d: ",i); v.write(stderr); fprintf(stderr," -> "); JointVecV<V1,V2>::write(stderr); fprintf(stderr,"\n"); + } + ////fprintf(stderr,"oComm "); JointVecV<V1,V2>::write(stderr); fprintf(stderr,"\n"); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template<class V1, class V2, class V3> +class NewCommutedVecV : public JointVecV<V1,JointVecV<V2,V3> > { + public: + NewCommutedVecV ( const JointVecV<V1,JointVecV<V3,V2> >& v ) : JointVecV<V1,JointVecV<V2,V3> >() { + ////fprintf(stderr,"iNewComm "); v.write(stderr); fprintf(stderr,"\n"); + // Iterate backward through new coind vector... + for ( int i=V1::NUM_ENTS+V2::NUM_ENTS+V3::NUM_ENTS-1; i>=0; i-- ) { + // If in third sub part V3 (second in v)... + if ( i>=V1::NUM_ENTS+V2::NUM_ENTS ) { + // If not valid... + if ( v.get(i-V2::NUM_ENTS)==-1 ) continue; + // If v[i-w2] intra-indexed... + else if ( v.get(i-V2::NUM_ENTS)<V1::NUM_ENTS+V3::NUM_ENTS ) + JointVecV<V1,JointVecV<V2,V3> >::set(i) = v.get(i-V2::NUM_ENTS)+V2::NUM_ENTS; + // If v[i-w2] cross-indexed to other sub-part (V2, third in v)... + else if ( v.get(i-V2::NUM_ENTS)<V1::NUM_ENTS+V3::NUM_ENTS+V2::NUM_ENTS ) { + if (get(v.get(i-V2::NUM_ENTS).toInt()-V3::NUM_ENTS)==-1) + JointVecV<V1,JointVecV<V2,V3> >::set(v.get(i-V2::NUM_ENTS).toInt()-V3::NUM_ENTS) = i; + JointVecV<V1,JointVecV<V2,V3> >::set(i) = get(v.get(i-V2::NUM_ENTS).toInt()-V3::NUM_ENTS); + } + // If v[i-w2] inter-indexed... + else JointVecV<V1,JointVecV<V2,V3> >::set(i) = v.get(i-V2::NUM_ENTS); + } + // If in second sub part V2 (third in v), and not coindexed already... + else if ( i>=V1::NUM_ENTS && JointVecV<V1,JointVecV<V2,V3> >::get(i)==-1 ) { + // If not valid... + if ( v.get(i+V3::NUM_ENTS)==-1 ) continue; + // If v[i+w1] intra-indexed... + else if ( v.get(i+V3::NUM_ENTS)<V1::NUM_ENTS+V3::NUM_ENTS+V2::NUM_ENTS ) + if ( get(v.get(i+V3::NUM_ENTS).toInt()-V3::NUM_ENTS)==-1 ) // should only happen if self-coindexed + JointVecV<V1,JointVecV<V2,V3> >::set(i) = v.get(i+V3::NUM_ENTS)-V3::NUM_ENTS; + else + JointVecV<V1,JointVecV<V2,V3> >::set(i) = get(v.get(i+V3::NUM_ENTS).toInt()-V3::NUM_ENTS); + // If v[i+w1] inter-indexed... + else JointVecV<V1,JointVecV<V2,V3> >::set(i) = v.get(i+V3::NUM_ENTS); + } + // If in first sub part V1 (also first in v)... + else if ( i<V1::NUM_ENTS ) { + // If not valid... + if ( v.get(i)==-1 ) continue; + // If v[i] intra-indexed to V1 or inter-indexed to cond... + else if ( v.get(i)<V1::NUM_ENTS || v.get(i)>=V1::NUM_ENTS+V3::NUM_ENTS+V2::NUM_ENTS ) + JointVecV<V1,JointVecV<V2,V3> >::set(i) = v.get(i); + // If v[i] inter-indexed to V3 (second in v)... + else if ( v.get(i)<V1::NUM_ENTS+V3::NUM_ENTS ) + JointVecV<V1,JointVecV<V2,V3> >::set(i) = v.get(i)+V2::NUM_ENTS; + // If v[i] inter-indexed to V2 (third in v) (note: may have gotten forwarded in commute)... + else JointVecV<V1,JointVecV<V2,V3> >::set(i) = get(v.get(i).toInt()-V3::NUM_ENTS); + } + //fprintf(stderr,"??????????? %d: ",i); v.write(stderr); fprintf(stderr," -> "); JointVecV<V1,JointVecV<V2,V3> >::write(stderr); fprintf(stderr,"\n"); + } + ////fprintf(stderr,"oNewComm\n"); //JointVecV<V2,V3>::write(stderr); fprintf(stderr,"\n"); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template<class V, class V1> +class AssociatedVecV : public V { + public: + AssociatedVecV ( const V1& v ) { + // Iterate through new coind vector... + for ( int i=0; i<V::NUM_ENTS; i++ ) + V::set(i) = v.get(i); + } +}; + +//////////////////////////////////////// + +template<class V1, class V2> +class MarginalVecV : public V1 { + public: + MarginalVecV ( const JointVecV<V1,V2>& v ) { + ////fprintf(stderr,"iMarg "); v.write(stderr); fprintf(stderr,"\n"); + // Iterate through new coind vector... + for ( int i=0; i<V1::NUM_ENTS; i++ ) { + // If not valid... + if ( v.get(i)==-1 ) continue; + // If intra-coref... + else if (v.get(i)<V1::NUM_ENTS) V1::set(i)=v.get(i); + // If inter-coref to marginalized var (assume target value must be self-coref or first coref would point there)... + else if (v.get(i)<V1::NUM_ENTS+V2::NUM_ENTS) { + for (int j=0; j<=i; j++) + if (v.get(j)==v.get(i)) V1::set(j)=i; + } + // If inter-coref to var after marginalized var... + else V1::set(i)=v.get(i)-V2::NUM_ENTS; + } + ////fprintf(stderr,"oMarg "); V1::write(stderr); fprintf(stderr,"\n"); + } +}; + +#endif /* _NL_DENOT__ */ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-dtree-cont.h b/contrib/synlm/hhmm/rvtl/include/nl-dtree-cont.h new file mode 100644 index 000000000..cf6b00d28 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-dtree-cont.h @@ -0,0 +1,479 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + + +#ifndef _NL_DTREE_CONTIN__ +#define _NL_DTREE_CONTIN__ + +#include "nl-dtree.h" + +typedef double Wt; + +//////////////////////////////////////////////////////////////////////////////// +// +// Cont DTree Node +// +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class P> +class ContDecisNode : public DecisNode<Y,P> { + private: + // Data members... + Wt wThreshold; // Threshold weight ("w_0") + map<A,Wt> awSeparator; // Hyperplane separator, weights on each attribute/dimension + Wt wSumSqr; // Sum of squares (parabolic) convolution coordinate weight + + public: + // Constructor / destructor methods... + ContDecisNode ( ) : wThreshold(0.0), wSumSqr(0.0) { } + + // Specification methods... + Wt& setWt ( ) { return wThreshold; } + Wt& setWt ( const A a ) { return (awSeparator.find(a)!=awSeparator.end()) ? awSeparator[a] : awSeparator[a]=0.0; } + Wt& setSsWt ( ) { return wSumSqr; } + + // Extraction methods... + const Wt getWt ( ) const { return wThreshold; } + const Wt getWt ( const A a ) const { return ( (awSeparator.find(a)!=awSeparator.end()) ? awSeparator.find(a)->second : 0.0 ); } + const Wt getSsWt ( ) const { return wSumSqr; } +}; + +//////////////////////////////////////////////////////////////////////////////// +// +// ContDTree Model +// +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X, class P> +class ContDTree2DModel : public Generic2DModel<Y,X,P>, public Tree<ContDecisNode<Y,P> > { + public: + // Downcasts (safe b/c no new data)... + ContDTree2DModel<Y,X,P>& setLeft() { return static_cast<ContDTree2DModel<Y,X,P>&> ( Tree<ContDecisNode<Y,P> >::setLeft() ); } + ContDTree2DModel<Y,X,P>& setRight() { return static_cast<ContDTree2DModel<Y,X,P>&> ( Tree<ContDecisNode<Y,P> >::setRight() ); } + const ContDTree2DModel<Y,X,P>& getLeft() const { return static_cast<const ContDTree2DModel<Y,X,P>&> ( Tree<ContDecisNode<Y,P> >::getLeft() ); } + const ContDTree2DModel<Y,X,P>& getRight() const { return static_cast<const ContDTree2DModel<Y,X,P>&> ( Tree<ContDecisNode<Y,P> >::getRight() ); } + // Extraction methods... + const P getProb ( const Y y, const X& x ) const { + const Tree<ContDecisNode<Y,P> >* ptr = this; + while ( !ptr->isTerm() ) { + double sumsqr=0.0; + for(A a;a<X::getSize();a.setNext()) sumsqr += pow(x.get(a.toInt()),2.0) / X::getSize(); + Wt wtdavg = -Tree<ContDecisNode<Y,P> >::getWt(); + for(A a;a<X::getSize();a.setNext()) wtdavg += Tree<ContDecisNode<Y,P> >::getWt(a) * x.get(a.toInt()); + wtdavg += Tree<ContDecisNode<Y,P> >::getSsWt() * sumsqr; + ptr = (wtdavg>0.0) ? &ptr->getRight() : &ptr->getLeft(); + } + return ptr->getProb(y); + } + // Input / output methods... + bool readFields ( char*[], int ) ; + void writeFields ( FILE* pf, string sPref ) { + char psPath[1000] = ""; + write ( pf, (sPref+"").c_str(), psPath, 0 ); + } + void write ( FILE* pf, const char psPrefix[], char psPath[], int iEnd ) const { + if (Tree<ContDecisNode<Y,P> >::isTerm()) { + Y y; + psPath[iEnd]='\0'; + for ( bool b=y.setFirst(); b; b=y.setNext() ) + { fprintf(pf, "%s [%s] : ", psPrefix, psPath); y.write(pf); fprintf(pf, " = %f\n", (double)Tree<ContDecisNode<Y,P> >::getProb(y)); } + ////psPath[iEnd]='\0'; fprintf ( pf, "%s [%s] : 0 = %f\n", psPrefix, psPath, (double)Tree<ContDecisNode<Y,P> >::getProb("0") ); + ////psPath[iEnd]='\0'; fprintf ( pf, "%s [%s] : 1 = %f\n", psPrefix, psPath, (double)Tree<ContDecisNode<Y,P> >::getProb("1") ); + } else { + psPath[iEnd]='\0'; fprintf ( pf, "%s [%s] =", psPrefix, psPath ); + fprintf ( pf, " %f", Tree<ContDecisNode<Y,P> >::getWt() ); + for(A a;a<X::getSize();a.setNext()) fprintf ( pf, "_%f", Tree<ContDecisNode<Y,P> >::getWt(a.toInt()) ); + fprintf ( pf, "_%f", Tree<ContDecisNode<Y,P> >::getSsWt() ); + fprintf ( pf, "\n" ); + psPath[iEnd]='0'; psPath[iEnd+1]='\0'; getLeft().write ( pf, psPrefix, psPath, iEnd+1 ); + psPath[iEnd]='1'; psPath[iEnd+1]='\0'; getRight().write ( pf, psPrefix, psPath, iEnd+1 ); + } + } +}; + +//////////////////// +template <class Y,class X, class P> +bool ContDTree2DModel<Y,X,P>::readFields ( char* aps[], int numFields ) { + if ( /*aps[0]==sId &&*/ (3==numFields || 4==numFields) ) { + //fprintf(stderr,"%s,%d\n",aps[3],numFields); + assert ( '['==aps[1][0] && ']'==aps[1][strlen(aps[1])-1] ); + + // Start at root... + Tree<ContDecisNode<Y,P> >* ptr = this; + assert(ptr); + + // Find appropriate node, creating nodes as necessary... + for(int i=1; i<strlen(aps[1])-1; i++) { + assert ( '0'==aps[1][i] || '1'==aps[1][i] ); + ptr = ( ('0'==aps[1][i]) ? &ptr->setLeft() : &ptr->setRight() ) ; + assert(ptr); + } + + // Specify bit (at nonterminal) or distribution (at terminal)... + if ( 3==numFields) { + char* psT=NULL; Tree<ContDecisNode<Y,P> >::setWt() = atof(strtok_r(aps[2],"_",&psT)); ////atof(aps[2]); + for(A a;a<X::getSize();a.setNext()) Tree<ContDecisNode<Y,P> >::setWt(a) = atof(strtok_r(NULL,"_",&psT)); + Tree<ContDecisNode<Y,P> >::setSsWt() = atof(strtok_r(NULL,"_",&psT)); } + // atof(aps[3+a.toInt()]); } + else if (4==numFields) ptr->setProb(aps[2]) = atof(aps[3]); + else assert(false); + + } else return false; + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X1,class X2, class P> +class ContDTree3DModel : public Generic3DModel<Y,X1,X2,P> { + private: + // Data members... + string sId; + SimpleHash<X1,ContDTree2DModel<Y,X2,P> > aqt; + public: + // Constructor / destructor methods... + ContDTree3DModel ( ) { } + ContDTree3DModel ( const string& s ) { sId = s; } + // Specification methods... + ContDTree2DModel<Y,X2,P>& setTree ( const X1& x1 ) { return aqt.set(x1); } + // Extraction methods... + bool setFirst ( Y& y ) const { return y.setFirst(); } + bool setNext ( Y& y ) const { return y.setNext(); } + P getProb ( const Y y, const X1& x1, const X2& x2 ) const { return aqt.get(x1).getProb(y,x2); } + // Input / output methods... + bool readFields ( char*[], int ) ; + void writeFields ( FILE* pf, string sPref ) { + char psPath[1000] = ""; + X1 x1; + for ( bool b=x1.setFirst(); b; b=x1.setNext() ) + aqt.get(x1).write ( pf, (sPref + " " + x1.getString()).c_str(), psPath, 0 ); + } +}; + +//////////////////// +template <class Y,class X1,class X2, class P> +bool ContDTree3DModel<Y,X1,X2,P>::readFields ( char* aps[], int numFields ) { + if ( /*aps[0]==sId &&*/ (4==numFields || 5==numFields) ) { + //fprintf(stderr,"%s,%d\n",aps[3],numFields); + assert ( '['==aps[2][0] && ']'==aps[2][strlen(aps[2])-1] ); + + // Start at root... + Tree<ContDecisNode<Y,P> >* ptr = &aqt.set(aps[1]); + assert(ptr); + + // Find appropriate node, creating nodes as necessary... + for(int i=1; i<strlen(aps[2])-1; i++) { + assert ( '0'==aps[2][i] || '1'==aps[2][i] ); + ptr = ( ('0'==aps[2][i]) ? &ptr->setLeft() : &ptr->setRight() ) ; + assert(ptr); + } + + // Specify bit (at nonterminal) or distribution (at terminal)... + if ( 4==numFields) { + char* psT=NULL; + ptr->setWt() = atof(strtok_r(aps[3],"_",&psT)); ////atof(aps[3]); + for(A a;a<X2::getSize();a.setNext()) ptr->setWt(a) = atof(strtok_r(NULL,"_",&psT)); + ptr->setSsWt() = atof(strtok_r(NULL,"_",&psT)); } + ////for(A a;a<X2::getSize();a.setNext()) ptr->setWt(a) = atof(aps[4+a.toInt()]); } + else if (5==numFields) ptr->setProb(aps[3]) = atof(aps[4]); + //// else if (5==numFields && 0==strcmp(aps[3],"0")) ptr->setProb() = 1.0 - atof(aps[4]); + //// else if (5==numFields && 0==strcmp(aps[3],"1")) ptr->setProb() = atof(aps[4]); + else assert(false); + + } else return false; + return true; +} + + +//////////////////////////////////////////////////////////////////////////////// +// +// Trainable ContDTree Model +// +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X, class P> +class TrainableContDTree2DModel : public ContDTree2DModel<Y,X,P> { + private: + List<Joint2DRV<X,Y> > lxy; + public: + // Downcasts (safe b/c no new data)... + TrainableContDTree2DModel<Y,X,P>& setLeft() { return static_cast<TrainableContDTree2DModel<Y,X,P>&>(ContDTree2DModel<Y,X,P>::setLeft()); } + TrainableContDTree2DModel<Y,X,P>& setRight() { return static_cast<TrainableContDTree2DModel<Y,X,P>&>(ContDTree2DModel<Y,X,P>::setRight()); } + const TrainableContDTree2DModel<Y,X,P>& getLeft() const { return static_cast<const TrainableContDTree2DModel<Y,X,P>&>(ContDTree2DModel<Y,X,P>::getLeft()); } + const TrainableContDTree2DModel<Y,X,P>& getRight() const { return static_cast<const TrainableContDTree2DModel<Y,X,P>&>(ContDTree2DModel<Y,X,P>::getRight()); } + // Specification methods... + void train ( List<Joint2DRV<X,Y> >&, const double ) ; + void train ( const double d ) { train(lxy,d); } + ////// Input / output methods... + bool readData ( char* vs[], int numFields ) { + if ( 3==numFields ) lxy.add() = Joint2DRV<X,Y> ( X(vs[1]), Y(vs[2]) ); + else return false; + return true; + } +}; + +//////////////////// +template<class Y, class X, class P> +void TrainableContDTree2DModel<Y,X,P>::train ( List<Joint2DRV<X,Y> >& lxy, const double DTREE_CHISQR_LIMIT ) { + + // Place to store counts... + //CPT3DModel<A,B,Y,double> aaaCounts; // hash was MUCH slower!! + SafeArray2D<B,Y,double> aaCounts ( 2, Y::getDomain().getSize(), 0.0 ); + double dTot = lxy.getCard(); + CPT1DModel<Y,double> modelY; + +// if (11613==dTot) { //if (12940<=dTot && dTot<12950) { //if ( 20779==dTot ) { //// (bU) +// ListedObject<Joint2DRV<X,Y> >* pxy; +// for ( pxy = lxy.getFirst(); pxy; pxy = lxy.getNext(pxy) ) { +// fprintf(stdout,"Ohist "); pxy->getSub1().write(stdout); fprintf(stdout," "); pxy->getSub2().write(stdout); fprintf(stdout,"\n"); +// } +// fprintf(stderr,"PRINTED\n"); +// } + + // For each datum in list... + ListedObject<Joint2DRV<X,Y> >* pxy; + for ( pxy = lxy.getFirst(); pxy; pxy = lxy.getNext(pxy) ) { + // Count Ys... + modelY.setProb(pxy->getSub2())++; + } + modelY.normalize(); + + double prRarest = (modelY.getProb("1")<modelY.getProb("0")) ? modelY.getProb("1") : modelY.getProb("0"); + +// // Set separator to pass through center of positives... +// for ( pxy = lxy.getFirst(); pxy; pxy = lxy.getNext(pxy) ) +// for ( A a; a<X::getSize(); a.setNext() ) { +// if ( Y("1")==pxy->getSub2() ) { +// Tree<ContDecisNode<Y,P> >::setWt() -= (pxy->getSub1().get(a.toInt())+pow(pxy->getSub1().get(a.toInt()),2.0))/dTot; //// (dTot*prRarest); +// Tree<ContDecisNode<Y,P> >::setWt(a) += pxy->getSub1().get(a.toInt())/dTot; //// (dTot*prRarest); +// Tree<ContDecisNode<Y,P> >::setSsWt() += pow(pxy->getSub1().get(a.toInt()),2.0)/dTot; //// (dTot*prRarest); +// } +// } + + // Set separator to pass through center of positives... + Tree<ContDecisNode<Y,P> >::setWt() = 1.0; + + + // For each gradient descent epoch... + for ( int epoch=1; epoch<=1000; epoch++ ) { + + double dCtr=0.0; + + double dPos = 0.0; + ListedObject<Joint2DRV<X,Y> >* pxy; + + if(OUTPUT_NOISY) { + double lgprTot = 0.0; +// // For each datum in list... +// for ( pxy = lxy.getFirst(); pxy; pxy = lxy.getNext(pxy) ) { +// // Calc tot prob... +// double wtdavg = -Tree<ContDecisNode<Y,P> >::getWt(); +// for(A a;a<X::getSize();a.setNext()) wtdavg += Tree<ContDecisNode<Y,P> >::getWt(a) * pxy->getSub1().get(a.toInt()); +// // Calc est val of Y using sigmoid transfer fn... +// P prY = 1.0 / ( 1.0 + exp(-wtdavg) ); +// if(epoch>1)fprintf(stderr," %f %f\n",(double)wtdavg,(double)prY); +// lgprTot += (pxy->getSub2()==1) ? log(prY) : log(1.0-prY) ; +// } + + if (OUTPUT_NOISY && epoch%10==0) { + // Report... + fprintf(stderr," tot=%08d totlogprob=%g separator=%f",(int)dTot,lgprTot,Tree<ContDecisNode<Y,P> >::getWt()); + for(A a;a<X::getSize();a.setNext()) fprintf(stderr,"_%f",Tree<ContDecisNode<Y,P> >::getWt(a)); + fprintf(stderr,"_%f",Tree<ContDecisNode<Y,P> >::getSsWt()); + fprintf(stderr,"\n"); + } + } + + fprintf(stderr," --- epoch %d ---\n",epoch); + + // For each datum in list... + for ( pxy = lxy.getFirst(); pxy; pxy = lxy.getNext(pxy) ) { +// // Use random subset of more frequent Y val so total wts for 1 and 0 are equal (CODE REVIEW: should be subset nearest to centroid of fewer)... +// if ( double(rand())/double(RAND_MAX) < prRarest/modelY.getProb(pxy->getSub2()) ) { + + dCtr++; + double gamma = dTot/(dTot+dCtr); // 1.0/(double(epoch)+dCtr/dTot); // 1.0/double(epoch); // 1.0/(double(epoch)+dCtr/(dTot*prRarest*2.0)); // + + // Weight deltas for next epoch... + Wt wDelta = 0.0; + SafeArray1D<A,Wt> awDeltas (X::getSize(),0.0); + Wt wSsDelta = 0.0; + + // Calc sum of squares for convolution coordinate... + double sumsqr=0.0; + for(A a;a<X::getSize();a.setNext()) sumsqr += pow(pxy->getSub1().get(a.toInt()),2.0) / X::getSize(); + + // Calc wtd avg of feats... + double wtdavg = -Tree<ContDecisNode<Y,P> >::getWt(); + for(A a;a<X::getSize();a.setNext()) wtdavg += Tree<ContDecisNode<Y,P> >::getWt(a) * pxy->getSub1().get(a.toInt()); + wtdavg += Tree<ContDecisNode<Y,P> >::getSsWt() * sumsqr; + //// Calc est val of Y using sigmoid transfer fn... + //P prY = ( ( ( (1.0/(1.0+exp(-wtdavg))) - .5 ) * exp(-wtdavg) ) + .5 ) ; + // Calc est val of Y using sigmoid transfer fn... + P prY = 1.0 / ( 1.0 + exp(-wtdavg) ); + + // Calc deltas for each feature/attribute/dimension... + double dEachWt = 1.0/dTot; // 1.0/dTot * modelY.getProb ( Y(1-pxy->getSub2().toInt()) ); // 1.0/(dTot*prRarest*2.0); // + wDelta += dEachWt * -1 * ( prY - P(double(pxy->getSub2().toInt())) ); + for ( A a; a<X::getSize(); a.setNext() ) + awDeltas.set(a) += dEachWt * pxy->getSub1().get(a.toInt()) * ( prY - P(double(pxy->getSub2().toInt())) ); + wSsDelta += dEachWt * sumsqr * ( prY - P(double(pxy->getSub2().toInt())) ); + + // Update weights by deltas... + //Tree<ContDecisNode<Y,P> >::setWt() -= gamma * wDelta; + ////double reldeduction = wDelta/Tree<ContDecisNode<Y,P> >::getWt(); + for ( A a; a<X::getSize(); a.setNext() ) + Tree<ContDecisNode<Y,P> >::setWt(a) -= gamma*awDeltas.get(a); //+ changeratio/Tree<ContDecisNode<Y,P> >::getWt(a); + Tree<ContDecisNode<Y,P> >::setSsWt() -= gamma*wSsDelta; //+ changeratio/Tree<ContDecisNode<Y,P> >::getSsWt(); + + dPos+=prY; // if (prY>0.5) dPos++; + + // Report... + if(OUTPUT_VERYNOISY) { + fprintf(stderr," A tot=%08d vals = %f",(int)dTot,-1.00); + for(A a;a<X::getSize();a.setNext()) fprintf(stderr,"_%f",pxy->getSub1().get(a.toInt())); + fprintf(stderr,"_%f",sumsqr); + fprintf(stderr," --> %f %f (gold: %f)\n",wtdavg,(double)prY,double(pxy->getSub2().toInt())); + fprintf(stderr," D tot=%08d delt = %f",(int)dTot,wDelta); + for(A a;a<X::getSize();a.setNext()) fprintf(stderr,"_%f",awDeltas.get(a)); + fprintf(stderr,"_%f",wSsDelta); + fprintf(stderr,"\n"); + } + + // Report... + if(OUTPUT_VERYNOISY) { + fprintf(stderr," _S tot=%08d sepr = %f",(int)dTot,Tree<ContDecisNode<Y,P> >::getWt()); + for(A a;a<X::getSize();a.setNext()) fprintf(stderr,"_%f",Tree<ContDecisNode<Y,P> >::getWt(a)); + fprintf(stderr,"_%f",Tree<ContDecisNode<Y,P> >::getSsWt()); + fprintf(stderr,"\n"); + } +// } + } // end loop pxy + +/* // Report... */ +/* if(OUTPUT_NOISY) { */ +/* fprintf(stderr," tot:%08d +:%08d -:%08d\n",(int)dTot,(int)dPos,(int)(dTot-dPos)); */ +/* fprintf(stderr," E tot=%08d separator=%f",(int)dTot,wDelta); */ +/* for(A a;a<X::getSize();a.setNext()) fprintf(stderr,"_%f",awDeltas.get(a)); */ +/* fprintf(stderr,"\n"); */ +/* } */ + } // end loop epoch + + // Split list into each 0/1 child of this node... + List<Joint2DRV<X,Y> > alxy[2]; + int actr[2] = {0,0}; + // For each datum in list... + while ( !lxy.isEmpty() ) { + Joint2DRV<X,Y>* pxy = lxy.getFirst(); + double sumsqr=0.0; + for(A a;a<X::getSize();a.setNext()) sumsqr += pow(pxy->getSub1().get(a.toInt()),2.0) / X::getSize(); + Wt wtdavg=-Tree<ContDecisNode<Y,P> >::getWt(); + for(A a;a<X::getSize();a.setNext()) wtdavg += Tree<ContDecisNode<Y,P> >::getWt(a) * pxy->getSub1().get(a.toInt()); + wtdavg += Tree<ContDecisNode<Y,P> >::getSsWt() * sumsqr; + alxy[(wtdavg>0.0)?1:0].add() = *pxy; + aaCounts.set((wtdavg>0.0)?1:0,pxy->getSub2())++; + actr[(wtdavg>0.0)?1:0]++; + if(OUTPUT_VERYNOISY){fprintf(stderr,"classify "); pxy->write(stderr); fprintf(stderr," wtdavg=%f class=%d\n",wtdavg,(wtdavg>0.0)?1:0);} + lxy.pop(); + } + + // Calc chisqr... + double chisqr = 0.0; + fprintf(stderr," tot=%08d split=",(int)dTot); + for ( int b=0; b<2; b++ ) { + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) { + fprintf(stderr," (%s->%d:%f)",y.getString().c_str(),b,aaCounts.get(b,y)); + if ( actr[b]>0.0 && modelY.getProb(y)>0.0 && dTot>0.0 ) { + double expect = actr[b] * modelY.getProb(y); + chisqr += pow ( aaCounts.get(b,y)-expect, 2.0 ) / expect; + } + } + } + fprintf(stderr,"\n"); + + // Report... + if(OUTPUT_NOISY) { + fprintf(stderr," tot=%08d separator=%f",(int)dTot,Tree<ContDecisNode<Y,P> >::getWt()); + for(A a;a<X::getSize();a.setNext()) fprintf(stderr,"_%f",Tree<ContDecisNode<Y,P> >::getWt(a)); + fprintf(stderr,"_%f",Tree<ContDecisNode<Y,P> >::getSsWt()); + fprintf(stderr," chisqr=%g\n",chisqr); + } + + // If separation is significant to chisqr limit... + if ( chisqr > DTREE_CHISQR_LIMIT ) { + // Recursively call train at each child... + setRight().train ( alxy[1], DTREE_CHISQR_LIMIT ); ////node*2LL+1LL); + setLeft().train ( alxy[0], DTREE_CHISQR_LIMIT ); ////node*2LL); + } + // If separation is not significant... + else { + // Add ratio as leaf... + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) + ContDecisNode<Y,P>::setProb(y) = (dTot>0.0) ? modelY.getProb(y) : 1.0/Y::getDomain().getSize(); + } +} + + +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X1, class X2, class P> +class TrainableContDTree3DModel : public ContDTree3DModel<Y,X1,X2,P> { + + private: + + map<X1,List<Joint2DRV<X2,Y> > > mqlxy; + + public: + + ////// Constructor... + TrainableContDTree3DModel() { } + TrainableContDTree3DModel(const char* ps) : ContDTree3DModel<Y,X1,X2,P>(ps) { } + + ////// setTree downcast... + TrainableContDTree2DModel<Y,X2,P>& setTree(const X1& x1) { return static_cast<TrainableContDTree2DModel<Y,X2,P>&>(ContDTree3DModel<Y,X1,X2,P>::setTree(x1)); } + + ////// Add training data to per-subphone lists... + bool readData ( char* vs[], int numFields ) { + if ( 4==numFields ) { + mqlxy[X1(vs[1])].add() = Joint2DRV<X2,Y> ( X2(vs[2]), Y(vs[3]) ); + ////mqlxy[X1(vs[1])].getLast()->write(stderr); fprintf(stderr,"\n"); + } + else return false; + return true; + } + + ////// Train each subphone... + void train ( const double DTREE_CHISQR_LIMIT ) { + int ctr = 0; + // For each subphone... + X1 x1; for ( bool b=x1.setFirst(); b; b=x1.setNext() ) { + if(OUTPUT_NOISY) + fprintf(stderr,"***** x1:%s (number %d) *****\n",x1.getString().c_str(),ctr++); + setTree(x1).train ( mqlxy[x1], DTREE_CHISQR_LIMIT ); + } + } +}; + +#endif // _NL_DTREE_CONTIN__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-dtree.h b/contrib/synlm/hhmm/rvtl/include/nl-dtree.h new file mode 100644 index 000000000..2396f395c --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-dtree.h @@ -0,0 +1,530 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_DTREE__ +#define _NL_DTREE__ + +#include <math.h> +#include <vector> +#include <string> +#include <map> +#include "nl-safeids.h" +//#include "nl-probmodel.h" +#include "nl-cpt.h" +#include "nl-randvar.h" +#include "nl-tree.h" +#include "nl-hash.h" +#include "nl-list.h" + +//#define UNIGR_FO .5 // IF NO EXAMPLES, USE APRIORI PROB OF FO + +//typedef Id<int> A; //BitNum; +// //typedef Id<char> B; + +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// +// +// DTree Node +// +//////////////////////////////////////////////////////////////////////////////// + +template<class X, class Y, class P> +class DecisNode { + public: + // Public types... + typedef Id<int> A; + + private: + // Private types... + typedef typename X::ElementType B; + // Data members... + A aNontermDecis; // Nonterminal nodes have an attribute (e.g. convexity bit) on which to condition + map<Y,P> aprTermDistrib; // Terminal nodes have a distribution over Y values + + public: + // Constructor / destructor methods... + DecisNode ( ) : aNontermDecis(-1) { } + + // Specification methods... + A& setA ( ) { return aNontermDecis; } + P& setProb ( const Y y ) { return aprTermDistrib[y]; } + + // Extraction methods... + const A getA ( ) const { return aNontermDecis; } + const P getProb ( const Y y ) const { return ( (aprTermDistrib.empty()) ? P(1.0/Y::getDomain().getSize()) : + (aprTermDistrib.find(y)!=aprTermDistrib.end()) ? aprTermDistrib.find(y)->second : P() ); } +}; + + +//////////////////////////////////////////////////////////////////////////////// +// +// DTree Model +// +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X, class P> +//class DTree2DModel : public Generic2DModel<Y,X,P>, public Tree < typename X::ElementType, DecisNode<X,Y,P> > { +class DTree2DModel : public Tree < typename X::ElementType, DecisNode<X,Y,P> > { + private: + // Type members... + typedef typename X::ElementType B; + public: + // Downcasts (safe b/c no new data)... + DTree2DModel<Y,X,P>& setBranch(const B& b) { return static_cast<DTree2DModel<Y,X,P>&> ( Tree<B,DecisNode<X,Y,P> >::setBranch(b) ); } + const DTree2DModel<Y,X,P>& getBranch(const B& b) const { return static_cast<const DTree2DModel<Y,X,P>&> ( Tree<B,DecisNode<X,Y,P> >::getBranch(b) ); } + // Extraction methods... + const P getProb ( const Y y, const X& x ) const { + const Tree<B,DecisNode<X,Y,P> >* ptr = this; + while ( !ptr->isTerm() ) /*{cerr<<x.get(ptr->getA().toInt())<<",";*/ ptr = &ptr->getBranch ( x.get(ptr->getA().toInt()) ); /*}*/ + return ptr->getProb(y); + } + // Input / output methods... + bool readFields ( Array<char*>& ) ; + void write ( FILE* pf, const char psPrefix[], char psPath[], int iEnd ) const { + if (Tree<B,DecisNode<X,Y,P> >::isTerm()) { + psPath[iEnd]='\0'; + Y y; + for ( bool b=y.setFirst(); b; b=y.setNext() ) + { fprintf(pf, "%s [%s] : ", psPrefix, psPath); fprintf(pf,"%s",y.getString().c_str()); fprintf(pf, " = %f\n", Tree<B,DecisNode<X,Y,P> >::getProb(y).toDouble()); } + ////psPath[iEnd]='\0'; fprintf ( pf, "%s [%s] : 0 = %f\n", psPrefix, psPath, (double)Tree<DecisNode<Y,P> >::getProb("0") ); + ////psPath[iEnd]='\0'; fprintf ( pf, "%s [%s] : 1 = %f\n", psPrefix, psPath, (double)Tree<DecisNode<Y,P> >::getProb("1") ); + } else { + psPath[iEnd]='\0'; fprintf ( pf, "%s [%s] = %d\n", psPrefix, psPath, (int)Tree<B,DecisNode<X,Y,P> >::getA().toInt() ); + B b; + for ( bool bb=b.setFirst(); bb; bb=b.setNext() ) { + psPath[iEnd]=b.getString().c_str()[0]; psPath[iEnd+1]='\0'; getBranch(b).write(pf,psPrefix,psPath,iEnd+1); +// psPath[iEnd]='0'; psPath[iEnd+1]='\0'; getLeft().write ( pf, psPrefix, psPath, iEnd+1 ); +// psPath[iEnd]='1'; psPath[iEnd+1]='\0'; getRight().write ( pf, psPrefix, psPath, iEnd+1 ); + } + } + } + void writeFields ( FILE* pf, string sPref ) { + char psPath[1000] = ""; + write ( pf, (sPref+"").c_str(), psPath, 0 ); + } + //// + friend pair<StringInput,DTree2DModel<Y,X,P>*> operator>> ( StringInput si, DTree2DModel<Y,X,P>& m ) { + return pair<StringInput,DTree2DModel<Y,X,P>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,DTree2DModel<Y,X,P>*> si_m, const char* psD ) { + if (StringInput(NULL)==si_m.first) return si_m.first; + Y y; String xs; StringInput si,si2; si=si_m.first; DTree2DModel<Y,X,P>* pm=si_m.second; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>xs>>" "; + while((si2=si>>" ")!=NULL)si=si2; + // Find appropriate node, creating nodes as necessary... + for(int i=1; i<int(strlen(xs.c_array()))-1; i++) { + char psTemp[2]=" "; psTemp[0]=xs.c_array()[i]; + pm = &pm->setBranch ( B(psTemp) ); + } + + if ( si!=NULL && si[0]==':' ) { + si=si>>": "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>y>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>"= "; + while((si2=si>>" ")!=NULL)si=si2; + // Specify attribute number (at nonterminal) or probability in distribution (at terminal)... + return (si!=NULL) ? si>>pm->setProb(y)>>psD : si; + } + else if ( si!=NULL && si[0]=='=' ) { + si=si>>"= "; //cerr<<" in after equals "<<((si==NULL) ? "yes" : "no") << endl; + while((si2=si>>" ")!=NULL)si=si2; + + //m.setA() = atoi(si.c_str()); + int aVar = 0; + si=si>>aVar>>psD; + pm->setA()=aVar; + ////cerr<<" at end "<<((si==NULL) ? "yes" : "no") << endl; + ////cerr<<" m.getA() is "<< m.getA().toInt() << endl; + return si; + //return (si!=NULL) ? si>>m.setA()>>psD : si; + } + else if ( si!=NULL ) cerr<<" ??? ["<<si.c_str()<<"]\n"; + return StringInput(NULL); + + /* + Y y; String xs; X x; StringInput si,si2; string sRt; DTree2DModel<Y,X,P>& m = *si_m.second; + si=si_m.first; + sRt = si.c_str(); + if (sRt.find(':')!=string::npos) { + while((si2=si>>" [")!=NULL)si=si2; + si=si>>xs>>"] "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>": "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>y>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>"= "; + + // For DTree, must find the node labeled by X + //Tree<B,DecisNode<X,Y,P> >* ptr = m; + //assert(ptr); + // Find appropriate node, creating nodes as necessary... + for(int i=1; i<int(strlen(xs.c_array()))-1; i++) { + char psTemp[2]="\0"; psTemp[0]=xs.c_array()[i]; + m = m.setBranch ( B(psTemp) ); + } + // Specify attribute number (at nonterminal) or probability in distribution (at terminal)... + return (si!=NULL) ? si>>m.setProb(y)>>psD : si; + } else { + while((si2=si>>" [")!=NULL)si=si2; + si=si>>xs>>"] "; //cerr<<" in bracket "<<((si==NULL) ? "yes" : "no") << endl; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>"= "; //cerr<<" in after equals "<<((si==NULL) ? "yes" : "no") << endl; + + //m.setA() = atoi(si.c_str()); + int aVar = 0; + si=si>>aVar>>psD; + m.setA()=aVar; + //cerr<<" at end "<<((si==NULL) ? "yes" : "no") << endl; + //cerr<<" m.getA() is "<< m.getA().toInt() << endl; + return si; + //return (si!=NULL) ? si>>m.setA()>>psD : si; + } + */ + } + //// +}; + +//////////////////// +template <class Y,class X, class P> +bool DTree2DModel<Y,X,P>::readFields ( Array<char*>& aps ) { + if ( /*aps[0]==sId &&*/ (3==aps.size() || 4==aps.size()) ) { + //fprintf(stderr,"%s,%d\n",aps[3],numFields); + assert ( '['==aps[1][0] && ']'==aps[1][strlen(aps[1])-1] ); + + // Start at root... + Tree<B,DecisNode<X,Y,P> >* ptr = this; + assert(ptr); + + // Find appropriate node, creating nodes as necessary... + for(int i=1; i<int(strlen(aps[1]))-1; i++) { + char psTemp[2]="\0"; psTemp[0]=aps[1][i]; + ptr = &ptr->setBranch ( B(psTemp) ); +// assert ( '0'==aps[1][i] || '1'==aps[1][i] ); +// ptr = ( ('0'==aps[1][i]) ? &ptr->setLeft() : &ptr->setRight() ) ; +// assert(ptr); + } + + // Specify attribute number (at nonterminal) or probability in distribution (at terminal)... + if (3==aps.size()) ptr->setA() = atoi(aps[2]); + else if (4==aps.size()) ptr->setProb(aps[2]) = atof(aps[3]); + else assert(false); + + } else return false; + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X1,class X2, class P> +//class DTree3DModel : public Generic3DModel<Y,X1,X2,P> { +class DTree3DModel { + private: + // Type members... + typedef typename X2::ElementType B; + // Data members... + string sId; + SimpleHash<X1,DTree2DModel<Y,X2,P> > aqt; + public: + // Constructor / destructor methods... + DTree3DModel ( ) { } + DTree3DModel ( const string& s ) { sId = s; } + // Specification methods... + DTree2DModel<Y,X2,P>& setTree ( const X1& x1 ) { return aqt.set(x1); } + // Extraction methods... + bool setFirst ( Y& y ) const { return y.setFirst(); } + bool setNext ( Y& y ) const { return y.setNext(); } + P getProb ( const Y y, const X1& x1, const X2& x2 ) const { return aqt.get(x1).getProb(y,x2); } + // Input / output methods... + bool readFields ( char*[], int ) ; + void writeFields ( FILE* pf, string sPref ) { + char psPath[1000] = ""; + X1 x1; + for ( bool b=x1.setFirst(); b; b=x1.setNext() ) + aqt.get(x1).write ( pf, (sPref + " " + x1.getString()).c_str(), psPath, 0 ); + } +}; + +//////////////////// +template <class Y,class X1,class X2, class P> +bool DTree3DModel<Y,X1,X2,P>::readFields ( char* aps[], int numFields ) { + if ( /*aps[0]==sId &&*/ (4==numFields || 5==numFields) ) { + //fprintf(stderr,"%s,%d\n",aps[3],numFields); + assert ( '['==aps[2][0] && ']'==aps[2][strlen(aps[2])-1] ); + + // Start at root... + Tree<B,DecisNode<X2,Y,P> >* ptr = &aqt.set(aps[1]); + assert(ptr); + + // Find appropriate node, creating nodes as necessary... + for(int i=1; i<strlen(aps[2])-1; i++) { + char psTemp[2]="\0\0"; psTemp[0]=aps[1][i]; + ptr = ptr->setBranch ( B(psTemp) ); +// assert ( '0'==aps[2][i] || '1'==aps[2][i] ); +// ptr = ( ('0'==aps[2][i]) ? &ptr->setLeft() : &ptr->setRight() ) ; +// assert(ptr); + } + + // Specify bit (at nonterminal) or distribution (at terminal)... + if (4==numFields) ptr->setA() = atoi(aps[3]); + else if (5==numFields) ptr->setProb(aps[3]) = atof(aps[4]); + //// else if (5==numFields && 0==strcmp(aps[3],"0")) ptr->setProb() = 1.0 - atof(aps[4]); + //// else if (5==numFields && 0==strcmp(aps[3],"1")) ptr->setProb() = atof(aps[4]); + else assert(false); + + } else return false; + return true; +} + + +//////////////////////////////////////////////////////////////////////////////// +// +// Trainable DTree Model +// +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X, class P> +class TrainableDTree2DModel : public DTree2DModel<Y,X,P> { + private: + // Type members... + typedef typename X::ElementType B; + // Static data members... + static List<Joint2DRV<X,Y> > lxyInitial; + public: + // Static member varaibles... + static bool OUTPUT_NOISY; + static bool OUTPUT_VERYNOISY; + // Downcasts (safe b/c no new data)... + TrainableDTree2DModel<Y,X,P>& setBranch(const B& b) { return static_cast<TrainableDTree2DModel<Y,X,P>&> ( Tree<B,DecisNode<X,Y,P> >::setBranch(b) ); } + const TrainableDTree2DModel<Y,X,P>& getBranch(const B& b) const { return static_cast<const TrainableDTree2DModel<Y,X,P>&> ( Tree<B,DecisNode<X,Y,P> >::getBranch(b) ); } + // Specification methods... + void train ( List<Joint2DRV<X,Y> >&, const DecisNode<X,Y,P>&, const double ) ; + void train ( const double d ) { train(lxyInitial,DecisNode<X,Y,P>(),d); } + ////// Input / output methods... + bool readData ( Array<char*>& aps ) { + if ( 3==aps.size() ) lxyInitial.add() = Joint2DRV<X,Y> ( X(aps[1]), Y(aps[2]) ); + else if ( 4==aps.size() ) { + for ( int i=atoi(aps[3]); i>0; i-- ) + lxyInitial.add() = Joint2DRV<X,Y> ( X(aps[1]), Y(aps[2]) ); + } + else return false; + return true; + } +}; +template <class Y, class X, class P> List<Joint2DRV<X,Y> > TrainableDTree2DModel<Y,X,P>::lxyInitial; +template <class Y, class X, class P> bool TrainableDTree2DModel<Y,X,P>::OUTPUT_NOISY = false; +template <class Y, class X, class P> bool TrainableDTree2DModel<Y,X,P>::OUTPUT_VERYNOISY = false; + +//////////////////// +template<class Y, class X, class P> +void TrainableDTree2DModel<Y,X,P>::train ( List<Joint2DRV<X,Y> >& lxy, const DecisNode<X,Y,P>& dnParent, const double DTREE_CHISQR_LIMIT ) { + + typedef typename DecisNode<X,Y,P>::A A; + + // Place to store counts... + //CPT3DModel<A,B,Y,double> aaaCounts; // hash was MUCH slower!! + SafeArray3D<A,B,Y,double> aaaCounts ( X::getSize(), B::getDomain().getSize(), Y::getDomain().getSize(), 0.0 ); + double dTot = lxy.getCard(); + CPT1DModel<Y,double> modelY; // ( "Y_prior" ); + + // For each datum in list... + ListedObject<Joint2DRV<X,Y> >* pxy; + for ( pxy = lxy.getFirst(); pxy; pxy = lxy.getNext(pxy) ) { + // For each attribute position... + for ( A a=0; a<X::getSize(); a.setNext() ) + // Add to counts... + aaaCounts.set ( a, pxy->first.get(a.toInt()), pxy->second )++; //(pxy->second==Y("1"))?1:0 )++; + modelY.setProb(pxy->second)++; + } + modelY.normalize(); + +// // If best attribute's prediction is not significant... +// else { + // Add ratio as leaf... + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) + DecisNode<X,Y,P>::setProb(y) = (dTot>100) ? modelY.getProb(y) : (double)dnParent.getProb(y); //1.0/Y::getDomain().getSize(); + //DecisNode<Y,P>::setProb("0") = (dXX>0) ? dX0/dXX : UNIGR_FO; + //DecisNode<Y,P>::setProb("1") = (dXX>0) ? dX1/dXX : UNIGR_FO; + ////DTree3DModel<Y,X1,X,P>::setLeafModel().setProb(Y("0"),x1,node) = dX0/dXX; + ////DTree3DModel<Y,X1,X,P>::setLeafModel().setProb(Y("1"),x1,node) = dX1/dXX; +// } + + double chisqr = 0.0; + A aBest=0; + + // Bail if will never be significant... + if ( !lxy.isEmpty() && lxy.getCard()>1000 ) { + + // For each attribute position... + double entBest=0.0; + for ( A a=0; a<X::getSize(); a.setNext() ) { + + // Local model for each attrib (bit num)... + CPT2DModel<Y,B,double> modelY_giv_B; // ( "Y_giv_B" ); + CPT1DModel<B,double> modelB; // ( "B_prior" ); + B b; + for ( bool bb=b.setFirst(); bb; bb=b.setNext() ) { + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) { + modelY_giv_B.setProb(y,b) = aaaCounts.get(a,b,y); + modelB.setProb(b) += aaaCounts.get(a,b,y); + } + } + modelY_giv_B.normalize(); + modelB.normalize(); + + // Calc entropy... + double ent = 0.0; + for ( bool bb=b.setFirst(); bb; bb=b.setNext() ) { + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) + ent -= (0.0==modelY_giv_B.getProb(y,b)) ? 0.0 : ( modelB.getProb(b) * modelY_giv_B.getProb(y,b) * log(modelY_giv_B.getProb(y,b)) ); + } + + // Record minimum entropy division... + if ( a==0 || ent<entBest ) { entBest=ent; aBest=a; } + if(OUTPUT_VERYNOISY) { +// fprintf(stderr," bit=%d ent=%g (%g:%g,%g:%g) (%g:%g,%g:%g,%g:%g)\n",(int)a,ent, +// aaaCounts.getProb(a,0,0),aaaCounts.getProb(a,0,1),aaaCounts.getProb(a,1,0),aaaCounts.getProb(a,1,1), +// modelB.getProb(0),modelB.getProb(1), +// modelY_giv_B.getProb(0,0),modelY_giv_B.getProb(1,0),modelY_giv_B.getProb(0,1),modelY_giv_B.getProb(1,1)); +// //d0X,d1X,d00giv0X,d01giv0X,d10giv1X,d11giv1X); + fprintf(stderr," bit=%d ent=%g\n",a.toInt(),ent); + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) + fprintf(stderr," \t%s\t%s%.1f %s%.1f %s%.1f %s%.1f %s%.1f %s%.1f ...\n",y.getString().c_str(), + B(0).getString().c_str(),modelY_giv_B.getProb(y,0), + B(1).getString().c_str(),modelY_giv_B.getProb(y,1), + B(2).getString().c_str(),modelY_giv_B.getProb(y,2), + B(3).getString().c_str(),modelY_giv_B.getProb(y,3), + B(4).getString().c_str(),modelY_giv_B.getProb(y,4), + B(5).getString().c_str(),modelY_giv_B.getProb(y,5)); + //fprintf(stderr," %s %g:%g\n",y.getString().c_str(),modelY_giv_B.getProb(y,0),modelY_giv_B.getProb(y,1)); + } + } + + if(OUTPUT_NOISY) + fprintf(stderr," tot=%08d bestbit=%d bestent=%g\n",(int)dTot,aBest.toInt(),entBest); + + chisqr = 0.0; + if ( X::getSize()>0 ) { + // Local model for each attrib (bit num)... + CPT1DModel<B,double> modelB_giv_Abest; // ( "B_giv_Abest" ); + B b; + for ( bool bb=b.setFirst(); bb; bb=b.setNext() ) { + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) { + modelB_giv_Abest.setProb(b) += aaaCounts.get(aBest,b,y); + } + } + modelB_giv_Abest.normalize(); + + // Calc chi sqr... + for ( bool bb=b.setFirst(); bb; bb=b.setNext() ) { + Y y; + for ( bool by=y.setFirst(); by; by=y.setNext() ) { + if ( modelB_giv_Abest.getProb(b)>0 && modelY.getProb(y)>0 && dTot>0 ) { + double exp = modelB_giv_Abest.getProb(b) * modelY.getProb(y) * dTot; + chisqr += pow ( aaaCounts.get(aBest,b,y)-exp, 2 ) / exp; + } + } + } + } + + if(OUTPUT_NOISY) + fprintf(stderr," chisqr=%g\n",chisqr); + } + + // If best bit prediction is significant... + if ( chisqr > DTREE_CHISQR_LIMIT ) { + //// // Add node to model fields... + //// DTree3DModel<Y,X1,X,P>::setNodeMap()[Joint2DRV<X1,NodeNum>(x1,node)] = aBest; + // Split list into each 0/1 child of this node... + SimpleHash<B,List<Joint2DRV<X,Y> > > alxy; + // For each datum in list... + while ( !lxy.isEmpty() ) { + Joint2DRV<X,Y>* pxy = lxy.getFirst(); + alxy[pxy->first.get(aBest.toInt())].add() = *pxy; + lxy.pop(); + } + // Recursively call train at each child... + DecisNode<X,Y,P>::setA()=aBest; + B b; + for ( bool bb=b.setFirst(); bb; bb=b.setNext() ) + setBranch(b).train ( alxy[b], *this, DTREE_CHISQR_LIMIT ); + // setLeft().train ( alxy[0], DTREE_CHISQR_LIMIT ); ////node*2LL); + // setRight().train ( alxy[1], DTREE_CHISQR_LIMIT ); ////node*2LL+1LL); + } +} + + +//////////////////////////////////////////////////////////////////////////////// + +template<class Y, class X1, class X2, class P> +class TrainableDTree3DModel : public DTree3DModel<Y,X1,X2,P> { + + private: + + map<X1,List<Joint2DRV<X2,Y> > > mqlxy; + + public: + + // Static member varaibles... + static bool OUTPUT_NOISY; + + ////// Constructor... + TrainableDTree3DModel(const char* ps) : DTree3DModel<Y,X1,X2,P>(ps) { } + + ////// setTree downcast... + TrainableDTree2DModel<Y,X2,P>& setTree(const X1& x1) { return static_cast<TrainableDTree2DModel<Y,X2,P>&>(DTree3DModel<Y,X1,X2,P>::setTree(x1)); } + + ////// Add training data to per-subphone lists... + bool readData ( Array<char*>& aps ) { + if ( 4==aps.size() ) { + mqlxy[X1(aps[1])].add() = Joint2DRV<X2,Y> ( X2(aps[2]), Y(aps[3]) ); + ////mqlxy[X1(aps[1])].getLast()->write(stderr); fprintf(stderr,"\n"); + } + else return false; + return true; + } + + ////// Train each subphone... + void train ( const double DTREE_CHISQR_LIMIT ) { + int ctr = 0; + // For each subphone... + X1 x1; for ( bool b=x1.setFirst(); b; b=x1.setNext() ) { + if(OUTPUT_NOISY) + fprintf(stderr,"***** x1:%s (number %d) *****\n",x1.getString().c_str(),ctr++); + setTree(x1).train ( mqlxy[x1], DTREE_CHISQR_LIMIT ); + } + } +}; + +template <class Y, class X1, class X2, class P> bool TrainableDTree3DModel<Y,X1,X2,P>::OUTPUT_NOISY = false; + + +#endif // _NL_DTREE__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-fixedmatrix.h b/contrib/synlm/hhmm/rvtl/include/nl-fixedmatrix.h new file mode 100644 index 000000000..dbb9d9d9d --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-fixedmatrix.h @@ -0,0 +1,242 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + + +//basically a SafeArray2D with operators defined +template<class T> +class Matrix : public SafeArray2D<Id<int>,Id<int>,T> { + //public: + //int xSize; + //int ySize; + public: + // Constructor / destructor methods... + //~Matrix( ) { delete[] at; } + Matrix ( ) : SafeArray2D<Id<int>,Id<int>,T>( ) { }//{ xSize=0; ySize=0; } + Matrix (int x, int y) : SafeArray2D<Id<int>,Id<int>,T>(x,y) { }//{ xSize=x; ySize=y; } + Matrix (int x, int y, const T& t) : SafeArray2D<Id<int>,Id<int>,T>(x,y,t) { }//{ xSize=x; ySize=y; } + Matrix (const Matrix& a) : SafeArray2D<Id<int>,Id<int>,T>(a.xSize(),a.ySize()) { //xSize=a.xSize; ySize=a.ySize; + for(int i=0;i<xSize();i++) for(int j=0;j<ySize();j++) this->set(i,j)=a.get(i,j); } + // Specification methods... + //Matrix& operator= ( const Matrix<T>& sat ) + // { xSize=sat.xSize; ySize=sat.ySize; //at=new T[xSize*ySize]; + // for(int i=0;i<xSize;i++) for(int j=0;j<ySize;j++) set(i,j)=sat.at[i]; return *this; } + void init ( int x,int y ) { (*this)=Matrix<T>(x,y,T()); }//xSize=x; ySize=y; } + void init ( int x,int y,const T& t ) { (*this)=Matrix<T>(x,y,t); }//xSize=x; ySize=y; } + void reset() { (*this)=Matrix<T>( ); }//xSize=0; ySize=0; } + + // Inherited methods + //T& set ( const X1& x,const X2& y); + //const T& get (const X1& x,const X2& y) const; + + int xSize( ) const { return this->getxSize(); } + int ySize( ) const { return this->getySize(); } + + // Math... + friend Matrix<T> operator* ( const Matrix<T>& a, const Matrix<T>& b ) { + if (a.ySize()!=b.xSize()) { + cerr<<"ERROR: matrix multiplication requires matching inner indices; "<<a.xSize()<<"x"<<a.ySize()<<" "<<b.xSize()<<"x"<<b.ySize()<<endl; + #ifndef NDEBUG + cerr<<" a= "<<a<<"\n\n b= "<<b<<endl; + #endif + return Matrix<T>(); + } + Matrix mOut(a.xSize(),b.ySize(),T()); + for (int i=0; i<a.xSize(); i++ ){ + for (int k=0; k<a.ySize(); k++ ) { + for (int j=0; j<b.ySize(); j++ ) { + mOut.set(i,j) += a.get(Id<int>(i),Id<int>(k))*b.get(Id<int>(k),Id<int>(j)); + } + } + } + //cerr<<" a= "<<a<<"\n b= "<<b<<"\n c= "<<mOut<<endl<<endl; + return mOut; + } + friend Matrix<T> operator& ( const Matrix<T>& a, const Matrix<T>& b ) { + if (a.xSize()!=b.xSize() || a.ySize()!=b.ySize()) { + cerr<<"ERROR: pt-by-pt multiplication requires matching indices; "<<a.xSize()<<"x"<<a.ySize()<<" "<<b.xSize()<<"x"<<b.ySize()<<endl; + #ifndef NDEBUG + cerr<<" a= "<<a<<"\n\n b= "<<b<<endl; + #endif + return Matrix<T>(); + } + Matrix mOut(a.xSize(),a.ySize(),T()); + for (int i=0; i<a.xSize(); i++ ){ + for (int j=0; j<b.ySize(); j++ ) { + mOut.set(i,j) += a.get(Id<int>(i),Id<int>(j))*b.get(Id<int>(i),Id<int>(j)); + } + } + //cerr<<" a= "<<a<<"\n b= "<<b<<"\n c= "<<mOut<<endl<<endl; + return mOut; + } + friend Matrix<T> operator+ ( const Matrix<T>& a, const Matrix<T>& b ) { + if (a.xSize()!=b.xSize() || a.ySize()!=b.ySize()) { + cerr<<"ERROR: matrix addition requires matching dimensions"<<endl; + return Matrix<T>(); + } + Matrix mOut(a.xSize(),b.ySize(),T()); + for (int i=0; i<a.xSize(); i++ ){ + for (int j=0; j<a.ySize(); j++ ) { + mOut.set(i,j) = a.get(Id<int>(i),Id<int>(j))+b.get(Id<int>(i),Id<int>(j)); + } + } + return mOut; + } + friend Matrix<T> operator- ( const Matrix<T>& a, const Matrix<T>& b ) { + if (a.xSize()!=b.xSize() || a.ySize()!=b.ySize()) { + cerr<<"ERROR: matrix subtraction requires matching dimensions"<<endl; + //cerr<<"aSize="<<a.xSize<<","<<a.ySize()<<" bSize="<<b.xSize<<","<<b.ySize()<<endl; + //cerr<<" a= "<<a<<"\n b= "<<b<<"\n c= "<<mOut<<endl<<endl; + return Matrix<T>(); + } + Matrix mOut(a.xSize(),b.ySize(),T()); + for (int i=0; i<a.xSize(); i++ ){ + for (int j=0; j<a.ySize(); j++ ) { + mOut.set(i,j) = a.get(Id<int>(i),Id<int>(j))-b.get(Id<int>(i),Id<int>(j)); + } + } + return mOut; + } + friend Matrix<T> operator* ( const Matrix<T>& a, const T& t ) { + Matrix mOut(a.xSize(),a.ySize()); + for (int i=0; i<a.xSize(); i++ ){ + for (int j=0; j<a.ySize(); j++ ) { + mOut.set(i,j) = a.get(Id<int>(i),Id<int>(j))*t; + } + } + return mOut; + } + friend Matrix<T> operator+ ( const Matrix<T>& a, const T& t ) { + Matrix mOut(a.xSize(),a.ySize()); + for (int i=0; i<a.xSize(); i++ ){ + for (int j=0; j<a.ySize(); j++ ) { + mOut.set(i,j) = a.get(Id<int>(i),Id<int>(j))+t; + } + } + return mOut; + } + friend Matrix<T> operator- ( const Matrix<T>& a, const T& t ) { + Matrix mOut(a.xSize(),a.ySize()); + for (int i=0; i<a.xSize(); i++ ){ + for (int j=0; j<a.ySize(); j++ ) { + mOut.set(i,j) = a.get(Id<int>(i),Id<int>(j))-t; + } + } + return mOut; + } + + // Scalar inf-norm (max) of matrix / vector... + T infnorm ( ) const { + T tOut = T(); + for (int i=0; i<xSize(); i++ ){ + for (int j=0; j<ySize(); j++ ) { + if ( this->get(Id<int>(i),Id<int>(j))>tOut ) tOut = this->get(Id<int>(i),Id<int>(j)); + } + } + return tOut; + } + + /* + // Argmax of matrix / vector... //NOT WORKING + pair<int,int> argmax ( ) const { + T tOut = T(); + pair<int,int> ij(); + for (int i=0; i<xSize(); i++ ){ + for (int j=0; j<ySize(); j++ ) { + if ( this->get(Id<int>(i),Id<int>(j))>tOut ) { + tOut = this->get(Id<int>(i),Id<int>(j)); + ij = make_pair(i,j); + } + } + } + return ij; //pair<int,int>( ij.getIndex(), ij.getIndex() ); + } + */ + // Diagonal matrix of vector... + friend Matrix<T> diag ( const Matrix<T>& a ) { + Matrix mOut(a.xSize(),a.xSize(),T()); // output is n x n + for (int i=0;i<a.xSize();i++) { + for (int j=0;j<a.ySize();j++) { + assert(j==0); // must be vector, n x 1 + mOut.set(Id<int>(i),Id<int>(i)) += a.get(Id<int>(i),Id<int>(j)); + } + } + return mOut; + } + + // Ordering method (treat as bit string)... + bool operator< ( const Matrix<T>& mt ) const { + if (xSize()<mt.xSize() || ySize()<mt.ySize()) return true; + if (xSize()>mt.xSize() || ySize()>mt.ySize()) return false; + for (int i=0; i<xSize(); i++ ) { + for (int j=0; j<ySize(); j++ ) { + if ( this->get(Id<int>(i),Id<int>(j)) < mt.get(Id<int>(i),Id<int>(j)) ) return true; + else if ( this->get(Id<int>(i),Id<int>(j)) > mt.get(Id<int>(i),Id<int>(j)) ) return false; + } + } + return false; + } + bool operator== ( const Matrix<T>& a ) const { + if (xSize()!=a.xSize() || ySize()!=a.ySize()) return false; + for (int i=0;i<a.xSize();i++) + for (int j=0;j<a.ySize();j++) + if (this->get(Id<int>(i),Id<int>(j))!=a.get(Id<int>(i),Id<int>(j))) return false; + return true; + } + + // Input/output methods... + friend ostream& operator<< ( ostream& os, const Matrix<T>& a ) { + os<<"\n "; + for (int i=0;i<a.xSize();i++) { + for (int j=0;j<a.ySize();j++) { + os<<((j==0)?"":",")<<a.get(Id<int>(i),Id<int>(j)); + } + os<<(i==a.xSize()-1?"\n":"\n "); + } + return os; + } + friend String& operator<< ( String& str, const Matrix<T>& a ) { + str<<"\n "; + for (int i=0;i<a.xSize();i++) { + for (int j=0;j<a.ySize();j++) { + str<<((j==0)?"":",")<<a.get(Id<int>(i),Id<int>(j)); + } + str<<";"; + } + return str; + } + string getString( ) const; + +}; +template <class T> +string Matrix<T>::getString() const { + string str; + for (int i=0;i<xSize();i++) { + for (int j=0;j<ySize();j++) { + str += ((j==0)?"":","); + str += this->get(Id<int>(i),Id<int>(j)); + } + str += ";"; + } + return str; +} + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-gauss.h b/contrib/synlm/hhmm/rvtl/include/nl-gauss.h new file mode 100644 index 000000000..a2213086f --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-gauss.h @@ -0,0 +1,287 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_GAUSS__ +#define _NL_GAUSS__ + +#include <vector> +#include <string> +#include <math.h> +#include "nl-cpt.h" +#include "nl-prob.h" +#include "nl-list.h" + +using namespace std; + +static const PDFVal MEAN_THRESHOLD = 0.01; //0.0001; //0.001 +static const PDFVal VARIANCE_THRESHOLD = 0.01; //0.0001; //0 + + +//////////////////////////////////////////////////////////////////////////////// +// +// Diagonal Multivariate Gaussian Model +// +//////////////////////////////////////////////////////////////////////////////// + +template <class Y> +class DiagGauss1DModel : public Generic1DModel<Y,PDFVal> { + private: + // Member variables... + string sId; + bool bModeled; + PDFVal prInvPowSqrt2PI; + SimpleHash<Id<int>,PDFVal> aMeans; + SimpleHash<Id<int>,PDFVal> aVariances; + PDFVal prInvRootNormVariances; + PDFVal prProduct; + SimpleHash<Id<int>,PDFVal> algprNegHalfInvVariances; + public: + // Constructor / destructor methods... + DiagGauss1DModel ( ) : bModeled(false) { } + DiagGauss1DModel ( const string& s ) : sId(s), bModeled(false) { } + // Specification methods... + void precomputeVarianceTerms ( ) ; + PDFVal& setMean ( int i ) { return aMeans.set(i); } + PDFVal& setVariance ( int i ) { return aVariances.set(i); } + PDFVal& setInvRootNormVar ( ) { return prInvRootNormVariances; } + PDFVal& setNegHalfInvVar ( int i ) { return algprNegHalfInvVariances.set(i); } + // Extraction methods... + PDFVal getMean ( int i ) const { return aMeans.get(i); } + PDFVal getVariance ( int i ) const { return aVariances.get(i); } + PDFVal getInvRootNormVar ( ) const { return prInvRootNormVariances; } + PDFVal getNegHalfInvVar ( int i ) const { return algprNegHalfInvVariances.get(i); } + int getNumFeats ( ) const { return Y::getSize(); } + PDFVal getProb ( const Y& ) const ; + // Input / output methods... + bool readFields ( char*[], int ) ; + void writeFields ( FILE*, const string& ) const ; +}; + +//////////////////////////////////////// +template <class Y> +inline void DiagGauss1DModel<Y>::precomputeVarianceTerms ( ) { + // Inverse square root of norm of variances... + setInvRootNormVar() = 1.0; + for ( int i=0; i<getNumFeats(); i++ ) setInvRootNormVar() *= 1.0/sqrt(getVariance(i)); + // Negative half of inverse of variances... + for ( int i=0; i<getNumFeats(); i++ ) setNegHalfInvVar(i) = -1.0/(2.0*getVariance(i)); + // Derived from variance terms... + prInvPowSqrt2PI = 1.0/pow(sqrt(2.0*M_PI),getNumFeats()); + prProduct = prInvPowSqrt2PI * getInvRootNormVar(); + bModeled = true; +} + +//////////////////////////////////////// +template <class Y> +inline PDFVal DiagGauss1DModel<Y>::getProb ( const Y& y ) const { +// fprintf(stderr,"--------------------\n"); +// y.write(stderr); +// fprintf(stderr,"\n----------\n"); +// writeFields(stderr,""); + assert(bModeled); + PDFVal logprob = 0.0; + for ( int i=0; i<getNumFeats(); i++ ) + logprob += getNegHalfInvVar(i) * pow(y.get(i)-getMean(i),2.0); +// for ( int i=0; i<getNumFeats(); i++ ) +// fprintf(stderr,"%d %g\n", i, getNegHalfInvVar(i) * pow(y.get(i)-getMean(i),2.0)); +// fprintf(stderr,"----------> %g\n",prProduct * exp(logprob)); + return ( prProduct * exp(logprob) ) ; +} + +//////////////////////////////////////// +template <class Y> +bool DiagGauss1DModel<Y>::readFields ( char* as[], int numFields ) { + if ( 0==strcmp(as[1],"m") && numFields>2 ) { + char* psT; + for(int i=0;i<getNumFeats();i++) + setMean(i)=atof(strtok_r((0==i)?as[2]:NULL,"_",&psT)); + } + else if ( 0==strcmp(as[1],"v") && numFields>2 ) { + char* psT; + for(int i=0;i<getNumFeats();i++) + setVariance(i)=atof(strtok_r((0==i)?as[2]:NULL,"_",&psT)); + } + else return false; + return true; +} + +//////////////////////////////////////// +template <class Y> +void DiagGauss1DModel<Y>::writeFields ( FILE* pf, const string& sPref ) const { + fprintf(pf,"%s m = ",sPref.c_str()); + for(int i=0; i<getNumFeats(); i++) fprintf(pf,"%s%f",(0==i)?"":"_",getMean(i)); + fprintf ( pf, "\n" ) ; + + fprintf(pf,"%s v = ",sPref.c_str()); + for(int i=0; i<getNumFeats(); i++) fprintf(pf,"%s%f",(0==i)?"":"_",getVariance(i)); + fprintf ( pf, "\n" ) ; +} + + +//////////////////////////////////////////////////////////////////////////////// + +/* +template <class Y,class X> +class DiagGauss2DModel : public Generic2DModel<Y,X,PDFVal> { + private: + // Member variables... + string sId; + SimpleHash<X,DiagGauss1DModel<Y> > mMY_giv_X; + public: + // Constructor / destructor methods... + DiagGauss2DModel ( const string& s ) : sId(s) { } + // Extraction methods... + Prob getProb ( const Y& y, const X& x ) const { return mMY_giv_X.get(x).getProb(y); } + // Input / output methods... + bool readFields ( char* as[], int numFields ) { + ////if ( as[0]!=sId ) return false; // HAVE TO CHECK IN CALLIN FN NOW + if ( 0==strcmp(as[1],"m") && numFields>3 ) + for ( int i=0; i<numFields-3; i++ ) mMY_giv_X.set(X(as[2])).setMean(i) = atof(as[i+4]) ; + else if ( 0==strcmp(as[1],"v") && numFields>3 ) + for ( int i=0; i<numFields-3; i++ ) mMY_giv_X.set(X(as[2])).setVariance(i) = atof(as[i+4]) ; + else return false; + return true; + } + void writeFields ( FILE* pf, const string& sPref ) const { + X x; + for(bool b=x.setFirst(); b; b=x.setNext()) + { fprintf(pf,"%s m ",sPref.c_str()); x.write(pf); fprintf(pf," ="); + for(int i=0; i<Y::getSize(); i++) fprintf(pf," %f",mMY_giv_X.getProb(x).getMean(i)); + fprintf ( pf, "\n" ) ; } + for(bool b=x.setFirst(); b; b=x.setNext()) + { fprintf(pf,"%s v ",sPref.c_str()); x.write(pf); fprintf(pf," ="); + for(int i=0; i<Y::getSize(); i++) fprintf(pf," %f",mMY_giv_X.getProb(x).getVariance(i)); + fprintf ( pf, "\n" ) ; } + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class Y,class X1,class X2> +class DiagGauss3DModel : public Generic3DModel<Y,X1,X2,PDFVal> { + private: + // Member variables... + string sId; + SimpleHash<Joint2DRV<X1,X2>,DiagGauss1DModel<Y> > mMY_giv_X1_X2; + public: + // Constructor / destructor methods... + DiagGauss3DModel ( const string& s ) : sId(s) { } + // Extraction methods... + Prob getProb ( const Y& y, const X1& x1, const X2& x2 ) const { return mMY_giv_X1_X2.get(x1,x2).getProb(y); } + // Input / output methods... + bool readFields ( char* as[], int numFields ) { + if ( as[0]!=sId ) return false; + if ( 0==strcmp(as[1],"m") && numFields>4 ) + for ( int i=0; i<numFields-4; i++ ) mMY_giv_X1_X2.set(Joint2DRV<X1,X2>(X1(as[2]),X2(as[2]))).setMean(i) = atof(as[i+4]) ; + else if ( 0==strcmp(as[1],"v") && numFields>4 ) + for ( int i=0; i<numFields-4; i++ ) mMY_giv_X1_X2.set(Joint2DRV<X1,X2>(X1(as[2]),X2(as[2]))).setVariance(i) = atof(as[i+4]) ; + else return false; + return true; + } + void writeFields ( FILE* pf, string& sPref ) const { + X1 x1; X2 x2; + for(bool b1=x1.setFirst(); b1; b1=x1.setNext()) { + for(bool b2=x2.setFirst(); b2; b2=x2.setNext()) + { fprintf(pf,"%s m ",sPref.c_str()); x1.write(pf); fprintf(pf," "); x2.write(pf); fprintf(pf," ="); + for(int i=0; i<Y::getSize(); i++) fprintf(pf," %f",mMY_giv_X1_X2.get(Joint2DRV<X1,X2>(x1,x2)).getMean(i)); + fprintf(pf,"\n"); } + for(bool b2=x2.setFirst(); b2; b2=x2.setNext()) + { fprintf(pf,"%s v ",sPref.c_str()); x1.write(pf); fprintf(pf," "); x2.write(pf); fprintf(pf," ="); + for(int i=0; i<Y::getSize(); i++) fprintf(pf," %f",mMY_giv_X1_X2.get(Joint2DRV<X1,X2>(x1,x2)).getVariance(i)); + fprintf(pf,"\n"); } + } + } +}; +*/ + +//////////////////////////////////////////////////////////////////////////////// +// +// Trainable Diagonal Multivariate Gaussian Model +// +//////////////////////////////////////////////////////////////////////////////// + +template <class Y> +class TrainableDiagGauss1DModel : public DiagGauss1DModel<Y> { + public: + TrainableDiagGauss1DModel ( ) : DiagGauss1DModel<Y>() { } + TrainableDiagGauss1DModel ( const string& s ) : DiagGauss1DModel<Y>(s) { } + // input / output methods... + void setFields ( const List<pair<const Y*,Prob> >& ) ; +}; + +//////////////////////////////////////// +template <class Y> +void TrainableDiagGauss1DModel<Y>::setFields ( const List<pair<const Y*,Prob> >& lyp ) { + + // For each dimension... + for ( int i=0; i<DiagGauss1DModel<Y>::getNumFeats(); i++ ) { + + // Calc means... + double curMean = DiagGauss1DModel<Y>::getMean(i); + DiagGauss1DModel<Y>::setMean(i) = 0.0; + + // For each Y... + for ( const ListedObject<pair<const Y*,Prob> >* pyp=lyp.getFirst(); pyp; pyp=lyp.getNext(pyp) ) { + const Y& y = *pyp->first; // data value + const Prob& prEmpY = pyp->second; // empirical prob + + //printf("cal mean i=%d x1=%s x2=%s aaaprYpsi.get(yd,x1,x2)=%f\n", i, x1.getString(), x2.getString(), (double)aaaprYpsi.getProb(yd,x1,x2)); + DiagGauss1DModel<Y>::setMean(i) += prEmpY * y.get(i); + } + +// // If any change exceeds thresh, continue... +// if ( bShouldStop && ( curMean - DiagGauss1DModel<Y>::getMean(i) > MEAN_THRESHOLD || +// curMean - DiagGauss1DModel<Y>::getMean(i) < -MEAN_THRESHOLD ) ) bShouldStop = false; + + //printf("cal mean i=%d getMean(i)=%f\n", i, DiagGauss1DModel<Y>::getMean(i)); + + // Calc variances... + double curVar = DiagGauss1DModel<Y>::getVariance(i); + DiagGauss1DModel<Y>::setVariance(i) = 0.0; + + // For each Y... + for ( const ListedObject<pair<const Y*,Prob> >* pyp=lyp.getFirst(); pyp; pyp=lyp.getNext(pyp) ) { + const Y& y = *pyp->first; // data value + const Prob& prEmpY = pyp->second; // empirical prob + + //printf("cal var i=%d yd=%s %f %f %f\n", i, yd.getString(), aaaprYpsi.get(yd,x1,x2), getMean(x1,x2,i), yd.get(i)); + DiagGauss1DModel<Y>::setVariance(i) += prEmpY * pow(DiagGauss1DModel<Y>::getMean(i)-y.get(i),2) ; + } + +// // If any change exceeds thresh, continue... +// if ( bShouldStop && ( curVar - DiagGauss1DModel<Y>::getVariance(i) > VARIANCE_THRESHOLD || +// curVar - DiagGauss1DModel<Y>::getVariance(i) < -VARIANCE_THRESHOLD ) ) bShouldStop = false; + + // Avoid div by zero... + if (DiagGauss1DModel<Y>::getVariance(i) < 1.0) DiagGauss1DModel<Y>::setVariance(i) = 1.0; + + //printf("cal variance i=%d var=%f\n", i, DiagGauss1DModel<Y>::getVariance(i)); + } + DiagGauss1DModel<Y>::precomputeVarianceTerms(); +} + + +#endif /*_NL_GAUSS__*/ + + + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-hash.h b/contrib/synlm/hhmm/rvtl/include/nl-hash.h new file mode 100644 index 000000000..809284db9 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-hash.h @@ -0,0 +1,105 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __NL_HASH_H_ +#define __NL_HASH_H_ + +#include <cassert> +//#include <tr1/unordered_map> +#include <ext/hash_map> +using namespace __gnu_cxx; + + +/////////////////////////////////////////////////////////////////////////////// + +template<class T> +class SimpleHashFn { + public: + size_t operator() ( const T& t ) const { return t.getHashKey(); } +}; + + +template<class T> +class SimpleHashEqual { + public: + bool operator() ( const T& t1, const T& t2 ) const { return (t1 == t2); } +}; + + +template<class X, class Y> +class SimpleHash : public hash_map<X,Y,SimpleHashFn<X>,SimpleHashEqual<X> > /*public tr1::unordered_map<X,Y,SimpleHashFn<X>,SimpleHashEqual<X> >*/ { + private: + typedef hash_map<X,Y,SimpleHashFn<X>,SimpleHashEqual<X> > OrigHash; +// typedef tr1::unordered_map<X,Y,SimpleHashFn<X>,SimpleHashEqual<X> > OrigHash; +// tr1::unordered_map<X,Y,SimpleHashFn<X>,SimpleHashEqual<X> > mxy; + static const Y yDummy; + //static Y yNonconstDummy; + + public: +// typedef typename OrigHash::const_iterator const_iterator; +// typedef typename OrigHash::iterator iterator; +// static const const_iterator iDummy; + // Constructor / destructor methods... + SimpleHash ( ) : OrigHash() { } + SimpleHash ( int i ) : OrigHash(i) { } + SimpleHash (const SimpleHash& s) : OrigHash(s) { } + // Specification methods... + Y& set ( const X& x ) { return OrigHash::operator[](x); } + // Extraction methods... + const Y& get ( const X& x ) const { return (OrigHash::end()!=OrigHash::find(x)) ? OrigHash::find(x)->second : yDummy; } + bool contains ( const X& x ) const { return (OrigHash::end()!=OrigHash::find(x)); } +// const Y& get ( const X& x ) const { return (mxy.end()!=mxy.find(x)) ? mxy.find(x)->second : yDummy; } +// Y& set ( const X& x ) { return mxy[x]; } + friend ostream& operator<< ( ostream& os, const SimpleHash<X,Y>& h ) { + for ( typename SimpleHash<X,Y>::const_iterator it=h.begin(); it!=h.end(); it++ ) + os<<((it==h.begin())?"":",")<<it->first<<":"<<it->second; + return os; + } +}; +template<class X, class Y> const Y SimpleHash<X,Y>::yDummy = Y(); +//template<class X, class Y> Y SimpleHash<X,Y>::yNonconstDummy; // = Y(); + +/* +template<class X, class Y> +class SimpleMultiHash : public tr1::unordered_multimap<X,Y,SimpleHashFn<X>,SimpleHashEqual<X> > { + private: + typedef tr1::unordered_multimap<X,Y,SimpleHashFn<X>,SimpleHashEqual<X> > OrigHash; + public: + typedef pair<typename OrigHash::const_iterator,typename OrigHash::const_iterator> const_iterator_pair; + + // Constructor / destructor methods... + SimpleMultiHash ( ) : OrigHash() { } + SimpleMultiHash ( int i ) : OrigHash(i) { } + // Specification methods... + Y& add ( const X& x ) { return insert(typename OrigHash::value_type(x,Y()))->second; } + // Extraction methods... + bool contains ( const X& x ) const { return (OrigHash::end()!=OrigHash::find(x)); } + bool contains ( const X& x, const Y& y ) const { + if (OrigHash::end()==OrigHash::find(x)) return false; + for ( const_iterator_pair ii=OrigHash::equal_range(x); ii.first!=ii.second; ii.first++ ) + if ( y == ii.first->second ) return true; + return false; + } +}; +*/ +#endif // __NL_HASH_H_ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-heap.h b/contrib/synlm/hhmm/rvtl/include/nl-heap.h new file mode 100644 index 000000000..31be6ed75 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-heap.h @@ -0,0 +1,181 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_MINHEAP_ +#define _NL_MINHEAP_ + +#include "nl-safeids.h" + +//////////////////////////////////////////////////////////////////////////////// + +template <class R, bool outrank(const R&, const R&)> +class Heap { + private: + Array<R> at; + int iNextToFill; + //SafeArray1D<Id<int>,R> at; + // Private specification methods... + int heapify ( unsigned int ) ; + public: + // Constructor / destructor methods... + Heap<R,outrank> ( ) : at(10), iNextToFill(0) { } + Heap<R,outrank> ( int i ) : at(i), iNextToFill(0) { } + Heap<R,outrank> ( int i, const R& r ) : at(i,r), iNextToFill(0) { } + // Specification methods... + void init ( int i ) { iNextToFill=0; at.init(i); } + void clear ( ) { iNextToFill=0; } + unsigned int fixIncRank ( unsigned int i ); + unsigned int fixDecRank ( unsigned int i ); + R& set ( unsigned int i ) { return at.set(i-1); } + void enqueue ( const R& r ) { set(iNextToFill+1)=r; fixIncRank(iNextToFill+1); iNextToFill++; } + R dequeueTop ( ) { R r=get(1); iNextToFill--; set(1)=get(iNextToFill+1); set(iNextToFill+1)=R(); fixDecRank(1); return r; } + ////R& set ( const Id<int>& i ) { return at.set(i); } + R& setTop ( ) { return at.set(1-1); } + // Extraction methods... + int getSize ( ) const { return iNextToFill; } + const R& getTop ( ) const { return at.get(1-1); } + const R& get ( unsigned int i ) const { return at.get(i-1); } + ////const R& get ( const Id<int>& i ) const { return at.get(i); } + // Input / output methods... + friend ostream& operator<< ( ostream& os, const Heap<R,outrank>& h ) { for(int i=0;i<h.iNextToFill;i++) os<<h.at.get(i)<<"\n"; return os; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class R, bool outrank(const R&, const R&)> +int Heap<R,outrank>::heapify ( unsigned int ind ) { + // Find best of parent, left child, right child... + unsigned int indBest = ind; + indBest = (ind*2 <= (unsigned int)iNextToFill && + outrank(get(ind*2),get(indBest))) + ? ind*2 : indBest; + indBest = (ind*2+1 <= (unsigned int)iNextToFill && + outrank(get(ind*2+1),get(indBest))) + ? ind*2+1 : indBest; + + // If parent isn't best, restore heap property... + if ( indBest != ind ) { + // Swap heap elements... + R rTemp = get(ind); + set(ind) = get(indBest); + set(indBest) = rTemp; + // Recurse... + return heapify(indBest); + } + else return ind; +} + +template <class R, bool outrank(const R&, const R&)> +unsigned int Heap<R,outrank>::fixIncRank ( unsigned int ind ) { //const R& rec ) { + // If child outranks parent, restore heap property... + if ( outrank(get(ind),get((ind==1)?1:ind/2)) ) { + // Swap heap elements... + R rTemp = get((ind==1)?1:ind/2); + set((ind==1)?1:ind/2) = get(ind); + set(ind) = rTemp; + // Recurse on parent... + return fixIncRank(ind/2); + } + else return ind; +} + +template <class R, bool outrank(const R&, const R&)> +unsigned int Heap<R,outrank>::fixDecRank ( unsigned int ind ) { //const R& rec ) { + return heapify(ind); +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class R> +class MinHeap { + private: + Array<R> at; + //SafeArray1D<Id<int>,R> at; + // Private specification methods... + int minHeapify ( unsigned int ) ; + public: + // Constructor / destructor methods... + MinHeap<R> ( ) : at(10) { } + MinHeap<R> ( int i ) : at(i) { } + MinHeap<R> ( int i, const R& r ) : at(i,r) { } + // Specification methods... + void init ( int i ) { at.init(i); } + void clear ( ) { at.clear(); } + int fixDecr ( int i ); + int fixIncr ( int i ); + R& set ( unsigned int i ) { return at.set(i); } + ////R& set ( const Id<int>& i ) { return at.set(i); } + R& setMin ( ) { return at.set(1-1); } + // Extraction methods... + int getSize ( ) const { return at.getSize(); } + const R& getMin ( ) const { return at.get(1-1); } + const R& get ( unsigned int i ) const { return at.get(i); } + ////const R& get ( const Id<int>& i ) const { return at.get(i); } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class R> +int MinHeap<R>::minHeapify ( unsigned int ind ) { + // Find min of parent, left child, right child... + unsigned int indMin = ind ; + indMin = (ind*2 <= (unsigned int)at.getSize() && + at.get(ind*2-1).getScore() < at.get(indMin-1).getScore()) + ? ind*2 : indMin ; + indMin = (ind*2+1 <= (unsigned int)at.getSize() && + at.get(ind*2+1-1).getScore() < at.get(indMin-1).getScore()) + ? ind*2+1 :indMin; + + // If parent isn't min, restore heap property... + if ( indMin != ind ) { + // Swap heap elements... + R rTemp = at.get(ind-1); + at.set(ind-1) = at.get(indMin-1); + at.set(indMin-1) = rTemp; + // Recurse... + return minHeapify(indMin); + } + else return ind; +} + +template <class R> +int MinHeap<R>::fixDecr ( int i ) { //const R& rec ) { + // If parent isn't min, restore heap property... + if ( at.get((i+1)/2).getScore() > at.get(i).getScore() ) { + // Swap heap elements... + R rTemp = at.get((i+1)/2); + at.set((i+1)/2) = at.get(i); + at.set(i) = rTemp; + // Recurse on parent... + return fixDecr((i+1)/2); + } + else return i; +} + +template <class R> +int MinHeap<R>::fixIncr ( int i ) { //const R& rec ) { + return minHeapify(i+1)-1; +} + +#endif //_NL_HEAP_ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-hmm.h b/contrib/synlm/hhmm/rvtl/include/nl-hmm.h new file mode 100644 index 000000000..2f6cd0104 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-hmm.h @@ -0,0 +1,1007 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_HMM_ +#define _NL_HMM_ + +#include <list> +#include <string> +#include <iostream> +#include <iomanip> +//#include <boost/thread/thread.hpp> +//#include <boost/thread/mutex.hpp> +//#include <boost/bind.hpp> +#include "nl-prob.h" +#include "nl-safeids.h" +#include "nl-beam.h" + +typedef int Frame; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// NullBackDat - default empty back-pointer data; can replace with word or sem relation +// +//////////////////////////////////////////////////////////////////////////////// + +template <class MY> +class NullBackDat { + static const string sDummy; + public: + NullBackDat () {} + NullBackDat (const MY& my) {} + void write (FILE*) const {} + string getString() const { return sDummy; } +}; +template <class MY> +const string NullBackDat<MY>::sDummy ( "" ); + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Index - pointer to source in previous beam heap +// +//////////////////////////////////////////////////////////////////////////////// + +class Index : public Id<int> { + public: + Index ( ) { } + Index (int i) {set(i);} + Index& operator++ ( ) {set(toInt()+1); return *this;} +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// TrellNode - node in viterbi trellis +// +//////////////////////////////////////////////////////////////////////////////// + +template <class S, class B> +class TrellNode { + private: + + // Data members... + Index indSource; + B backptrData; + S sId; + LogProb lgprMax; + + public: + + // Constructor / destructor methods... + TrellNode ( ) { } + TrellNode ( const Index& indS, const S& sI, const B& bDat, LogProb lgpr) + { indSource=indS; sId=sI; lgprMax=lgpr; backptrData=bDat; /* fo = -1; */ } + + // Specification methods... + const Index& setSource ( ) const { return indSource; } + const B& setBackData( ) const { return backptrData; } + const S& setId ( ) const { return sId; } + LogProb& setScore ( ) { return lgprMax; } + + // Extraction methods... + bool operator== ( const TrellNode<S,B>& tnsb ) const { return(sId==tnsb.sId); } +// size_t getHashKey ( ) const { return sId.getHashKey(); } + const Index& getSource ( ) const { return indSource; } + const B& getBackData( ) const { return backptrData; } + const S& getId ( ) const { return sId; } + LogProb getLogProb ( ) const { return lgprMax; } + LogProb getScore ( ) const { return lgprMax; } +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// HMM +// +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S=typename MY::RandVarType, class B=NullBackDat<typename MY::RandVarType> > +class HMM { + private: + typedef std::pair<Index,B> IB; + // Data members... + const MY& my; + const MX& mx; + SafeArray2D<Id<Frame>,Id<int>,TrellNode<S,B> > aatnTrellis; + Frame frameLast; + int iNextNode; + public: + // Static member varaibles... + static bool OUTPUT_QUIET; + static bool OUTPUT_NOISY; + static bool OUTPUT_VERYNOISY; + static int BEAM_WIDTH; + // Constructor / destructor methods... + HMM ( const MY& my1, const MX& mx1 ) : my(my1), mx(mx1) { } + // Specification methods... + void init ( int, int, const S& ) ; + void init ( int, int, SafeArray1D<Id<int>,pair<S,LogProb> >* ); + void updateRanked ( const typename MX::RandVarType&, bool ) ; + void updateSerial ( const typename MX::RandVarType& ) ; + void updatePara ( const typename MX::RandVarType& ) ; + bool unknown ( const typename MX::RandVarType& ) ; + void each ( const typename MX::RandVarType&, Beam<LogProb,S,IB>&, SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> >& ) ; + // Extraction methods... + const TrellNode<S,B>& getTrellNode ( int i ) const { return aatnTrellis.get(frameLast,i); } + int getBeamUsed ( int ) const ; + // Input / output methods... + void writeMLS ( FILE* ) const ; + void writeMLS ( FILE*, const S& ) const ; + void debugPrint() const; + double getCurrSum(int) const; + //void writeCurr ( FILE*, int ) const ; + void writeCurr ( ostream&, int ) const ; + void writeCurrSum ( FILE*, int ) const ; + void gatherElementsInBeam( SafeArray1D<Id<int>,pair<S,LogProb> >* result, int f ) const; + void writeCurrEntropy ( FILE*, int ) const; + //void writeCurrDepths ( FILE*, int ) const; + void writeFoll ( FILE*, int, int, const typename MX::RandVarType& ) const ; + void writeFollRanked ( FILE*, int, int, const typename MX::RandVarType&, bool ) const ; + std::list<string> getMLS() const; + std::list<TrellNode<S,B> > getMLSnodes() const; + std::list<string> getMLS(const S&) const; + std::list<TrellNode<S,B> > getMLSnodes(const S&) const; +}; +template <class MY, class MX, class S, class B> bool HMM<MY,MX,S,B>::OUTPUT_QUIET = false; +template <class MY, class MX, class S, class B> bool HMM<MY,MX,S,B>::OUTPUT_NOISY = false; +template <class MY, class MX, class S, class B> bool HMM<MY,MX,S,B>::OUTPUT_VERYNOISY = false; +template <class MY, class MX, class S, class B> int HMM<MY,MX,S,B>::BEAM_WIDTH = 1; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::init ( int numFr, int numS, const S& s ) { + + // Alloc trellis... + BEAM_WIDTH = numS; + aatnTrellis.init(numFr,BEAM_WIDTH); + + frameLast=0; + + // Set initial element at first time slice... + aatnTrellis.set(frameLast,0) = TrellNode<S,B> ( Index(0), s, B(), 0 ) ; +} + +template <class MY, class MX, class S, class B> + void HMM<MY,MX,S,B>::init ( int numFr, int beamSize, SafeArray1D<Id<int>,pair<S,LogProb> >* existingBeam ) { + + // Alloc trellis... + // int numToCopy = existingBeam->getSize(); + BEAM_WIDTH = beamSize; + aatnTrellis.init(numFr,BEAM_WIDTH); + + frameLast=0; + + // Set initial beam elements at first time slice... + for ( int i=0, n=existingBeam->getSize(); i<n; i++ ) { + aatnTrellis.set(frameLast,i) = TrellNode<S,B> ( Index(0), existingBeam->get(i).first, B(), existingBeam->get(i).second ) ; + } + +} + +template <class MY, class MX, class S, class B> + void HMM<MY,MX,S,B>::debugPrint() const{ + + for (int frame=0, numFrames=aatnTrellis.getxSize(); frame<numFrames; frame++) { + + for (int beamIndex=0, beamSize=aatnTrellis.getySize(); beamIndex<beamSize; beamIndex++) { + + if (aatnTrellis.get(frame,beamIndex).getLogProb().toDouble() > 0) { + cerr << "\t" << "aatnTrellis.get(frame=" << frame << ",beamIndex=" << beamIndex << ") is\t" << aatnTrellis.get(frame,beamIndex).getId() << "\tprob=" << aatnTrellis.get(frame,beamIndex).getLogProb().toDouble() << endl; + } + + } + + } + +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class A, class B> +bool outRank ( const quad<A,B,LogProb,Id<int> >& a1, + const quad<A,B,LogProb,Id<int> >& a2 ) { return (a1.third>a2.third); } + +template <class MY, class MX, class S, class B> + bool HMM<MY,MX,S,B>::unknown( const typename MX::RandVarType& x ) { + return mx.unknown(x); +} + + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::updateRanked ( const typename MX::RandVarType& x, bool b1 ) { + // Increment frame counter... + frameLast++; + + // Init beam for new frame... + Beam<LogProb,S,IB> btn(BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted (BEAM_WIDTH); + + Heap < quad<int,typename MY::IterVal,LogProb,Id<int> >, outRank<int,typename MY::IterVal> > ashpiQueue; + typedef quad<int,typename MY::IterVal,LogProb,Id<int> > SHPI; + SHPI shpi, shpiTop; + int aCtr; + + ashpiQueue.clear(); + //shpi.first = -1; + //shpi.second = HModel::IterVal(); + //shpi.third = 1.0; + shpi.first = 0; + shpi.third = aatnTrellis.get(frameLast-1,shpi.first).getScore(); + shpi.third *= my.setIterProb ( shpi.second, aatnTrellis.get(frameLast-1,shpi.first).getId(), x, b1, aCtr=-1 ); + //S s; my.setTrellDat(s,shpi.second); + shpi.fourth = -1; + ////cerr<<"????? "<<shpi<<"\n"; + ashpiQueue.enqueue(shpi); + + bool bFull=false; + + // For each ranked value of transition destination... + for ( int iTrg=0; !bFull && ashpiQueue.getSize()>0; iTrg++ ) { + // Iterate A* (best-first) search until a complete path is at the top of the queue... + while ( ashpiQueue.getSize() > 0 && ashpiQueue.getTop().fourth < MY::IterVal::NUM_ITERS ) { + // Remove top... + shpiTop = ashpiQueue.dequeueTop(); + // Fork off (try to advance each elementary variable a)... + for ( int a=shpiTop.fourth.toInt(); a<=MY::IterVal::NUM_ITERS; a++ ) { + // Copy top into new queue element... + shpi = shpiTop; + // At variable position -1, advance beam element for transition source... + if ( a == -1 ) shpi.first++; + // Incorporate prob from transition source... + shpi.third = aatnTrellis.get(frameLast-1,shpi.first).getScore(); + if ( shpi.third > LogProb() ) { + // Try to advance variable at position a and return probability (subsequent variables set to first, probability ignored)... + shpi.third *= my.setIterProb ( shpi.second, aatnTrellis.get(frameLast-1,shpi.first).getId(), x, b1, aCtr=a ); + // At end of variables, incorporate observation probability... + if ( a == MY::IterVal::NUM_ITERS && shpi.fourth != MY::IterVal::NUM_ITERS ) + { S s; my.setTrellDat(s,shpi.second); shpi.third *= mx.getProb(x,s); } + // Record variable position at which this element was forked off... + shpi.fourth = a; + //cerr<<" from partial: "<<shpiTop<<"\n to partial: "<<shpi<<"\n"; + if ( shpi.third > LogProb() ) { + ////if ( frameLast == 4 ) cerr<<" from partial: "<<shpiTop<<"\n to partial: "<<shpi<<"\n"; + // If valid, add to queue... + ashpiQueue.enqueue(shpi); + //cerr<<"--------------------\n"<<ashpiQueue; + } + } + } + // Remove top... + //cerr<<"/-----A-----\\\n"<<ashpiQueue<<"\\-----A-----/\n"; + //if ( ashpiQueue.getTop().fourth != MY::IterVal::NUM_ITERS ) ashpiQueue.dequeueTop(); + ////cerr<<"/-----B-----\\\n"<<ashpiQueue<<"\\-----B-----/\n"; + ////cerr<<ashpiQueue.getSize()<<" queue elems, "<<ashpiQueue.getTop()<<"\n"; + } + + ////cerr<<"-----*-----\n"<<ashpiQueue<<"-----*-----\n"; + ////cerr<<ashpiQueue.getSize()<<" queue elems **\n"; + + // Add best transition (top of queue)... + //mx.getProb(o,my.setTrellDat(ashpiQueue.getTop().first,ashpiQueue.getTop().second)); + if ( ashpiQueue.getSize() > 0 ) { + S s; my.setTrellDat(s,ashpiQueue.getTop().second); + bFull |= btn.tryAdd ( s, IB(ashpiQueue.getTop().first,my.setBackDat(ashpiQueue.getTop().second)), ashpiQueue.getTop().third ); + ////cerr<<ashpiQueue.getSize()<<" queue elems A "<<ashpiQueue.getTop()<<"\n"; + ////cerr<<"/-----A-----\\\n"<<ashpiQueue<<"\\-----A-----/\n"; + ashpiQueue.dequeueTop(); + ////cerr<<"/-----B-----\\\n"<<ashpiQueue<<"\\-----B-----/\n"; + ////cerr<<ashpiQueue.getSize()<<" queue elems B "<<ashpiQueue.getTop()<<"\n"; + //cerr<<"."; cerr.flush(); + } + } + + ////cerr<<"-----*-----\n"<<ashpiQueue<<"-----*-----\n"; + + btn.sort(atnSorted); + + // Copy sorted beam to trellis... + for(int i=0;i<BEAM_WIDTH;i++) { + const std::pair<std::pair<S,IB>,LogProb>* tn1 = &atnSorted.get(i); + aatnTrellis.set(frameLast,i)=TrellNode<S,B>(tn1->first.second.first, + tn1->first.first, + tn1->first.second.second, + tn1->second); + } + + my.update(); +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::updateSerial ( const typename MX::RandVarType& x ) { + // Increment frame counter... + frameLast++; + + // Init beam for new frame... + Beam<LogProb,S,IB> btn(BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted (BEAM_WIDTH); + +// // Copy beam to trellis... +// for ( int i=0; i<BEAM_WIDTH; i++ ) +// btn.set ( i, aatnTrellis.set(frameLast,i) ); + + // For each possible hidden value for previous frame... + for ( int i=0; i<BEAM_WIDTH; i++ ) { + const TrellNode<S,B>& tnsbPrev = aatnTrellis.get(frameLast-1,i); + S sPrev = tnsbPrev.getId(); + // If prob still not below beam minimum... + if ( aatnTrellis.get(frameLast-1,i).getLogProb() > btn.getMin().getScore() ) { + //if (OUTPUT_VERYNOISY) { fprintf(stderr,"FROM: "); sPrev.write(stderr); fprintf(stderr,"\n"); } + // For each possible transition... + typename MY::IterVal y; + for ( bool b=my.setFirst(y,sPrev); b; b=my.setNext(y,sPrev) ) { + //if (OUTPUT_VERYNOISY) { fprintf(stderr," TO: "); y.write(stderr); fprintf(stderr," %d*%d*%d\n",tnsbPrev.getLogProb().toInt(),lgprY.toInt(),lgprO.toInt()); } + S s; +#ifdef X_BEFORE_Y + LogProb lgprX = mx.getProb(x,my.setTrellDat(s,y)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprX ) continue; + LogProb lgprY = my.getProb(y,sPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprY ) continue; +#else + LogProb lgprY = my.getProb(y,sPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprY ) continue; + LogProb lgprX = mx.getProb(x,my.setTrellDat(s,y)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprX ) continue; +#endif + LogProb lgprFull = aatnTrellis.get(frameLast-1,i).getLogProb() * lgprY * lgprX; + if (OUTPUT_VERYNOISY) { + cout<<" "<<tnsbPrev.getId()<<" ==("<<tnsbPrev.getLogProb().toInt()<<"*"<<lgprY.toInt()<<"*"<<lgprX.toInt()<<"="<<lgprFull.toInt()<<")==> "<<y<<"\n"; + //sPrev.write(stdout); + //fprintf(stdout," ==(%d*%d*%d=%d)==> ",tnsbPrev.getLogProb().toInt(),lgprY.toInt(),lgprX.toInt(),lgprFull.toInt()); + //y.write(stdout); fprintf(stdout," "); s.write(stdout); + //fprintf(stdout,"\n"); + } + // Incorporate into trellis... + btn.tryAdd ( s, IB(i,my.setBackDat(y)), lgprFull ); + //if(OUTPUT_VERYNOISY) + // fprintf ( stderr," (S_t-1:[e^%0.6f] * Y:e^%0.6f * X:e^%0.6f = S_t:[e^%0.6f])\n", + // float(aatnTrellis.get(frameLast-1,i).getLogProb().toInt())/100.0, + // float(lgprY.toInt())/100.0, + // float(lgprX.toInt())/100.0, + // float(lgprFull.toInt())/100.0 ) ; + } + } + } + +// for(int i=0;i<BEAM_WIDTH;i++) { +// fprintf(stderr,"> "); btn.get(i)->first.write(stderr); fprintf(stderr,"\n"); +// } + + btn.sort(atnSorted); + + // Copy sorted beam to trellis... + for(int i=0;i<BEAM_WIDTH;i++) { + const std::pair<std::pair<S,IB>,LogProb>* tn1 = &atnSorted.get(i); + aatnTrellis.set(frameLast,i)=TrellNode<S,B>(tn1->first.second.first, + tn1->first.first, + tn1->first.second.second, + tn1->second); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +/* +boost::mutex mutexHmmCtrSema; +boost::mutex mutexHmmParanoiaLock; + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::each ( const typename MX::RandVarType& x, Beam<LogProb,S,IB>& btn, SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> >& atnOut ) { + + //// for(int j=0; j<100; j+=1) for(int i=0; i<1000000; i+=1); // peg processor +//{ +// boost::mutes::scoped_lock lock1(mutexHmmParanoiaLock); + + while ( true ) { + int i; + bool bStop=false; + { boost::mutex::scoped_lock lock(mutexHmmCtrSema); + if ( (i=iNextNode) < BEAM_WIDTH ) iNextNode++; + else bStop = true; + } // locked to ensure no double-duty + if ( bStop ) break; + + const TrellNode<S,B>& tnsbPrev = aatnTrellis.get(frameLast-1,i); + // If prob still not below beam minimum... + if ( tnsbPrev.getLogProb() > btn.getMin().getScore() ) { + //if (OUTPUT_VERYNOISY) { fprintf(stderr,"FROM: "); tnsbPrev.getId().write(stderr); fprintf(stderr,"\n"); } + + // For each possible transition... + const S& sPrev = tnsbPrev.getId(); + typename MY::IterVal y; + for ( bool b=my.setFirst(y,sPrev); b; b=my.setNext(y,sPrev) ) { + S s; + LogProb lgprX; + LogProb lgprY; + LogProb lgprFull; + #ifdef X_BEFORE_Y ////////////////////////////////////////////////////// + lgprX = mx.getProb(x,my.setTrellDat(s,y)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprX ) continue; + lgprY = my.getProb(y,sPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprY ) continue; + #else ////////////////////////////////////////////////////////////////// + lgprY = my.getProb(y,sPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprY ) continue; + lgprX = mx.getProb(x,my.setTrellDat(s,y)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprX ) continue; + #endif ///////////////////////////////////////////////////////////////// + lgprFull = tnsbPrev.getLogProb() * lgprY * lgprX; + if (OUTPUT_VERYNOISY) { + boost::mutex::scoped_lock lock1(mutexHmmParanoiaLock); + //fprintf(stderr," TO: "); y.write(stderr); fprintf(stderr,"\n"); + cout<<" "<<tnsbPrev.getId()<<" ==("<<tnsbPrev.getLogProb().toInt()<<"*"<<lgprY.toInt()<<"*"<<lgprX.toInt()<<"="<<lgprFull.toInt()<<")==> "<<y<<"\n"; + //tnsbPrev.getId().write(stdout); + //fprintf(stdout," ==(%d*%d*%d=%d)==> ",tnsbPrev.getLogProb().toInt(),lgprY.toInt(),lgprO.toInt(),lgprFull.toInt()); + //y.write(stdout); fprintf(stdout," "); s.write(stdout); + //fprintf(stdout,"\n"); + } + // Incorporate into trellis... + btn.tryAdd ( s, IB(i,my.setBackDat(y)), lgprFull ); +// if(OUTPUT_VERYNOISY) +// fprintf ( stderr," (S_t-1:[e^%0.6f] * Y:e^%0.6f * X:e^%0.6f = S_t:[e^%0.6f])\n", +// float(aatnTrellis.get(frameLast-1,i).getLogProb().toInt())/100.0, +// float(lgprY.toInt())/100.0, +// float(lgprO.toInt())/100.0, +// float(lgprFull.toInt())/100.0 ) ; + } + } + } + + if (OUTPUT_NOISY) + { boost::mutex::scoped_lock lock1(mutexHmmParanoiaLock); + //// btn.sort(atnOut); + fprintf(stdout,"========================================\n"); + for(int i=0;i<BEAM_WIDTH;i++) { + String str; str<<" "<<btn.get(i)->first<<":"<<btn.get(i)->second.second.first.toInt()<<":"<<btn.get(i).getScore().toInt()<<"\n"; + fprintf(stdout,str.c_array()); + //fprintf(stdout," "); btn.get(i)->first.write(stdout); fprintf(stdout,":%d:%d\n",btn.get(i)->second.second.first.toInt(),btn.get(i).getScore().toInt()); + //// atnOut.get(i).first.first.write(stdout); fprintf(stdout,":%d\n",(int)atnOut.get(i).second); + } + fprintf(stdout,"----------------------------------------\n"); + } + +//} +} + +//// Something there is (about boost::bind) that doesn't love a ref. So use a pointer for hmm and beam. +template <class MY, class MX, class S, class B> + void global_parahmm_each ( HMM<MY,MX,S,B>* phmm, const typename MX::RandVarType* px, Beam<LogProb,S,std::pair<Index,B> >* pbtn, + SafeArray1D<Id<int>,std::pair<std::pair<S,std::pair<Index,B> >,LogProb> >* patn ) { +//void HMM<MY,MX,S,B>::each_static ( HMM<MY,MX,S,B>* phmm, const MX& x, Beam<LogProb,S,IB>* pbtn, SafeArray1D<int,std::pair<std::pair<S,IB>,LogProb> >* patn ) { + phmm->each(*px,*pbtn,*patn); +} + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::updatePara ( const typename MX::RandVarType& o ) { + // Increment frame counter... + frameLast++; + +// fprintf(stdout,"++++%d",frameLast); o.write(stdout); fprintf(stdout,"\n"); + + // Init beam for new frame... + Beam<LogProb,S,IB> btn1 (BEAM_WIDTH); + Beam<LogProb,S,IB> btn2 (BEAM_WIDTH); + Beam<LogProb,S,IB> btn3 (BEAM_WIDTH); + Beam<LogProb,S,IB> btn4 (BEAM_WIDTH); + Beam<LogProb,S,IB> btn5 (BEAM_WIDTH); + Beam<LogProb,S,IB> btn6 (BEAM_WIDTH); + Beam<LogProb,S,IB> btn7 (BEAM_WIDTH); + Beam<LogProb,S,IB> btn8 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted1 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted2 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted3 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted4 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted5 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted6 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted7 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted8 (BEAM_WIDTH); + + //if(0==frameLast%20) + // fprintf(stderr,"frame %d...\n",frameLast); + iNextNode=0; + boost::thread th1(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn1,&atnSorted1)); + boost::thread th2(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn2,&atnSorted2)); + boost::thread th3(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn3,&atnSorted3)); + boost::thread th4(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn4,&atnSorted4)); + boost::thread th5(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn5,&atnSorted5)); + boost::thread th6(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn6,&atnSorted6)); + boost::thread th7(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn7,&atnSorted7)); + boost::thread th8(boost::bind(&global_parahmm_each<MY,MX,S,B>,this,&o,&btn8,&atnSorted8)); + th1.join(); + th2.join(); + th3.join(); + th4.join(); + th5.join(); + th6.join(); + th7.join(); + th8.join(); + + for(int i=0;i<BEAM_WIDTH;i++) { + btn1.tryAdd(btn2.get(i)->first, btn2.get(i)->second.second, btn2.get(i).getScore()); + btn1.tryAdd(btn3.get(i)->first, btn3.get(i)->second.second, btn3.get(i).getScore()); + btn1.tryAdd(btn4.get(i)->first, btn4.get(i)->second.second, btn4.get(i).getScore()); + btn1.tryAdd(btn5.get(i)->first, btn5.get(i)->second.second, btn5.get(i).getScore()); + btn1.tryAdd(btn6.get(i)->first, btn6.get(i)->second.second, btn6.get(i).getScore()); + btn1.tryAdd(btn7.get(i)->first, btn7.get(i)->second.second, btn7.get(i).getScore()); + btn1.tryAdd(btn8.get(i)->first, btn8.get(i)->second.second, btn8.get(i).getScore()); + } + + btn1.sort(atnSorted1); + + for(int i=0;i<BEAM_WIDTH;i++) { + const std::pair<std::pair<S,IB>,LogProb>* tn1 = &atnSorted1.get(i); + aatnTrellis.set(frameLast,i)=TrellNode<S,B>(tn1->first.second.first, + tn1->first.first, + tn1->first.second.second, + tn1->second); + } + +// for(int i=0;i<BEAM_WIDTH;i++) { +// aatnTrellis.set(frameLast,i)=TrellNode<S,B>(btn1.get(i)->second.second.first, +// btn1.get(i)->first, +// btn1.get(i)->second.second.second, +// btn1.get(i).getScore()); +// } +} +*/ + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::writeMLS ( FILE* pf ) const { + // Find best value at last frame... + int iBest = 0; + LogProb lgprEnd ; + for ( int i=0; i<BEAM_WIDTH; i++ ) + iBest = (aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(pf, "WARNING: There is no most likely sequence\n"); + return; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { + fprintf(pf,"HYPOTH %04d> ",fr-1); + String str; str<<aatnTrellis.get(fr,iBest).getId()<<" "<<aatnTrellis.get(fr,iBest).getBackData()<<"\n"; + fprintf(pf,str.c_array()); + //aatnTrellis.get(fr,iBest).getId().write(pf); + //fprintf(pf," "); + //aatnTrellis.get(fr,iBest).getBackData().write(pf); + //fprintf(pf,"\n"); + } + if(!OUTPUT_QUIET) fprintf(stderr,"Log prob of best path: %d\n",lgprEnd.toInt()); +} + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::writeMLS ( FILE* pf, const S& sLast ) const { + // Find best value at last frame... + int iBest = -1; + LogProb lgprEnd ; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if ( sLast.compareFinal(aatnTrellis.get(frameLast,i).getId()) ) +// if ( sLast.getSub1().get(0).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(0).getSub1() && +// sLast.getSub1().get(1).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(1).getSub1() ) + iBest = (-1==iBest || aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; +// iBest = i; + if ( -1==iBest ) iBest=0; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(pf, "WARNING: There is no most likely sequence\n"); + return; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { + fprintf(pf,"HYPOTH %04d> ",fr-1); + String str; str<<aatnTrellis.get(fr,iBest).getId()<<" "<<aatnTrellis.get(fr,iBest).getBackData()<<"\n"; + fprintf(pf,str.c_array()); + //aatnTrellis.get(fr,iBest).getId().write(pf); + //fprintf(pf," "); + //aatnTrellis.get(fr,iBest).getBackData().write(pf); + //fprintf(pf,"\n"); + } + if(!OUTPUT_QUIET) fprintf(stderr,"Log prob of best path: %d\n",lgprEnd.toInt()); +} + +//////////////////////////////////////////////////////////////////////////////// + +/* for getting the MLS in string form... */ +template <class MY, class MX, class S, class B> +std::list<string> HMM<MY,MX,S,B>::getMLS() const { + std::list<string> rList; + int iBest = 0; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + iBest = (aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + string r = "WARNING: There is no most likely sequence\n"; + rList.push_front(r); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { + char tmp[14]; + sprintf(tmp,"HYPOTH %04d> ", fr-1); + string tString(tmp); + tString += aatnTrellis.get(fr,iBest).getId().getString() + " " + + aatnTrellis.get(fr,iBest).getBackData().getString() + "\n"; + rList.push_front(tString); + } + return rList; +} + +/* for getting MLS in struct form... */ +template <class MY, class MX, class S, class B> +std::list<TrellNode<S,B> > HMM<MY,MX,S,B>::getMLSnodes() const { + std::list<TrellNode<S,B> > rList; + int iBest = 0; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + iBest = (aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(stderr, "WARNING: There is no most likely sequence\n"); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) + rList.push_front(aatnTrellis.get(fr,iBest)); + return rList; +} + +/* for getting the MLS in string form... */ +template <class MY, class MX, class S, class B> +std::list<string> HMM<MY,MX,S,B>::getMLS(const S& sLast) const { + std::list<string> rList; + int iBest = -1; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if ( sLast.compareFinal(aatnTrellis.get(frameLast,i).getId()) ) +// if ( sLast.getSub1().get(0).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(0).getSub1() && +// sLast.getSub1().get(1).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(1).getSub1() ) + iBest = (-1==iBest || aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; +// iBest = i; + if ( -1==iBest ) iBest=0; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + string r = "WARNING: There is no most likely sequence\n"; + rList.push_front(r); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { +//// char tmp[14]; +//// sprintf(tmp,"HYPOTH %04d> ", fr-1); +//// string tString(tmp); +//// tString += + string tString = +//// aatnTrellis.get(fr,iBest).getId().getString() + " " + + aatnTrellis.get(fr,iBest).getBackData().getString() +//// + "\n" + ; + tString = (B().getString()==tString) ? "" : " " + tString; // zero out or add space + rList.push_front(tString); + } + return rList; +} + +/* for getting MLS in struct form... */ +template <class MY, class MX, class S, class B> +std::list<TrellNode<S,B> > HMM<MY,MX,S,B>::getMLSnodes(const S& sLast) const { + std::list<TrellNode<S,B> > rList; + int iBest = -1; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if ( sLast.compareFinal(aatnTrellis.get(frameLast,i).getId()) ) +// if ( sLast.getSub1().get(0).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(0).getSub1() && +// sLast.getSub1().get(1).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(1).getSub1() ) + iBest = (-1==iBest || aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; +// iBest = i; + if ( -1==iBest ) iBest=0; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(stderr, "WARNING: There is no most likely sequence\n"); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) + rList.push_front(aatnTrellis.get(fr,iBest)); + return rList; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +//void HMM<MY,MX,S,B>::writeCurr ( FILE* pf, int f=-1 ) const { +void HMM<MY,MX,S,B>::writeCurr ( ostream& os, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + //fprintf(pf,"at f=%04d b=%04d: ",f,i); + os<<"at "<<std::setfill('0')<<std::setw(4)<<f<<" "<<std::setw(4)<<i<<": "; + //String str; str<<aatnTrellis.get(f,i).getId(); //.write(pf); + //fprintf(pf,str.c_array()); + os<<aatnTrellis.get(f,i).getId(); + if(f>0){ + //fprintf(pf," (from "); + //String str; str<<aatnTrellis.get(f-1,aatnTrellis.get(f,i).getSource().toInt()).getId(); //.write(pf); + //fprintf(pf,str.c_array()); + //fprintf(pf,")"); + os<<" (from "<<aatnTrellis.get(f-1,aatnTrellis.get(f,i).getSource().toInt()).getId()<<")"; + } + //fprintf(pf," : e^%0.6f\n",double(aatnTrellis.get(f,i).getLogProb().toInt())/100.0); + os<<" : e^"<<(double(aatnTrellis.get(f,i).getLogProb().toInt())/100.0)<<"\n"; + } +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::writeCurrSum ( FILE* pf, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + LogProb sum = 0.0; + LogProb logtop = 0.0; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + if(i==0) { logtop=aatnTrellis.get(f,i).getLogProb(); } + LogProb big1 = sum - logtop; + LogProb big2 = aatnTrellis.get(f,i).getLogProb() - logtop; + sum = LogProb( big1.toProb() + big2.toProb() ) + logtop; + } + fprintf(pf,"f=%04d sum=e^%0.6f\n",f,double(sum.toInt())/100.0); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +double HMM<MY,MX,S,B>::getCurrSum ( int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + LogProb negInfinity; + double sum = 0; + //LogProb logtop = 0; + int sumCount=0; + for ( int i=0; i<BEAM_WIDTH; i++ ) { + LogProb current = aatnTrellis.get(f,i).getLogProb(); + double currentValue = current.toDouble(); + // cerr << "Current value is " << currentValue << endl; + // if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + if (current != negInfinity) { + if (sumCount==0) { + sum = currentValue; + } else { + sum += currentValue; + } + sumCount += 1; + // if(i==0) { logtop=aatnTrellis.get(f,i).getLogProb(); } + //LogProb big1 = sum;// / logtop; + // LogProb big2 = aatnTrellis.get(f,i).getLogProb();// / logtop; + // sum = LogProb( big1.toProb() + big2.toProb() );// * logtop; + } + } + // cerr << "Summed " << sumCount << " values, total==" << sum << endl; + return sum;// /sumCount; + } else { + // Should an error be thrown here instead? + cerr << "ERROR: Invalid frame number: " << f << " of " << frameLast << endl; + return MIN_LOG_PROB; + } +} + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::gatherElementsInBeam( SafeArray1D<Id<int>,pair<S,LogProb> >* result, int f=-1) const { + result->init(BEAM_WIDTH); + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + for ( int i=0; i<BEAM_WIDTH && &(aatnTrellis.get(f,i))!=NULL; i++ ) { + result->set(i).first = aatnTrellis.get(f,i).getId(); + result->set(i).second = aatnTrellis.get(f,i).getLogProb(); + } + } else { + //TODO How should exceptional situations be handled? + cerr << "ERROR: Invalid argument " << f << " to gatherElementsInBeam" << endl; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::writeCurrEntropy ( FILE* pf, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + LogProb logh = 0.0; + LogProb logtop = 0.0; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + if(i==0) { logtop=aatnTrellis.get(f,i).getLogProb(); } + LogProb big1 = logh - logtop; + LogProb prob = aatnTrellis.get(f,i).getLogProb(); + LogProb big2 = prob - logtop; + double log2prob = double(aatnTrellis.get(f,i).getLogProb().toInt()) / double(LogProb(2).toInt()); + logh = LogProb( big1.toProb() - big2.toProb()*log2prob ) + logtop; + } + fprintf(pf,"f=%04d entropy=e^%0.6f\n",f,double(logh.toInt())/100.0); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +// finds the average depth. but... depends on HHMMLangModel used. below is for gf. +/* +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::writeCurrDepths ( FILE* pf, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + LogProb sum = 0.0; + LogProb logtop = 0.0; + Array<int> depths = Array<int>(); + Array<LogProb> logprobs = Array<LogProb>(); + double avgdepth = 0.0; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + + if(i==0) { logtop=aatnTrellis.get(f,i).getLogProb(); } + logprobs.set(i) = aatnTrellis.get(f,i).getLogProb(); + + // loop over values in S node to find lowest meaningful depth + for ( int j=0; j<aatnTrellis.get(f,i).getId().first.getSize(); j++) { + // store the depth, if it's equal to G_BOT/G_BOT + string ac = aatnTrellis.get(f,i).getId().first.get(j).first.getString(); + string aw = aatnTrellis.get(f,i).getId().first.get(j).second.getString(); + depths.set(i) = 0; + if (ac=="-" && aw=="-") { + //fprintf(pf,"depth at b%d = %d\n",i,j); + depths.set(i) = j; + break; + } + } + + LogProb big1 = sum - logtop; + LogProb big2 = aatnTrellis.get(f,i).getLogProb() - logtop; + sum = LogProb( big1.toProb() + big2.toProb() ) + logtop; + } + + // hack-y stuff because the logprob sum loses prob mass. this will give a quantized version + Array<LogProb> normprobs = Array<LogProb>(); + Prob normprobsum = Prob(); + for ( int i=0; i<BEAM_WIDTH; i++ ) { + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + normprobs.set(i) = logprobs.get(i)/sum; + normprobsum += normprobs.get(i).toProb(); + } + } + for ( int i=0; i<BEAM_WIDTH; i++ ) { + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + avgdepth += depths.get(i) * normprobs.get(i).toDouble()/normprobsum.toDouble(); + } + } + fprintf(pf,"f=%04d avg(d)=%1.6f\n",f,avgdepth); + } +} +*/ + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::writeFollRanked ( FILE* pf, int f, int b, const typename MX::RandVarType& x, bool b1 ) const { + const TrellNode<S,B>& tnsbPrev = aatnTrellis.get(f,b); + S sPrev = tnsbPrev.getId(); + fprintf(pf,"from f=%04d b=%04d: ",f,b); + cout<<tnsbPrev.getId()<<"\n"; + + //// DON'T KNOW WHAT TO DO WITH B1 (FINAL EOS FLAG) + + Heap < quad<int,typename MY::IterVal,LogProb,Id<int> >, outRank<int,typename MY::IterVal> > ashpiQueue; + typedef quad<int,typename MY::IterVal,LogProb,Id<int> > SHPI; + SHPI shpi, shpiTop; + int aCtr; + + shpi.first = b; + shpi.third = tnsbPrev.getScore(); + shpi.third *= my.setIterProb ( shpi.second, tnsbPrev.getId(), x, b1, aCtr=-1 ); + shpi.fourth = 0; + ashpiQueue.enqueue(shpi); + + //cerr<<"?? "<<shpi<<" "<<MY::IterVal::NUM_ITERS<<"\n"; + + // For each ranked value of transition destination... + for ( int iTrg=0; iTrg<BEAM_WIDTH && ashpiQueue.getSize()>0; iTrg++ ) { + // Iterate A* (breadth-first) search until a complete path is at the top of the queue... + while ( ashpiQueue.getSize() > 0 && ashpiQueue.getTop().fourth < MY::IterVal::NUM_ITERS ) { + // Remove top... + shpiTop = ashpiQueue.dequeueTop(); + // Fork off (try to advance each elementary variable a)... + for ( int a=shpiTop.fourth.toInt(); a<=MY::IterVal::NUM_ITERS; a++ ) { + // Copy top into new queue element... + shpi = shpiTop; + // At variable position -1, advance beam element for transition source... + if ( a == -1 ) shpi.first++; + // Incorporate prob from transition source... + shpi.third = aatnTrellis.get(f,shpi.first).getScore(); + if ( shpi.third > LogProb() ) { + // Try to advance variable at position a and return probability (subsequent variables set to first, probability ignored)... + shpi.third *= my.setIterProb ( shpi.second, aatnTrellis.get(f,shpi.first).getId(), x, b1, aCtr=a ); + // At end of variables, incorporate observation probability... + if ( a == MY::IterVal::NUM_ITERS && shpi.fourth != MY::IterVal::NUM_ITERS ) + { S s; my.setTrellDat(s,shpi.second); shpi.third *= mx.getProb(x,s); } + // Record variable position at which this element was forked off... + shpi.fourth = a; + ////cerr<<" from partial: "<<shpiTop<<"\n to partial: "<<shpi<<"\n"; + if ( shpi.third > LogProb() ) { + cerr<<" "<<shpi<<"\n"; + // If valid, add to queue... + ashpiQueue.enqueue(shpi); + } + } + } + } + if ( ashpiQueue.getSize() > 0 ) { + //cerr<<ashpiQueue.getTop()<<"\n"; + ashpiQueue.dequeueTop(); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +void HMM<MY,MX,S,B>::writeFoll ( FILE* pf, int f, int b, const typename MX::RandVarType& o ) const { + const TrellNode<S,B>& tnsbPrev = aatnTrellis.get(f,b); + S sPrev = tnsbPrev.getId(); + fprintf(pf,"from f=%04d b=%04d: ",f,b); + cout<<tnsbPrev.getId()<<"\n"; + // For each possible transition... + typename MY::IterVal h; + cout<<"HMM<MY,MX,S,B>::writeFoll OUT OF ORDER\n"; + /* + for ( bool b=my.setFirst(h,sPrev); b; b=my.setNext(h,sPrev) ) { + S s; + LogProb lgprO = mx.getProb(o,my.setTrellDat(s,h)); + LogProb lgprH = my.getProb(h,sPrev); + LogProb lgprFull = tnsbPrev.getLogProb() * lgprH * lgprO; + cout<<" ==("<<tnsbPrev.getLogProb().toInt()<<"*"<<lgprH.toInt()<<"*"<<lgprO.toInt()<<"="<<lgprFull.toInt()<<")==> "<<h<<"\n"; + } + */ +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +int HMM<MY,MX,S,B>::getBeamUsed ( int f=-1 ) const { + if ( -1==f ) f=frameLast; + int ctr=0; + if ( 0<=f && f<=frameLast ) + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + ctr++; + } + return ctr; +} + +#endif //_NL_HMM_ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-hmm2.h b/contrib/synlm/hhmm/rvtl/include/nl-hmm2.h new file mode 100644 index 000000000..711d589be --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-hmm2.h @@ -0,0 +1,911 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_HMM_ +#define _NL_HMM_ + +#include <list> +#include <string> +#include <boost/thread/thread.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/bind.hpp> +#include "nl-prob.h" +#include "nl-safeids.h" +#include "nl-beam.h" +using namespace std; + +typedef int Frame; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// NullBackDat - default empty back-pointer data; can replace with word or sem relation +// +//////////////////////////////////////////////////////////////////////////////// + +template <class MH> +class NullBackDat { + static const string sDummy; + public: + NullBackDat () {} + NullBackDat (const MH& mh) {} + void write (FILE*) const {} + string getString() const { return sDummy; } +}; +template <class MH> +const string NullBackDat<MH>::sDummy ( "" ); + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Index - pointer to source in previous beam heap +// +//////////////////////////////////////////////////////////////////////////////// + +class Index : public Id<int> { + public: + Index ( ) { } + Index (int i) {set(i);} + Index& operator++ ( ) {set(toInt()+1); return *this;} +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// TrellNode - node in viterbi trellis +// +//////////////////////////////////////////////////////////////////////////////// + +template <class X, class B> +class TrellNode { + private: + + // Data members... + Index indSource; + B backptrData; + X xId; + LogProb lgprMax; + + public: + + // Constructor / destructor methods... + TrellNode ( ) { } + TrellNode ( const Index& indS, const X& xI, const B& bDat, LogProb lgpr) + { indSource=indS; xId=xI; lgprMax=lgpr; backptrData=bDat; /* fo = -1; */ } + + // Specification methods... + const Index& setSource ( ) const { return indSource; } + const B& setBackData( ) const { return backptrData; } + const X& setId ( ) const { return xId; } + LogProb& setScore ( ) { return lgprMax; } + + // Extraction methods... + bool operator== ( const TrellNode<X,B>& tnxb ) const { return(xId==tnxb.xId); } +// size_t getHashKey ( ) const { return xId.getHashKey(); } + const Index& getSource ( ) const { return indSource; } + const B& getBackData( ) const { return backptrData; } + const X& getId ( ) const { return xId; } + LogProb getLogProb ( ) const { return lgprMax; } + LogProb getScore ( ) const { return lgprMax; } +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// HMM +// +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X=typename MH::RandVarType, class B=NullBackDat<typename MH::RandVarType> > +class HMM { + private: + typedef std::pair<Index,B> IB; + // Data members... + const MH& mh; + const MO& mo; + SafeArray2D<Id<Frame>,Id<int>,TrellNode<X,B> > aatnTrellis; + Frame frameLast; + int iNextNode; + public: + // Static member varaibles... + static bool OUTPUT_QUIET; + static bool OUTPUT_NOISY; + static bool OUTPUT_VERYNOISY; + static int BEAM_WIDTH; + // Constructor / destructor methods... + HMM ( const MH& mh1, const MO& mo1 ) : mh(mh1), mo(mo1) { } + // Specification methods... + void init ( int, int, const X& ) ; + void updateRanked ( const typename MO::RandVarType& ) ; + void updateSerial ( const typename MO::RandVarType& ) ; + void updatePara ( const typename MO::RandVarType& ) ; + void each ( const typename MO::RandVarType&, Beam<LogProb,X,IB>&, SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> >& ) ; + // Extraction methods... + const TrellNode<X,B>& getTrellNode ( int i ) const { return aatnTrellis.get(frameLast,i); } + int getBeamUsed ( int ) const ; + // Input / output methods... + void writeMLS ( FILE* ) const ; + void writeMLS ( FILE*, const X& ) const ; + void writeCurr ( FILE*, int ) const ; + void writeCurrSum ( FILE*, int ) const ; + void writeCurrEntropy ( FILE*, int ) const; + //void writeCurrDepths ( FILE*, int ) const; + void writeFoll ( FILE*, int, int, const typename MO::RandVarType& ) const ; + void writeFollRanked ( FILE*, int, int, const typename MO::RandVarType& ) const ; + std::list<string> getMLS() const; + std::list<TrellNode<X,B> > getMLSnodes() const; + std::list<string> getMLS(const X&) const; + std::list<TrellNode<X,B> > getMLSnodes(const X&) const; +}; +template <class MH, class MO, class X, class B> bool HMM<MH,MO,X,B>::OUTPUT_QUIET = false; +template <class MH, class MO, class X, class B> bool HMM<MH,MO,X,B>::OUTPUT_NOISY = false; +template <class MH, class MO, class X, class B> bool HMM<MH,MO,X,B>::OUTPUT_VERYNOISY = false; +template <class MH, class MO, class X, class B> int HMM<MH,MO,X,B>::BEAM_WIDTH = 1; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::init ( int numFr, int numX, const X& x ) { + + // Alloc trellis... + BEAM_WIDTH = numX; + aatnTrellis.init(numFr,BEAM_WIDTH); + + frameLast=0; + + // Set initial element at first time slice... + aatnTrellis.set(frameLast,0) = TrellNode<X,B> ( Index(0), x, B(), 0 ) ; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class A, class B> +bool outRank ( const quad<A,B,LogProb,Id<int> >& a1, + const quad<A,B,LogProb,Id<int> >& a2 ) { return (a1.third>a2.third); } + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::updateRanked ( const typename MO::RandVarType& o ) { + // Increment frame counter... + frameLast++; + + // Init beam for new frame... + Beam<LogProb,X,IB> btn(BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted (BEAM_WIDTH); + + Heap < quad<int,typename MH::IterVal,LogProb,Id<int> >, outRank<int,typename MH::IterVal> > axhpiQueue; + typedef quad<int,typename MH::IterVal,LogProb,Id<int> > XHPI; + XHPI xhpi, xhpiTop; + int aCtr; + + axhpiQueue.clear(); + //xhpi.first = -1; + //xhpi.second = HModel::IterVal(); + //xhpi.third = 1.0; + xhpi.first = 0; + xhpi.third = aatnTrellis.get(frameLast-1,xhpi.first).getScore(); + //cerr<<" xhpi not inialized yet? "<<xhpi.second<<endl; + //xhpi.second is current hidden state h, the second parameter is hP or sP. + xhpi.third *= mh.setIterProb ( xhpi.second, aatnTrellis.get(frameLast-1,xhpi.first).getId(), aCtr=-1 ); + //X x; mh.setTrellDat(x,xhpi.second); + //xhpi.fourth is a + xhpi.fourth = -1; + //cerr<<"xhphi in the begining: "<<xhpi<<"\n"; + axhpiQueue.enqueue(xhpi); + + bool bFull=false; + + // For each ranked value of transition destination... + for ( int iTrg=0; !bFull && axhpiQueue.getSize()>0; iTrg++ ) { + // Iterate A* (best-first) search until a complete path is at the top of the queue... + //in order to make while loop stop, either axhphiQueue.getSize<0 (this is abnormal) or axhpiQueue.getTop().fourth<MH::IterVal::NUM_ITERS + //it seems that the second condition sohould be the normal stopping criterio. How does it happen? + //I know how, xhpi.fourth will say which depth of the search starts. From that state, a is 0 and decreasing to the end. above that, + //the value is increasing to the first variable. i.e., that is the water-edge of positive a and negative a. + // cerr<<axhpiQueue.getTop().fourth<<" axhpiQueue.getTop().fourth vs MH::IterVal::NUM_ITERS "<<MH::IterVal::NUM_ITERS<<endl; + while ( axhpiQueue.getSize() > 0 && axhpiQueue.getTop().fourth < MH::IterVal::NUM_ITERS ) { + // Remove top... + xhpiTop = axhpiQueue.dequeueTop(); + // Fork off (try to advance each elementary variable a)... + for ( int a=xhpiTop.fourth.toInt(); a<=MH::IterVal::NUM_ITERS; a++ ) { + // Copy top into new queue element... + xhpi = xhpiTop; + // At variable position -1, advance beam element for transition source... + if ( a == -1 ) xhpi.first++; + //cerr<<" xhpi.first in xhpi.first++: "<<xhpi.first<<" xhpi: "<<xhpi; + // Incorporate prob from transition source... + xhpi.third = aatnTrellis.get(frameLast-1,xhpi.first).getScore(); + // cerr<<" xhpi.third: "<<xhpi.third<<" framLast-1: "<<frameLast<<" xhpi: "<<xhpi<<endl; + if ( xhpi.third > LogProb() ) { + // Try to advance variable at position a and return probability (subsequent variables set to first, probability ignored)... + xhpi.third *= mh.setIterProb ( xhpi.second, aatnTrellis.get(frameLast-1,xhpi.first).getId(), aCtr=a ); + // At end of variables, incorporate observation probability... + if ( a == MH::IterVal::NUM_ITERS && xhpi.fourth != MH::IterVal::NUM_ITERS ) + { X x; mh.setTrellDat(x,xhpi.second); xhpi.third *= mo.getProb(o,x); } + // Record variable position at which this element was forked off... + //cerr<<" a after the xhpi.fourth=a: "<<a<<" xhpi.fourth: "<<xhpi.fourth<<" from partial: "<<xhpiTop<<"\n to partial: "<<xhpi<<" NUM_ITERS: "<<MH::IterVal::NUM_ITERS<<"\n"; + xhpi.fourth = a; + //cerr<<" a after the xhpi.fourth=a: "<<a<<" xhpi.fourth: "<<xhpi.fourth<<" from partial: "<<xhpiTop<<"\n to partial: "<<xhpi<<" NUM_ITERS: "<<MH::IterVal::NUM_ITERS<<"\n"; + if ( xhpi.third > LogProb() ) { + ////if ( frameLast == 4 ) cerr<<" from partial: "<<xhpiTop<<"\n to partial: "<<xhpi<<"\n"; + // If valid, add to queue... + axhpiQueue.enqueue(xhpi); + //cerr<<"--------------------\n"<<axhpiQueue; + } + } + } + //the inside for loop is done here. + // Remove top... + //cerr<<"/-----A-----\\\n"<<axhpiQueue<<"\\-----A-----/\n"; + //if ( axhpiQueue.getTop().fourth != MH::IterVal::NUM_ITERS ) axhpiQueue.dequeueTop(); + ////cerr<<"/-----B-----\\\n"<<axhpiQueue<<"\\-----B-----/\n"; + //cerr<<axhpiQueue.getSize()<<" queue elems, "<<axhpiQueue.getTop()<<"\n"; + } + ////cerr<<"-----*-----\n"<<axhpiQueue<<"-----*-----\n"; + //cerr<<" outside the while loop: "<<axhpiQueue.getSize()<<" queue elems **\n"; + // Add best transition (top of queue)... + //mo.getProb(o,mh.setTrellDat(axhpiQueue.getTop().first,axhpiQueue.getTop().second)); + if ( axhpiQueue.getSize() > 0 ) { + X x; mh.setTrellDat(x,axhpiQueue.getTop().second); + bFull |= btn.tryAdd ( x, IB(axhpiQueue.getTop().first,mh.setBackDat(axhpiQueue.getTop().second)), axhpiQueue.getTop().third ); + //cerr<<axhpiQueue.getSize()<<" queue elems A "<<axhpiQueue.getTop()<<"\n"; + //cerr<<"/-----A-----\\\n + bFull: "<<bFull<<"\naxhpiQueue: \n"<<axhpiQueue<<"\\-----A-----/\n"; + axhpiQueue.dequeueTop(); + //cerr<<"/-----B-----\\\n"<<axhpiQueue<<"\\-----B-----/\n"; + //cerr<<axhpiQueue.getSize()<<" queue elems B "<<axhpiQueue.getTop()<<"\n"; + //cerr<<"."; cerr.flush(); + } + } + //the outmost for loop is over here. + ////cerr<<"-----*-----\n"<<axhpiQueue<<"-----*-----\n"; + btn.sort(atnSorted); + // Copy sorted beam to trellis... + for(int i=0;i<BEAM_WIDTH;i++) { + const std::pair<std::pair<X,IB>,LogProb>* tn1 = &atnSorted.get(i); + //cerr<<" tn1: "<<tn1->first.second.first<<" "<<tn1->first.first<<" "<<tn1->first.second.second<<" "<<tn1->second<<endl; + aatnTrellis.set(frameLast,i)=TrellNode<X,B>(tn1->first.second.first, + tn1->first.first, + tn1->first.second.second, + tn1->second); + } + + mh.update(); +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::updateSerial ( const typename MO::RandVarType& o ) { + // Increment frame counter... + frameLast++; + + // Init beam for new frame... + Beam<LogProb,X,IB> btn(BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted (BEAM_WIDTH); + +// // Copy beam to trellis... +// for ( int i=0; i<BEAM_WIDTH; i++ ) +// btn.set ( i, aatnTrellis.set(frameLast,i) ); + + // For each possible hidden value for previous frame... + for ( int i=0; i<BEAM_WIDTH; i++ ) { + const TrellNode<X,B>& tnxbPrev = aatnTrellis.get(frameLast-1,i); + X xPrev = tnxbPrev.getId(); + // If prob still not below beam minimum... + if ( aatnTrellis.get(frameLast-1,i).getLogProb() > btn.getMin().getScore() ) { + //if (OUTPUT_VERYNOISY) { fprintf(stderr,"FROM: "); xPrev.write(stderr); fprintf(stderr,"\n"); } + // For each possible transition... + typename MH::IterVal h; + for ( bool b=mh.setFirst(h,xPrev); b; b=mh.setNext(h,xPrev) ) { + //if (OUTPUT_VERYNOISY) { fprintf(stderr," TO: "); h.write(stderr); fprintf(stderr," %d*%d*%d\n",tnxbPrev.getLogProb().toInt(),lgprH.toInt(),lgprO.toInt()); } + X x; +#ifdef O_BEFORE_H + LogProb lgprO = mo.getProb(o,mh.setTrellDat(x,h)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprO ) continue; + LogProb lgprH = mh.getProb(h,xPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprH ) continue; +#else + LogProb lgprH = mh.getProb(h,xPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprH ) continue; + LogProb lgprO = mo.getProb(o,mh.setTrellDat(x,h)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprO ) continue; +#endif + LogProb lgprFull = aatnTrellis.get(frameLast-1,i).getLogProb() * lgprH * lgprO; + if (OUTPUT_VERYNOISY) { + cout<<" "<<tnxbPrev.getId()<<" ==("<<tnxbPrev.getLogProb().toInt()<<"*"<<lgprH.toInt()<<"*"<<lgprO.toInt()<<"="<<lgprFull.toInt()<<")==> "<<h<<"\n"; + //xPrev.write(stdout); + //fprintf(stdout," ==(%d*%d*%d=%d)==> ",tnxbPrev.getLogProb().toInt(),lgprH.toInt(),lgprO.toInt(),lgprFull.toInt()); + //h.write(stdout); fprintf(stdout," "); x.write(stdout); + //fprintf(stdout,"\n"); + } + // Incorporate into trellis... + btn.tryAdd ( x, IB(i,mh.setBackDat(h)), lgprFull ); + //if(OUTPUT_VERYNOISY) + // fprintf ( stderr," (X_t-1:[e^%0.6f] * H:e^%0.6f * O:e^%0.6f = X_t:[e^%0.6f])\n", + // float(aatnTrellis.get(frameLast-1,i).getLogProb().toInt())/100.0, + // float(lgprH.toInt())/100.0, + // float(lgprO.toInt())/100.0, + // float(lgprFull.toInt())/100.0 ) ; + } + } + } + +// for(int i=0;i<BEAM_WIDTH;i++) { +// fprintf(stderr,"> "); btn.get(i)->first.write(stderr); fprintf(stderr,"\n"); +// } + + btn.sort(atnSorted); + + // Copy sorted beam to trellis... + for(int i=0;i<BEAM_WIDTH;i++) { + const std::pair<std::pair<X,IB>,LogProb>* tn1 = &atnSorted.get(i); + aatnTrellis.set(frameLast,i)=TrellNode<X,B>(tn1->first.second.first, + tn1->first.first, + tn1->first.second.second, + tn1->second); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +boost::mutex mutexHmmCtrSema; +boost::mutex mutexHmmParanoiaLock; + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::each ( const typename MO::RandVarType& o, Beam<LogProb,X,IB>& btn, SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> >& atnOut ) { + + //// for(int j=0; j<100; j+=1) for(int i=0; i<1000000; i+=1); // peg processor +//{ +// boost::mutex::scoped_lock lock1(mutexHmmParanoiaLock); + + while ( true ) { + int i; + bool bStop=false; + { boost::mutex::scoped_lock lock(mutexHmmCtrSema); + if ( (i=iNextNode) < BEAM_WIDTH ) iNextNode++; + else bStop = true; + } // locked to ensure no double-duty + if ( bStop ) break; + + const TrellNode<X,B>& tnxbPrev = aatnTrellis.get(frameLast-1,i); + // If prob still not below beam minimum... + if ( tnxbPrev.getLogProb() > btn.getMin().getScore() ) { + //if (OUTPUT_VERYNOISY) { fprintf(stderr,"FROM: "); tnxbPrev.getId().write(stderr); fprintf(stderr,"\n"); } + + // For each possible transition... + const X& xPrev = tnxbPrev.getId(); + typename MH::IterVal h; + for ( bool b=mh.setFirst(h,xPrev); b; b=mh.setNext(h,xPrev) ) { + X x; + LogProb lgprO; + LogProb lgprH; + LogProb lgprFull; + #ifdef O_BEFORE_H ////////////////////////////////////////////////////// + lgprO = mo.getProb(o,mh.setTrellDat(x,h)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprO ) continue; + lgprH = mh.getProb(h,xPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprH ) continue; + #else ////////////////////////////////////////////////////////////////// + lgprH = mh.getProb(h,xPrev); if ( !OUTPUT_VERYNOISY && LogProb()==lgprH ) continue; + lgprO = mo.getProb(o,mh.setTrellDat(x,h)); if ( !OUTPUT_VERYNOISY && LogProb()==lgprO ) continue; + #endif ///////////////////////////////////////////////////////////////// + lgprFull = tnxbPrev.getLogProb() * lgprH * lgprO; + if (OUTPUT_VERYNOISY) { + boost::mutex::scoped_lock lock1(mutexHmmParanoiaLock); + //fprintf(stderr," TO: "); h.write(stderr); fprintf(stderr,"\n"); + cout<<" "<<tnxbPrev.getId()<<" ==("<<tnxbPrev.getLogProb().toInt()<<"*"<<lgprH.toInt()<<"*"<<lgprO.toInt()<<"="<<lgprFull.toInt()<<")==> "<<h<<"\n"; + //tnxbPrev.getId().write(stdout); + //fprintf(stdout," ==(%d*%d*%d=%d)==> ",tnxbPrev.getLogProb().toInt(),lgprH.toInt(),lgprO.toInt(),lgprFull.toInt()); + //h.write(stdout); fprintf(stdout," "); x.write(stdout); + //fprintf(stdout,"\n"); + } + // Incorporate into trellis... + btn.tryAdd ( x, IB(i,mh.setBackDat(h)), lgprFull ); +// if(OUTPUT_VERYNOISY) +// fprintf ( stderr," (X_t-1:[e^%0.6f] * H:e^%0.6f * O:e^%0.6f = X_t:[e^%0.6f])\n", +// float(aatnTrellis.get(frameLast-1,i).getLogProb().toInt())/100.0, +// float(lgprH.toInt())/100.0, +// float(lgprO.toInt())/100.0, +// float(lgprFull.toInt())/100.0 ) ; + } + } + } + + if (OUTPUT_NOISY) + { boost::mutex::scoped_lock lock1(mutexHmmParanoiaLock); + //// btn.sort(atnOut); + fprintf(stdout,"========================================\n"); + for(int i=0;i<BEAM_WIDTH;i++) { + String str; str<<" "<<btn.get(i)->first<<":"<<btn.get(i)->second.second.first.toInt()<<":"<<btn.get(i).getScore().toInt()<<"\n"; + fprintf(stdout,str.c_array()); + //fprintf(stdout," "); btn.get(i)->first.write(stdout); fprintf(stdout,":%d:%d\n",btn.get(i)->second.second.first.toInt(),btn.get(i).getScore().toInt()); + //// atnOut.get(i).first.first.write(stdout); fprintf(stdout,":%d\n",(int)atnOut.get(i).second); + } + fprintf(stdout,"----------------------------------------\n"); + } + +//} +} + +//// Something there is (about boost::bind) that doesn't love a ref. So use a pointer for hmm and beam. +template <class MH, class MO, class X, class B> + void global_parahmm_each ( HMM<MH,MO,X,B>* phmm, const typename MO::RandVarType* po, Beam<LogProb,X,std::pair<Index,B> >* pbtn, + SafeArray1D<Id<int>,std::pair<std::pair<X,std::pair<Index,B> >,LogProb> >* patn ) { +//void HMM<MH,MO,X,B>::each_static ( HMM<MH,MO,X,B>* phmm, const MO& o, Beam<LogProb,X,IB>* pbtn, SafeArray1D<int,std::pair<std::pair<X,IB>,LogProb> >* patn ) { + phmm->each(*po,*pbtn,*patn); +} + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::updatePara ( const typename MO::RandVarType& o ) { + // Increment frame counter... + frameLast++; + +// fprintf(stdout,"++++%d",frameLast); o.write(stdout); fprintf(stdout,"\n"); + + // Init beam for new frame... + Beam<LogProb,X,IB> btn1 (BEAM_WIDTH); + Beam<LogProb,X,IB> btn2 (BEAM_WIDTH); + Beam<LogProb,X,IB> btn3 (BEAM_WIDTH); + Beam<LogProb,X,IB> btn4 (BEAM_WIDTH); + Beam<LogProb,X,IB> btn5 (BEAM_WIDTH); + Beam<LogProb,X,IB> btn6 (BEAM_WIDTH); + Beam<LogProb,X,IB> btn7 (BEAM_WIDTH); + Beam<LogProb,X,IB> btn8 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted1 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted2 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted3 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted4 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted5 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted6 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted7 (BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<X,IB>,LogProb> > atnSorted8 (BEAM_WIDTH); + + //if(0==frameLast%20) + // fprintf(stderr,"frame %d...\n",frameLast); + iNextNode=0; + boost::thread th1(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn1,&atnSorted1)); + boost::thread th2(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn2,&atnSorted2)); + boost::thread th3(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn3,&atnSorted3)); + boost::thread th4(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn4,&atnSorted4)); + boost::thread th5(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn5,&atnSorted5)); + boost::thread th6(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn6,&atnSorted6)); + boost::thread th7(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn7,&atnSorted7)); + boost::thread th8(boost::bind(&global_parahmm_each<MH,MO,X,B>,this,&o,&btn8,&atnSorted8)); + th1.join(); + th2.join(); + th3.join(); + th4.join(); + th5.join(); + th6.join(); + th7.join(); + th8.join(); + + for(int i=0;i<BEAM_WIDTH;i++) { + btn1.tryAdd(btn2.get(i)->first, btn2.get(i)->second.second, btn2.get(i).getScore()); + btn1.tryAdd(btn3.get(i)->first, btn3.get(i)->second.second, btn3.get(i).getScore()); + btn1.tryAdd(btn4.get(i)->first, btn4.get(i)->second.second, btn4.get(i).getScore()); + btn1.tryAdd(btn5.get(i)->first, btn5.get(i)->second.second, btn5.get(i).getScore()); + btn1.tryAdd(btn6.get(i)->first, btn6.get(i)->second.second, btn6.get(i).getScore()); + btn1.tryAdd(btn7.get(i)->first, btn7.get(i)->second.second, btn7.get(i).getScore()); + btn1.tryAdd(btn8.get(i)->first, btn8.get(i)->second.second, btn8.get(i).getScore()); + } + + btn1.sort(atnSorted1); + + for(int i=0;i<BEAM_WIDTH;i++) { + const std::pair<std::pair<X,IB>,LogProb>* tn1 = &atnSorted1.get(i); + aatnTrellis.set(frameLast,i)=TrellNode<X,B>(tn1->first.second.first, + tn1->first.first, + tn1->first.second.second, + tn1->second); + } + +// for(int i=0;i<BEAM_WIDTH;i++) { +// aatnTrellis.set(frameLast,i)=TrellNode<X,B>(btn1.get(i)->second.second.first, +// btn1.get(i)->first, +// btn1.get(i)->second.second.second, +// btn1.get(i).getScore()); +// } +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeMLS ( FILE* pf ) const { + // Find best value at last frame... + int iBest = 0; + LogProb lgprEnd ; + for ( int i=0; i<BEAM_WIDTH; i++ ) + iBest = (aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(pf, "WARNING: There is no most likely sequence\n"); + return; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { + fprintf(pf,"HYPOTH %04d> ",fr-1); + String str; str<<aatnTrellis.get(fr,iBest).getId()<<" "<<aatnTrellis.get(fr,iBest).getBackData()<<"\n"; + fprintf(pf,str.c_array()); + //aatnTrellis.get(fr,iBest).getId().write(pf); + //fprintf(pf," "); + //aatnTrellis.get(fr,iBest).getBackData().write(pf); + //fprintf(pf,"\n"); + } + if(!OUTPUT_QUIET) fprintf(stderr,"Log prob of best path: %d\n",lgprEnd.toInt()); +} + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeMLS ( FILE* pf, const X& xLast ) const { + // Find best value at last frame... + int iBest = -1; + LogProb lgprEnd ; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if ( xLast.compareFinal(aatnTrellis.get(frameLast,i).getId()) ) +// if ( xLast.getSub1().get(0).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(0).getSub1() && +// xLast.getSub1().get(1).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(1).getSub1() ) + iBest = (-1==iBest || aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; +// iBest = i; + if ( -1==iBest ) iBest=0; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(pf, "WARNING: There is no most likely sequence\n"); + return; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { + fprintf(pf,"HYPOTH %04d> ",fr-1); + String str; str<<aatnTrellis.get(fr,iBest).getId()<<" "<<aatnTrellis.get(fr,iBest).getBackData()<<"\n"; + fprintf(pf,str.c_array()); + //aatnTrellis.get(fr,iBest).getId().write(pf); + //fprintf(pf," "); + //aatnTrellis.get(fr,iBest).getBackData().write(pf); + //fprintf(pf,"\n"); + } + if(!OUTPUT_QUIET) fprintf(stderr,"Log prob of best path: %d\n",lgprEnd.toInt()); +} + +//////////////////////////////////////////////////////////////////////////////// + +/* for getting the MLS in string form... */ +template <class MH, class MO, class X, class B> +std::list<string> HMM<MH,MO,X,B>::getMLS() const { + std::list<string> rList; + int iBest = 0; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + iBest = (aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + string r = "WARNING: There is no most likely sequence\n"; + rList.push_front(r); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { + char tmp[14]; + sprintf(tmp,"HYPOTH %04d> ", fr-1); + string tString(tmp); + tString += aatnTrellis.get(fr,iBest).getId().getString() + " " + + aatnTrellis.get(fr,iBest).getBackData().getString() + "\n"; + rList.push_front(tString); + } + return rList; +} + +/* for getting MLS in struct form... */ +template <class MH, class MO, class X, class B> +std::list<TrellNode<X,B> > HMM<MH,MO,X,B>::getMLSnodes() const { + std::list<TrellNode<X,B> > rList; + int iBest = 0; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + iBest = (aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(stderr, "WARNING: There is no most likely sequence\n"); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) + rList.push_front(aatnTrellis.get(fr,iBest)); + return rList; +} + +/* for getting the MLS in string form... */ +template <class MH, class MO, class X, class B> +std::list<string> HMM<MH,MO,X,B>::getMLS(const X& xLast) const { + std::list<string> rList; + int iBest = -1; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if ( xLast.compareFinal(aatnTrellis.get(frameLast,i).getId()) ) +// if ( xLast.getSub1().get(0).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(0).getSub1() && +// xLast.getSub1().get(1).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(1).getSub1() ) + iBest = (-1==iBest || aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; +// iBest = i; + if ( -1==iBest ) iBest=0; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + string r = "WARNING: There is no most likely sequence\n"; + rList.push_front(r); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) { +//// char tmp[14]; +//// sprintf(tmp,"HYPOTH %04d> ", fr-1); +//// string tString(tmp); +//// tString += + string tString = +//// aatnTrellis.get(fr,iBest).getId().getString() + " " + + aatnTrellis.get(fr,iBest).getBackData().getString() +//// + "\n" + ; + tString = (B().getString()==tString) ? "" : " " + tString; // zero out or add space + rList.push_front(tString); + } + return rList; +} + +/* for getting MLS in struct form... */ +template <class MH, class MO, class X, class B> +std::list<TrellNode<X,B> > HMM<MH,MO,X,B>::getMLSnodes(const X& xLast) const { + std::list<TrellNode<X,B> > rList; + int iBest = -1; + LogProb lgprEnd; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if ( xLast.compareFinal(aatnTrellis.get(frameLast,i).getId()) ) +// if ( xLast.getSub1().get(0).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(0).getSub1() && +// xLast.getSub1().get(1).getSub1()==aatnTrellis.get(frameLast,i).getId().getSub1().get(1).getSub1() ) + iBest = (-1==iBest || aatnTrellis.get(frameLast,i).getLogProb() > aatnTrellis.get(frameLast,iBest).getLogProb()) ? i : iBest ; +// iBest = i; + if ( -1==iBest ) iBest=0; + lgprEnd = aatnTrellis.get(frameLast,iBest).getLogProb(); + if(lgprEnd == LogProb()){ + fprintf(stderr, "WARNING: There is no most likely sequence\n"); + return rList; + } + // Trace back most likely sequence... + for ( Frame fr=frameLast; fr-1>=0; iBest=aatnTrellis.get(fr--,iBest).getSource().toInt() ) + rList.push_front(aatnTrellis.get(fr,iBest)); + return rList; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeCurr ( FILE* pf, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + fprintf(pf,"at f=%04d b=%04d: ",f,i); + String str; str<<aatnTrellis.get(f,i).getId(); //.write(pf); + fprintf(pf,str.c_array()); + if(f>0){ + fprintf(pf," (from "); + String str; str<<aatnTrellis.get(f-1,aatnTrellis.get(f,i).getSource().toInt()).getId(); //.write(pf); + fprintf(pf,str.c_array()); + fprintf(pf,")"); + } + fprintf(pf," : e^%0.6f\n",double(aatnTrellis.get(f,i).getLogProb().toInt())/100.0); + } +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeCurrSum ( FILE* pf, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + LogProb sum = 0.0; + LogProb logtop = 0.0; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + if(i==0) { logtop=aatnTrellis.get(f,i).getLogProb(); } + LogProb big1 = sum - logtop; + LogProb big2 = aatnTrellis.get(f,i).getLogProb() - logtop; + sum = LogProb( big1.toProb() + big2.toProb() ) + logtop; + } + fprintf(pf,"f=%04d sum=e^%0.6f\n",f,double(sum.toInt())/100.0); + } +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeCurrEntropy ( FILE* pf, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + LogProb logh = 0.0; + LogProb logtop = 0.0; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + if(i==0) { logtop=aatnTrellis.get(f,i).getLogProb(); } + LogProb big1 = logh - logtop; + LogProb prob = aatnTrellis.get(f,i).getLogProb(); + LogProb big2 = prob - logtop; + double log2prob = double(aatnTrellis.get(f,i).getLogProb().toInt()) / double(LogProb(2).toInt()); + logh = LogProb( big1.toProb() - big2.toProb()*log2prob ) + logtop; + } + fprintf(pf,"f=%04d entropy=e^%0.6f\n",f,double(logh.toInt())/100.0); + } +} + + +//////////////////////////////////////////////////////////////////////////////// + +// finds the average depth. but... depends on HHMMLangModel used. below is for gf. +/* +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeCurrDepths ( FILE* pf, int f=-1 ) const { + if ( -1==f ) f=frameLast; + if ( 0<=f && f<=frameLast ) { + LogProb sum = 0.0; + LogProb logtop = 0.0; + Array<int> depths = Array<int>(); + Array<LogProb> logprobs = Array<LogProb>(); + double avgdepth = 0.0; + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + + if(i==0) { logtop=aatnTrellis.get(f,i).getLogProb(); } + logprobs.set(i) = aatnTrellis.get(f,i).getLogProb(); + + // loop over values in S node to find lowest meaningful depth + for ( int j=0; j<aatnTrellis.get(f,i).getId().first.getSize(); j++) { + // store the depth, if it's equal to G_BOT/G_BOT + string ac = aatnTrellis.get(f,i).getId().first.get(j).first.getString(); + string aw = aatnTrellis.get(f,i).getId().first.get(j).second.getString(); + depths.set(i) = 0; + if (ac=="-" && aw=="-") { + //fprintf(pf,"depth at b%d = %d\n",i,j); + depths.set(i) = j; + break; + } + } + + LogProb big1 = sum - logtop; + LogProb big2 = aatnTrellis.get(f,i).getLogProb() - logtop; + sum = LogProb( big1.toProb() + big2.toProb() ) + logtop; + } + + // hack-y stuff because the logprob sum loses prob mass. this will give a quantized version + Array<LogProb> normprobs = Array<LogProb>(); + Prob normprobsum = Prob(); + for ( int i=0; i<BEAM_WIDTH; i++ ) { + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + normprobs.set(i) = logprobs.get(i)/sum; + normprobsum += normprobs.get(i).toProb(); + } + } + for ( int i=0; i<BEAM_WIDTH; i++ ) { + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + avgdepth += depths.get(i) * normprobs.get(i).toDouble()/normprobsum.toDouble(); + } + } + fprintf(pf,"f=%04d avg(d)=%1.6f\n",f,avgdepth); + } +} +*/ + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeFollRanked ( FILE* pf, int f, int b, const typename MO::RandVarType& o ) const { + const TrellNode<X,B>& tnxbPrev = aatnTrellis.get(f,b); + X xPrev = tnxbPrev.getId(); + fprintf(pf,"from f=%04d b=%04d: ",f,b); + cout<<tnxbPrev.getId()<<"\n"; + + Heap < quad<int,typename MH::IterVal,LogProb,Id<int> >, outRank<int,typename MH::IterVal> > axhpiQueue; + typedef quad<int,typename MH::IterVal,LogProb,Id<int> > XHPI; + XHPI xhpi, xhpiTop; + int aCtr; + + xhpi.first = b; + xhpi.third = tnxbPrev.getScore(); + xhpi.third *= mh.setIterProb ( xhpi.second, tnxbPrev.getId(), aCtr=-1 ); + xhpi.fourth = 0; + axhpiQueue.enqueue(xhpi); + + //cerr<<"?? "<<xhpi<<" "<<MH::IterVal::NUM_ITERS<<"\n"; + + // For each ranked value of transition destination... + for ( int iTrg=0; iTrg<BEAM_WIDTH && axhpiQueue.getSize()>0; iTrg++ ) { + // Iterate A* (breadth-first) search until a complete path is at the top of the queue... + while ( axhpiQueue.getSize() > 0 && axhpiQueue.getTop().fourth < MH::IterVal::NUM_ITERS ) { + // Remove top... + xhpiTop = axhpiQueue.dequeueTop(); + // Fork off (try to advance each elementary variable a)... + for ( int a=xhpiTop.fourth.toInt(); a<=MH::IterVal::NUM_ITERS; a++ ) { + // Copy top into new queue element... + xhpi = xhpiTop; + // At variable position -1, advance beam element for transition source... + if ( a == -1 ) xhpi.first++; + // Incorporate prob from transition source... + xhpi.third = aatnTrellis.get(f,xhpi.first).getScore(); + if ( xhpi.third > LogProb() ) { + // Try to advance variable at position a and return probability (subsequent variables set to first, probability ignored)... + xhpi.third *= mh.setIterProb ( xhpi.second, aatnTrellis.get(f,xhpi.first).getId(), aCtr=a ); + // At end of variables, incorporate observation probability... + if ( a == MH::IterVal::NUM_ITERS && xhpi.fourth != MH::IterVal::NUM_ITERS ) + { X x; mh.setTrellDat(x,xhpi.second); xhpi.third *= mo.getProb(o,x); } + // Record variable position at which this element was forked off... + xhpi.fourth = a; + ////cerr<<" from partial: "<<xhpiTop<<"\n to partial: "<<xhpi<<"\n"; + if ( xhpi.third > LogProb() ) { + cerr<<" "<<xhpi<<"\n"; + // If valid, add to queue... + axhpiQueue.enqueue(xhpi); + } + } + } + } + if ( axhpiQueue.getSize() > 0 ) { + //cerr<<axhpiQueue.getTop()<<"\n"; + axhpiQueue.dequeueTop(); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +void HMM<MH,MO,X,B>::writeFoll ( FILE* pf, int f, int b, const typename MO::RandVarType& o ) const { + const TrellNode<X,B>& tnxbPrev = aatnTrellis.get(f,b); + X xPrev = tnxbPrev.getId(); + fprintf(pf,"from f=%04d b=%04d: ",f,b); + cout<<tnxbPrev.getId()<<"\n"; + // For each possible transition... + typename MH::IterVal h; + cout<<"HMM<MH,MO,X,B>::writeFoll OUT OF ORDER\n"; + /* + for ( bool b=mh.setFirst(h,xPrev); b; b=mh.setNext(h,xPrev) ) { + X x; + LogProb lgprO = mo.getProb(o,mh.setTrellDat(x,h)); + LogProb lgprH = mh.getProb(h,xPrev); + LogProb lgprFull = tnxbPrev.getLogProb() * lgprH * lgprO; + cout<<" ==("<<tnxbPrev.getLogProb().toInt()<<"*"<<lgprH.toInt()<<"*"<<lgprO.toInt()<<"="<<lgprFull.toInt()<<")==> "<<h<<"\n"; + } + */ +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class MH, class MO, class X, class B> +int HMM<MH,MO,X,B>::getBeamUsed ( int f=-1 ) const { + if ( -1==f ) f=frameLast; + int ctr=0; + if ( 0<=f && f<=frameLast ) + for ( int i=0; i<BEAM_WIDTH; i++ ) + if(!(aatnTrellis.get(f,i).getLogProb() == LogProb())){ + ctr++; + } + return ctr; +} + +#endif //_NL_HMM_ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-hmmloop.h b/contrib/synlm/hhmm/rvtl/include/nl-hmmloop.h new file mode 100644 index 000000000..c476b4271 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-hmmloop.h @@ -0,0 +1,397 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_HMMLOOP_ +#define _NL_HMMLOOP_ +#include <list> +#include <string> +#include <boost/thread/thread.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/bind.hpp> +#include "nl-prob.h" +#include "nl-safeids.h" +#include "nl-beam.h" + +typedef int Frame; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// NullBackDat - default empty back-pointer data; can replace with word or sem relation +// +//////////////////////////////////////////////////////////////////////////////// + +template <class Y> +class NullBackDat { + static const string sDummy; + char dummy_data_member_to_avoid_compile_warning; + public: + NullBackDat () { dummy_data_member_to_avoid_compile_warning=0; } + NullBackDat (const Y& y) { dummy_data_member_to_avoid_compile_warning=0; } + void write (FILE*) const { } + string getString() const { return sDummy; } + friend ostream& operator<< ( ostream& os, const NullBackDat& nb ) { return os; } +}; +template <class Y> +const string NullBackDat<Y>::sDummy ( "" ); + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Index - pointer to source in previous beam heap +// +//////////////////////////////////////////////////////////////////////////////// + +class Index : public Id<int> { + public: + Index ( ) { } + Index (int i) {set(i);} + Index& operator++ ( ) {set(toInt()+1); return *this;} +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// TrellNode - node in viterbi trellis +// +//////////////////////////////////////////////////////////////////////////////// + +template <class S, class B> +class TrellNode { + private: + + // Data members... + Index indSource; + B backptrData; + S sId; + LogProb lgprMax; + + public: + + // Constructor / destructor methods... + TrellNode ( ) { } + TrellNode ( const Index& indS, const S& sI, const B& bDat, LogProb lgpr) + { indSource=indS; sId=sI; lgprMax=lgpr; backptrData=bDat; /* fo = -1; */ } + + // Specification methods... + const Index& setSource ( ) const { return indSource; } + const B& setBackData( ) const { return backptrData; } + const S& setId ( ) const { return sId; } + LogProb& setScore ( ) { return lgprMax; } + + // Extraction methods... + bool operator== ( const TrellNode<S,B>& tnsb ) const { return(sId==tnsb.sId); } +// size_t getHashKey ( ) const { return sId.getHashKey(); } + const Index& getSource ( ) const { return indSource; } + const B& getBackData( ) const { return backptrData; } + const S& getId ( ) const { return sId; } + LogProb getLogProb ( ) const { return lgprMax; } + LogProb getScore ( ) const { return lgprMax; } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const TrellNode& tn ) { return os<<tn.indSource<<","<<tn.backptrData<<","<<tn.sId<<","<<tn.lgprMax; } +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// HMMLoop +// +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S=typename MY::RandVarType, class B=NullBackDat<typename MY::RandVarType> > +class HMMLoop { + private: + typedef std::pair<Index,B> IB; + // Data members... + MY modY; + MX modX; + SafeArray2D<Id<Frame>,Id<int>,TrellNode<S,B> > aatnTrellis; + const int BEAM_WIDTH, LOOP_LENGTH; + Frame frameLast; + int iNextNode; + public: + // Static member varaibles... + static bool OUTPUT_QUIET; + static bool OUTPUT_NOISY; + static bool OUTPUT_VERYNOISY; +// static int BEAM_WIDTH; + // Constructor / destructor methods... + HMMLoop ( int, const char*[], int, int, const S& ) ; + // Specification methods... +// void init ( int, int, const S& ) ; +// void init ( int, int, SafeArray1D<Id<int>,pair<S,LogProb> >* ); + const TrellNode<S,B>& update ( const typename MX::RandVarType& ) ; + const TrellNode<S,B>& getTrellNode ( Frame t, Index i ) { return aatnTrellis.get(t%LOOP_LENGTH,i); } + TrellNode<S,B>& setTrellNode ( Frame t, Index i ) { return aatnTrellis.set(t%LOOP_LENGTH,i); } + + /* + void updateSerial ( const typename MX::RandVarType& ) ; + void updatePara ( const typename MX::RandVarType& ) ; + void each ( const typename MX::RandVarType&, Beam<LogProb,S,IB>&, SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> >& ) ; + // Extraction methods... + const TrellNode<S,B>& getTrellNode ( int i ) const { return aatnTrellis.get(frameLast,i); } + int getBeamUsed ( int ) const ; + // Input / output methods... + void writeMLS ( FILE* ) const ; + void writeMLS ( FILE*, const S& ) const ; + double getCurrSum(int) const; + void writeCurr ( FILE*, int ) const ; + void writeCurrSum ( FILE*, int ) const ; + void gatherElementsInBeam( SafeArray1D<Id<int>,pair<S,LogProb> >* result, int f ) const; + void writeCurrEntropy ( FILE*, int ) const; + //void writeCurrDepths ( FILE*, int ) const; + void writeFoll ( FILE*, int, int, const typename MX::RandVarType& ) const ; + void writeFollRanked ( FILE*, int, int, const typename MX::RandVarType&, bool ) const ; + std::list<string> getMLS() const; + std::list<TrellNode<S,B> > getMLSnodes() const; + std::list<string> getMLS(const S&) const; + std::list<TrellNode<S,B> > getMLSnodes(const S&) const; + */ +}; +template <class MY, class MX, class S, class B> bool HMMLoop<MY,MX,S,B>::OUTPUT_QUIET = false; +template <class MY, class MX, class S, class B> bool HMMLoop<MY,MX,S,B>::OUTPUT_NOISY = false; +template <class MY, class MX, class S, class B> bool HMMLoop<MY,MX,S,B>::OUTPUT_VERYNOISY = false; +//template <class MY, class MX, class S, class B> int HMMLoop<MY,MX,S,B>::BEAM_WIDTH = 1; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +template <class MY, class MX, class S, class B> +HMMLoop<MY,MX,S,B>::HMMLoop ( int nArgs, const char* apsArgs[], int w, int l, const S& sInit ) : BEAM_WIDTH(w), LOOP_LENGTH(l) { + + // For each model file in command line arguments... + for ( int iArg=1; iArg<nArgs; iArg++ ) { + + // Try to open model file... + FILE* pf = fopen(apsArgs[iArg],"r"); + // Complain if can't open model file... + if ( NULL == pf ) { + cout<<"ERROR: can't open file '"<<apsArgs[iArg]<<"'!\n"; + cout<<"Terminating process with failure code 1.\n"; + exit(1); + } + + // Initialize stream buffer and line number... + IStreamSource iss(pf); + int linenum=0; + + cout<<"Reading file '"<<apsArgs[iArg]<<"'...\n"; + + // For each line of input... + for ( IStream is(iss),is1; IStream()!=is; is=is1,iss.compress() ) { + + // Increment line number... + linenum++; + // Count off every 100K lines... + if (linenum%100000==0) cout<<" Reading line "<<linenum<<"...\n"; + + // Try to read each line into each model... + String s; + if ( (is1=(is>>"#">>s>>"\n")) == IStream() && + (is1=(is>>modY>> "\n")) == IStream() && + (is1=(is>>modX>> "\n")) == IStream() && + (is1=(is>>s >> "\n")) != IStream() ) + // Complain if bad format... + cout<<" ERROR in '"<<apsArgs[iArg]<<"', line "<<linenum<<": can't process '"<<s<<"'!\n"; + } + cout<<"Done reading file '"<<apsArgs[iArg]<<"'.\n"; + fclose(pf); + } + cout<<"Done reading all model files.\n"; + //modY.dump(cout,"Y"); + //modX.dump(cout,"X"); + + // Alloc trellis... + aatnTrellis.init(LOOP_LENGTH,BEAM_WIDTH); + frameLast=LOOP_LENGTH; + // Set initial element at first time slice... + setTrellNode(frameLast,0) = TrellNode<S,B> ( Index(0), sInit, B(), 0 ) ; + + cout<<"Begin processing input...\n"; + IStreamSource iss(stdin); + typename MX::RandVarType x; + + // For each frame... + for ( IStream is(iss); is!=IStream(); iss.compress() ) { + +// // Show beam... +// cout<<"-----BEAM:t="<<frameLast-LOOP_LENGTH<<"-----\n"; +// for(int i=0;i<BEAM_WIDTH;i++) +// cout<<getTrellNode(frameLast,i)<<"\n"; +// cout<<"--------------\n"; + + // Read spectrum (as frame audio)... + is=is>>x; + +// // Show spectrum... +// cout<<frameLast-2*LOOP_LENGTH+1<<" "<<x<<"\n"; +// // Show spectrum with bin numbers... +// cout<<frameLast-2*LOOP_LENGTH+1; +// for(int i=0; i<NUM_FREQUENCIES; i++) +// cout<<((i==0)?' ':',')<<i<<":"<<x.get(i); +// cout<<"\n"; + + // Update trellis... + const TrellNode<S,B>& tn = update(x); + + // Show recognized hidden variable values... + cout<<frameLast-2*LOOP_LENGTH+1<<":'"<<tn<<"'\n"; + cout.flush(); + } + cout<<"Done processing input.\n"; +} + + +//////////////////////////////////////////////////////////////////////////////// + +template <class A, class B> +inline bool outRank ( const quad<A,B,LogProb,Id<int> >& a1, + const quad<A,B,LogProb,Id<int> >& a2 ) { return (a1.third>a2.third); } + +template <class MY, class MX, class S, class B> +const TrellNode<S,B>& HMMLoop<MY,MX,S,B>::update ( const typename MX::RandVarType& x ) { + + // Increment frame counter... + frameLast++; + + // Init beam for new frame... + Beam<LogProb,S,IB> btn(BEAM_WIDTH); + SafeArray1D<Id<int>,std::pair<std::pair<S,IB>,LogProb> > atnSorted (BEAM_WIDTH); + + typedef quad<int,typename MY::IterVal,LogProb,Id<int> > SHPI; + Heap < SHPI, outRank<int,typename MY::IterVal > > ashpiQueue; + SHPI shpi, shpiTop; + int aCtr; + + ashpiQueue.clear(); + //shpi.first = -1; + //shpi.second = YModel::IterVal(); + //shpi.third = 1.0; + shpi.first = 0; + shpi.third = getTrellNode(frameLast-1,shpi.first).getScore(); + shpi.third *= modY.setIterProb ( shpi.second, getTrellNode(frameLast-1,shpi.first).getId(), aCtr=-1 ); // , x, aCtr=-1 ); + //S s; modY.setTrellDat(s,shpi.second); + shpi.fourth = -1; + ////cerr<<"????? "<<shpi<<"\n"; + ashpiQueue.enqueue(shpi); + + bool bFull=false; + + // For each ranked value of transition destination... + for ( int iTrg=0; !bFull && ashpiQueue.getSize()>0; iTrg++ ) { + // Iterate A* (best-first) search until a complete path is at the top of the queue... + while ( ashpiQueue.getSize() > 0 && ashpiQueue.getTop().fourth < MY::IterVal::NUM_ITERS ) { + // Remove top... + shpiTop = ashpiQueue.dequeueTop(); + // Fork off (try to advance each elementary variable a)... + for ( int a=shpiTop.fourth.toInt(); a<=MY::IterVal::NUM_ITERS; a++ ) { + // Copy top into new queue element... + shpi = shpiTop; + // At variable position -1, advance beam element for transition source... + if ( a == -1 ) shpi.first++; + // Incorporate prob from transition source... + shpi.third = getTrellNode(frameLast-1,shpi.first).getScore(); + if ( shpi.third > LogProb() ) { + // Try to advance variable at position a and return probability (subsequent variables set to first, probability ignored)... + shpi.third *= modY.setIterProb ( shpi.second, getTrellNode(frameLast-1,shpi.first).getId(), aCtr=a ); // , x, aCtr=a ); + // At end of variables, incorporate observation probability... + if ( a == MY::IterVal::NUM_ITERS && shpi.fourth != MY::IterVal::NUM_ITERS ) + shpi.third *= modX.getProb ( x, S(shpi.second) ); + //// { S s; modY.setTrellDat(s,shpi.second); shpi.third *= modX.getProb(x,s); } + // Record variable position at which this element was forked off... + shpi.fourth = a; + //cerr<<" from partial: "<<shpiTop<<"\n to partial: "<<shpi<<"\n"; + if ( shpi.third > LogProb() ) { + ////if ( frameLast == 4 ) cerr<<" from partial: "<<shpiTop<<"\n to partial: "<<shpi<<"\n"; + // If valid, add to queue... + ashpiQueue.enqueue(shpi); + //cerr<<"--------------------\n"<<ashpiQueue; + } + } + } + // Remove top... + //cerr<<"/-----A-----\\\n"<<ashpiQueue<<"\\-----A-----/\n"; + //if ( ashpiQueue.getTop().fourth != MY::IterVal::NUM_ITERS ) ashpiQueue.dequeueTop(); + ////cerr<<"/-----B-----\\\n"<<ashpiQueue<<"\\-----B-----/\n"; + ////cerr<<ashpiQueue.getSize()<<" queue elems, "<<ashpiQueue.getTop()<<"\n"; + } + + ////cerr<<"-----*-----\n"<<ashpiQueue<<"-----*-----\n"; + ////cerr<<ashpiQueue.getSize()<<" queue elems **\n"; + + // Add best transition (top of queue)... + //modX.getProb(o,modY.setTrellDat(ashpiQueue.getTop().first,ashpiQueue.getTop().second)); + if ( ashpiQueue.getSize() > 0 ) { + S s ( ashpiQueue.getTop().second ); + ////S s; modY.setTrellDat(s,ashpiQueue.getTop().second); + bFull |= btn.tryAdd ( s, IB(ashpiQueue.getTop().first,B(ashpiQueue.getTop().second)), ashpiQueue.getTop().third ); + ////cerr<<ashpiQueue.getSize()<<" queue elems A "<<ashpiQueue.getTop()<<"\n"; + ////cerr<<"/-----A-----\\\n"<<ashpiQueue<<"\\-----A-----/\n"; + ashpiQueue.dequeueTop(); + ////cerr<<"/-----B-----\\\n"<<ashpiQueue<<"\\-----B-----/\n"; + ////cerr<<ashpiQueue.getSize()<<" queue elems B "<<ashpiQueue.getTop()<<"\n"; + //cerr<<"."; cerr.flush(); + } + } + + ////cerr<<"-----*-----\n"<<ashpiQueue<<"-----*-----\n"; + + btn.sort(atnSorted); + + // Copy sorted beam to trellis... + Index iOriginOfBest; + int j=0; + for(int i=0;i<BEAM_WIDTH;i++) { + const std::pair<std::pair<S,IB>,LogProb>* tn1 = &atnSorted.get(i); + Index iOrigin = tn1->first.second.first; + // Determine origin at beginning of loop... + for ( Frame t=frameLast-1; t>frameLast-LOOP_LENGTH+1; t-- ) + iOrigin = getTrellNode(t,iOrigin).getSource(); + if ( 0 == i ) iOriginOfBest = iOrigin; + // If new hypothesis has same origin, add to beam... + if ( iOriginOfBest == iOrigin ) { + setTrellNode(frameLast,j++)=TrellNode<S,B>(tn1->first.second.first, + tn1->first.first, + tn1->first.second.second, + tn1->second); + } + } + // Clear out rest of beam... + for ( ; j<BEAM_WIDTH; j++ ) + setTrellNode(frameLast,j) = TrellNode<S,B>(); + + ////modY.update(); + + return getTrellNode(frameLast-LOOP_LENGTH+1,iOriginOfBest); +} + + + + +#endif //_NL_HMMLOOP_ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-iomacros.h b/contrib/synlm/hhmm/rvtl/include/nl-iomacros.h new file mode 100644 index 000000000..ccb2eb095 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-iomacros.h @@ -0,0 +1,63 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef NL_IO_MACROS__ +#define NL_IO_MACROS__ + +//#include <sys/types.h> +//#include <sys/socket.h> +//#include <netinet/in.h> +//#include <netdb.h> + + +#define NUM(c) ((c>='0' && c<='9')) +#define ALPHANUM(c) ((c>='a' && c<='z') || (c>='A' && c<='Z') || (c>='0' && c<='9')) +#define SPACE(c) (c==' ') +#define WHITESPACE(c) (c==' ' || c=='\t' || c=='\n') + +#define CONSUME_OPT(f,c,b,l) c = (b) ? getc(f)+0*(l+=(c=='\n')?1:0) : c + +#define CONSUME_ONE(f,c,b,s,l) c = (b) ? getc(f)+0*(l+=(c=='\n')?1:0) : c+(0*fprintf(stderr,"\nERROR: %s in line %d (char=%c).\n\n",s,l,c)) +#define CONSUME_ONE_STDIN(c,b,s,l) c = (b) ? getchar()+0*(l+=(c=='\n')?1:0) : c+(0*fprintf(stderr,"\nERROR: %s in line %d.\n\n",s,l)) + +#define CONSUME_ALL(f,c,b,l) for ( ; b; c=getc(f)+0*(l+=(c=='\n')?1:0) ) +#define CONSUME_ALL_STDIN(c,b,l) for ( ; b; c=getchar()+0*(l+=(c=='\n')?1:0) ) + +#define CONSUME_STR(f,c,b,s,i,l) for ( i=0; (b) || false != (s[i++]='\0'); s[i++]=c, c=getc(f)+0*(l+=(c=='\n')?1:0) ) +#define CONSUME_STR_SAFE(f,c,b,s,i,m,l) for ( i=0; i<m-1&&((b)||false!=(s[i++]='\0')); s[i++]=c, c=getc(f)+0*(l+=(c=='\n')?1:0) ) +#define CONSUME_STR_STDIN(c,b,s,i,l) for ( i=0; (b) || false != (s[i++]='\0'); s[i++]=c, c=getchar()+0*(l+=(c=='\n')?1:0) ) + +#define CONSUME_INT(f,c,i,l) for ( i=0; (c>='0' && c<='9'); i=(i*10)+(c-'0'), c=getc(f)+0*(l+=(c=='\n')?1:0) ) +#define CONSUME_INT_STDIN(c,i,l) for ( i=0; (c>='0' && c<='9'); i=(i*10)+(c-'0'), c=getchar()+0*(l+=(c=='\n')?1:0) ) + +#define CONSUME_DEC(f,c,i,j,l) for ( j=1; (c>='0' && c<='9'); j*=10, i+=(c-'0')/j, c=getc(f)+0*(l+=(c=='\n')?1:0) ) + +#define CONSUME_HEX(f,c,i,l) for ( i=0; (c>='0' && c<='9') || (c>='a' && c<='f'); i=(i*16)+((c<'a')?c-'0':c+10-'a'), c=getc(f)+0*(l+=(c=='\n')?1:0) ) + +#define CONSUME_ALL_SOCKET(f,c,b,l) for ( char s[1]; b; c=((recv(f,&s[0],1,MSG_WAITALL)==1) ? s[0]+0*(l+=(c=='\n')?1:0) : '\0') ) +#define CONSUME_STR_SOCKET(f,c,b,s,i,l) for ( i=0; (b) || ('\0'!=(s[i++]='\0')); s[i++]=c, c=((recv(f,&s[i],1,MSG_WAITALL)==1) ? s[i]+0*(l+=(c=='\n')?1:0) : (s[i]='\0')) ) + +//#define CONSUME_ALL_STRING(f,c,b,l) for ( int ii=0; b && f[ii]!='\0'; c=f[ii]+0*(l+=(c=='\n')?1:0), ii++ ) +//#define CONSUME_STR_STRING(f,c,b,s,i,l) for ( i=0; (b && f[i]!='\0') || false != (s[i++]='\0'); s[i++]=c, c=f[i]+0*(l+=(c=='\n')?1:0) ) + +#endif //_NL_IO_MACROS__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-linsep.h b/contrib/synlm/hhmm/rvtl/include/nl-linsep.h new file mode 100644 index 000000000..ac3ef3312 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-linsep.h @@ -0,0 +1,301 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////////////////// + +template<class X> +class QuadConvolved : public X { + public: + typedef typename X::ElementType ElementType; + static const unsigned int SIZE = X::SIZE+1; + private: + ElementType eC; + public: + QuadConvolved<X> ( ) : eC(1.0) { } + ElementType& operator[] ( unsigned int i ) { assert(i<SIZE); return (i<SIZE-1) ? X::operator[](i) : eC; } + ElementType operator[] ( unsigned int i ) const { assert(i<SIZE); return (i<SIZE-1) ? X::operator[](i) : eC; } + ElementType& set ( unsigned int i ) { assert(i<SIZE); return (i<SIZE-1) ? X::set(i) : eC; } + ElementType get ( unsigned int i ) const { assert(i<SIZE); return (i<SIZE-1) ? X::get(i) : eC; } + friend pair<StringInput,QuadConvolved<X>*> operator>> ( StringInput si, QuadConvolved<X>& x ) { return pair<StringInput,QuadConvolved<X>*>(si,&x); } + friend StringInput operator>> ( pair<StringInput,QuadConvolved<X>*> si_x, const char* psD ) { + StringInput si = si_x.first>>static_cast<X&>(*si_x.second)>>psD; + si_x.second->set(SIZE-1)=0.0; + for(int i=0;i<SIZE-1;i++) si_x.second->set(SIZE-1)+=pow(si_x.second->get(i)/1000000.0,2.0); + return si; } + friend ostream& operator<< ( ostream& os, const QuadConvolved<X>& x ) { os<<x.get(0); for(int i=1;i<SIZE;i++)os<<","<<x.get(i); return os; } +}; + + +//////////////////////////////////////////////////////////////////////////////// + +template<class X> +class Shifted : public X { + public: + typedef typename X::ElementType ElementType; + static const unsigned int SIZE = X::SIZE+1; + private: + ElementType e0; + public: + Shifted<X> ( ) : e0(0.0) { } + ElementType& operator[] ( unsigned int i ) { assert(i<SIZE); return (i<SIZE-1) ? X::operator[](i) : e0; } + ElementType operator[] ( unsigned int i ) const { assert(i<SIZE); return (i<SIZE-1) ? X::operator[](i) : e0; } + ElementType& set ( unsigned int i ) { assert(i<SIZE); return (i<SIZE-1) ? X::set(i) : e0; } + ElementType get ( unsigned int i ) const { assert(i<SIZE); return (i<SIZE-1) ? X::get(i) : e0; } + friend pair<StringInput,Shifted<X>*> operator>> ( StringInput si, Shifted<X>& x ) { return pair<StringInput,Shifted<X>*>(si,&x); } + //friend StringInput operator>> ( pair<StringInput,Shifted<X>*> si_x, const char* psD ) { + // StringInput si=si_x.first; for(int i=0;i<SIZE;i++) si=si>>si_x.second->set(i)>>((i<SIZE-1)?",":psD); return si; } + friend StringInput operator>> ( pair<StringInput,Shifted<X>*> si_x, const char* psD ) { + StringInput si = si_x.first>>static_cast<X&>(*si_x.second)>>psD; si_x.second->set(SIZE-1)=1.0; return si; } + //friend ostream& operator<< ( ostream& os, const Shifted<X>& x ) { os<<x.get(0); for(uint i=1;i<SIZE;i++)os<<","<<x.get(i); return os; } + friend ostream& operator<< ( ostream& os, Shifted<X>& x ) { if(x.get(SIZE-1)!=1.0)for(uint i=1;i<SIZE;i++)x.set(i)/=x.get(SIZE-1); return os<<((X)x); } +}; + + +//////////////////////////////////////////////////////////////////////////////// + +template<class X> +class Vector : public X { + public: + typedef typename X::ElementType ElementType; + static const unsigned int SIZE = X::SIZE; + Vector ( ) { } + Vector ( const Vector<X>& v ) { for(uint i=0;i<SIZE;i++) X::set(i)=v.get(i); } + ElementType operator* ( const Vector<X>& v ) const { ElementType d=0.0; for(uint i=0;i<SIZE;i++) d += X::get(i)*v.get(i); return d; } // inner prod + Vector<X> operator+ ( const Vector<X>& v ) const { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i]=X::get(i)+v[i]; return vO; } // vector sum + Vector<X> operator- ( const Vector<X>& v ) const { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i]=X::get(i)-v[i]; return vO; } // vector sum + Vector<X> operator* ( ElementType d ) const { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = X::get(i)*d; return vO; } + Vector<X> operator/ ( ElementType d ) const { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = X::get(i)/d; return vO; } + Vector<X> operator+ ( ElementType d ) const { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = X::get(i)+d; return vO; } + Vector<X> operator- ( ElementType d ) const { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = X::get(i)-d; return vO; } + friend Vector<X> operator* ( ElementType d, const Vector<X>& v ) { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = d*v[i]; return vO; } + friend Vector<X> operator/ ( ElementType d, const Vector<X>& v ) { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = d/v[i]; return vO; } + friend Vector<X> operator+ ( ElementType d, const Vector<X>& v ) { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = d+v[i]; return vO; } + friend Vector<X> operator- ( ElementType d, const Vector<X>& v ) { Vector<X> vO; for(uint i=0;i<SIZE;i++) vO[i] = d-v[i]; return vO; } + Vector<X>& operator*= ( ElementType d ) { for(uint i=0;i<SIZE;i++) X::set(i)*=d; return *this; } + Vector<X>& operator/= ( ElementType d ) { for(uint i=0;i<SIZE;i++) X::set(i)/=d; return *this; } + Vector<X>& operator+= ( ElementType d ) { for(uint i=0;i<SIZE;i++) X::set(i)+=d; return *this; } + Vector<X>& operator-= ( ElementType d ) { for(uint i=0;i<SIZE;i++) X::set(i)-=d; return *this; } + Vector<X>& operator+= ( const Vector<X>& v ) { for(uint i=0;i<SIZE;i++) X::set(i)+=v.X::get(i); return *this; } + Vector<X>& operator-= ( const Vector<X>& v ) { for(uint i=0;i<SIZE;i++) X::set(i)-=v.X::get(i); return *this; } +}; + + +//////////////////////////////////////////////////////////////////////////////// + +//#define GD 1 + +template<class Q,class X,class P> +class LinSepModel { + public: + class CondVarType : public Vector<Shifted<X> > { public: CondVarType(){} CondVarType(const Vector<Shifted<X> >& v):Vector<Shifted<X> >(v){} }; + class InputTrainingExample : public Joint2DRV<Q,CondVarType> { +// public: +// friend pair<StringInput,InputTrainingExample*> operator>> ( StringInput si, InputTrainingExample& x ) { return pair<StringInput,InputTrainingExample*>(si,&x); } +// friend StringInput operator>> ( pair<StringInput,InputTrainingExample*> si_x, const char* psD ) { +// StringInput si = si_x.first>>static_cast<X&>(*si_x.second)>>psD; si_x.second->set(SIZE-1)=1.0; return si; } + }; + typedef P ProbType; + CondVarType w; // weight vector + + private: + + typename X::ElementType sigmoid ( double a ) { return (!isnan(1.0/(1.0+exp(-a)))) ? 1.0/(1.0+exp(-a)) : (a>0.0) ? 1.0 : 0.0; } + + public: + + template<class F> + void train ( const SubArray<SafePtr<const InputTrainingExample> > apyx, F& yfn, double iP, double iN ) { + const double SCALER = 1000.0; // 1000.0 works just as well + //cerr<<" ---"<<iN<<" "<<iP<<" "<<N<<"\n"; + ////cout<<" ---"<<iN<<" "<<iP<<" "<<N<<"\n"; + iN*=2.0/apyx.size(); iP*=2.0/apyx.size(); + cerr<<" ---"<<iN<<" "<<iP<<"\n"; + + #ifdef GD + #else + //typename X::ElementType lambda=1.0; // inverse variance of prior over total weight + typename X::ElementType beta; // hestenes steifel factor + //typename X::ElementType ddl; // double deriv (for newton step) + #endif + CondVarType gPrev; // previous gradient + CondVarType g; // current gradient + CondVarType u; // step direction + typename X::ElementType z=0.0; // step size + double errPrev=200000.0; + double err =100000.0; + Array<typename X::ElementType> wtx(apyx.size(),0.0); // weighted sum + + for ( int k=0; k<100 && err>0.0 && (errPrev-err>0.001||k<=3); k++ ) { + errPrev = err; + + ////cerr<<" ? "<<g<<"\n"; + g = CondVarType(); //-lambda*w + ////cerr<<" ??? "<<iP<<" "<<iN<<"\n"; + for ( unsigned int n=0; n<apyx.size(); n++ ) { + g += ( sigmoid ( -((yfn(apyx[n].getRef().first))?1.0:-1.0) * wtx[n] ) + * ((yfn(apyx[n].getRef().first))?1.0:-1.0) * apyx[n].getRef().second/SCALER + * ((yfn(apyx[n].getRef().first))?iN:iP) ); + } + //cerr<<" g="<<g<<"\n"; + + #ifdef GD + u = g; + #else + beta = (k<=1) ? 0.0 : (g*(g-gPrev)) / (u*(g-gPrev)); + //cerr<<" beta="<<beta<<"\n"; + u = gPrev - beta*u; + #endif + + #ifdef GD + #else + //ddl = 0.0; //(lambda*u*u); + //for ( unsigned int n=0; n<apyx.size(); n++ ) + // ddl += sigmoid(wtx[n]) * sigmoid(-wtx[n]) * pow(u*apyx[n].getRef().second,2); + //z = (g*u)/ddl; + #endif + + //if(apyx.size()==123) cerr<<" u="<<u<<"\n"; + + if (k<=0) { + //w=u; + errPrev=100000.0; + } else { + double zDiff=10.0; + z=0.0; + for ( int t=0; abs(zDiff)>.001 && t<1; t++ ) { + double a = 0.0; + double b = 0.0; + ////cerr<<" w = "<<w<<"\n"; + for ( unsigned int n=0; n<apyx.size(); n++ ) { + ////cout<<" !!: "<<apyx[n].getRef().second<<"\n"; + + // If only one z loop, use wtx... + double ywzux = ((yfn(apyx[n].getRef().first))?1.0:-1.0) * wtx[n]; + //double ywzux = ((yfn(apyx[n].getRef().first))?1.0:-1.0) * (w+z*u) * apyx[n].getRef().second/SCALER; + + //if(apyx.size()==123) cerr<<" w="<<w<<"\n"; + //if(apyx.size()==123) cerr<<" z="<<z<<"\n"; + //if(apyx.size()==123) cerr<<" u="<<u<<"\n"; + //if(apyx.size()==123) cerr<<" z*u="<<z*u<<"\n"; + //if(apyx.size()==123) cerr<<" (w+z*u)="<<(w+(z*u))<<"\n"; + //if(apyx.size()==123) cerr<<" x="<<(apyx[n].getRef().second/SCALER)<<"\n"; + a += sigmoid(-ywzux) * ((yfn(apyx[n].getRef().first))?1.0:-1.0) * u * apyx[n].getRef().second/SCALER * ((yfn(apyx[n].getRef().first))?iN:iP); + b += sigmoid(ywzux) * sigmoid(-ywzux) * pow(u*apyx[n].getRef().second/SCALER,2.0) * ((yfn(apyx[n].getRef().first))?iN:iP); + //if(apyx.size()==123) cerr<<" x_"<<n<<"="<<ywzux<<" a="<<a<<" b="<<b<<"\n"; + } + zDiff = (a==0.0) ? 0.0 : a/b; + cerr<<" (z="<<z<<")"; + z += zDiff; + cerr<<" a="<<a<<" b="<<b<<" zDiff="<<zDiff<<" ==> z="<<z<<"\n"; + } + w += z*u; + } + + //cerr<<" w="<<w<<"\n"; + for ( unsigned int n=0; n<apyx.size(); n++ ) + wtx[n] += z*u*apyx[n].getRef().second/SCALER; + gPrev = g; + + err = 0.0; + for ( unsigned int n=0; n<apyx.size(); n++ ) + err += log ( 1.0 + exp ( -((yfn(apyx[n].getRef().first))?1.0:-1.0) * w*apyx[n].getRef().second/SCALER ) ) * ((yfn(apyx[n].getRef().first))?iN:iP); + cerr<<" ? k="<<k<<" "<<err<<"\n"; + } + + //for ( unsigned int n=0; n<apyx.size(); n++ ) + // if ( yfn(apyx[n].getRef().first) ) cerr<<" "<<( ((yfn(apyx[n].getRef().first))?1.0:-1.0) * w*apyx[n].getRef().second )<<"\n"; + } + bool classify ( const CondVarType& x ) const { /*cerr<<"----------\n"<<w<<"\n *\n"<<x<<"\n =\n"<<(w*x)<<"\n";*/ return ( w*x > 0.0 ); } + friend ostream& operator<< ( ostream& os, const LinSepModel<Q,X,P>& m ) { return os<<m.w; } + friend pair<StringInput,LinSepModel<Q,X,P>*> operator>> ( StringInput si, LinSepModel<Q,X,P>& m ) { + return pair<StringInput,LinSepModel<Q,X,P>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,LinSepModel<Q,X,P>*> si_m, const char* psD ) { + return (si_m.first==NULL) ? StringInput(NULL) : si_m.first>>si_m.second->w>>psD; } +}; + + +//////////////////////////////////////////////////////////////////////////////// + +template<class Q,class X,class P> + class SimpleLinSepModel { + public: + class CondVarType : public Vector<X> { }; + class InputTrainingExample : public Joint2DRV<Q,CondVarType> { +// public: +// friend pair<StringInput,InputTrainingExample*> operator>> ( StringInput si, InputTrainingExample& x ) { return pair<StringInput,InputTrainingExample*>(si,&x); } +// friend StringInput operator>> ( pair<StringInput,InputTrainingExample*> si_x, const char* psD ) { +// StringInput si = si_x.first>>static_cast<X&>(*si_x.second)>>psD; si_x.second->set(SIZE-1)=1.0; return si; } + }; + typedef P ProbType; + CondVarType w; // weight vector + + private: + + typename X::ElementType sigmoid ( double a ) { return 1.0/(1.0+exp(-a)); } + + public: + + ////void train ( const SafePtr<const InputTrainingExample> apyx[], const int N, bool fn(const Q&), double iPnotused, double iNnotused ) { + template<class F> + void train ( const SubArray<SafePtr<const InputTrainingExample> > apyx, F& yfn, double iPnotused, double iNnotused ) { + double dIntercept = 0.0; + CondVarType vP,vN,vDelt,vMid; // pos/neg centroids, difference, midpoint + int iP=0,iN=0; // pos/neg counts + // For each example... + for ( unsigned int i=0; i<apyx.size(); i++ ) { + // If matches target, avg into pos centroid... + if ( yfn(apyx[i].getRef().first) ) { + iP++; + for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) vP[j]+=apyx[i].getRef().second[j]; + } + // If no match, avg into neg centroid... + else { + iN++; + for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) vN[j]+=apyx[i].getRef().second[j]; + } + } + //cerr<<"prob of "<<qTarget<<" at "<<branch<<" = "<<iP<<"/"<<N<<":"<<iN<<"/"<<N<<" = "<<double(iP)/double(N)<<"\n"; + // Turn sums into avgs... + if ( iP>0 ) for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) vP[j]/=double(iP); + if ( iN>0 ) for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) vN[j]/=double(iN); + + // Calc vec of delta between centroids... + for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) vDelt[j]=vP[j]-vN[j]; + // Calc vec of midpoint between centroids... + for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) vMid[j]=(vP[j]+vN[j])/2.0; + // Calc y-intercept of gradient (negative so test is > -1.0)... + for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) dIntercept-=vDelt[j]*vMid[j]; + // Calc gradient between centroids... + for ( unsigned int j=0; j<CondVarType::SIZE; j++ ) w[j]=vDelt[j]/dIntercept; + } + // NOTE: bool does not necessarily correspond to true=positive -- have to check by hand if you care (oblidtree doesn't care)... + bool classify ( const CondVarType& x ) const { /*cerr<<"----------\n"<<w<<"\n *\n"<<x<<"\n =\n"<<(w*x)<<"\n";*/ return ( w*x > -1.0 ); } + + friend ostream& operator<< ( ostream& os, const SimpleLinSepModel<Q,X,P>& m ) { return os<<m.w; } + + friend pair<StringInput,SimpleLinSepModel<Q,X,P>*> operator>> ( StringInput si, SimpleLinSepModel<Q,X,P>& m ) { + return pair<StringInput,SimpleLinSepModel<Q,X,P>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,SimpleLinSepModel<Q,X,P>*> si_m, const char* psD ) { + return (si_m.first==NULL) ? StringInput(NULL) : si_m.first>>si_m.second->w>>psD; } +}; diff --git a/contrib/synlm/hhmm/rvtl/include/nl-list.h b/contrib/synlm/hhmm/rvtl/include/nl-list.h new file mode 100644 index 000000000..65f07f8b5 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-list.h @@ -0,0 +1,481 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_LIST_ ////////////////////////////////////////////////////////////// +#define _NL_LIST_ ////////////////////////////////////////////////////////////// + +#include <stdlib.h> + +#define Listed(x) ListedObject<x> + +//////////////////////////////////////////////////////////////////////////////// +// +// Container macros +// +//////////////////////////////////////////////////////////////////////////////// + +// Standard loop... +#define foreach(p,c) for ( p=(c).getNext(NULL); p!=NULL; p=(c).getNext(p) ) + +// True unless proven false... +#define setifall(y,p,c,x) for ( p=(c).getNext(NULL), y=true; p!=NULL && y; y &= (x), p=(c).getNext(p) ) + +// False unless proven true... +#define setifexists(y,p,c,x) for ( p=(c).getNext(NULL), y=false; p!=NULL && !y; y |= (x), p=(c).getNext(p) ) + + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +class Ptr + { + private: + T* ptObj ; + public: + Ptr ( ) { ptObj=NULL; } + Ptr ( T* pt ) { ptObj=pt; } + Ptr ( T& t ) { ptObj=&t; } + Ptr ( const Ptr<T>& pt ) { ptObj=pt.ptObj; } + bool operator> ( const Ptr<T>& pt ) const { return(ptObj>pt.ptObj); } + bool operator< ( const Ptr<T>& pt ) const { return(ptObj<pt.ptObj); } + bool operator>= ( const Ptr<T>& pt ) const { return(ptObj>=pt.ptObj); } + bool operator<= ( const Ptr<T>& pt ) const { return(ptObj<=pt.ptObj); } + bool operator== ( const Ptr<T>& pt ) const { return(ptObj==pt.ptObj); } + bool operator!= ( const Ptr<T>& pt ) const { return(ptObj!=pt.ptObj); } + Ptr<T>& operator= ( const Ptr<T>& pt ) { ptObj=pt.ptObj; return *this; } + T& operator* ( ) const { return *ptObj; } + T* operator-> ( ) const { return ptObj; } + } ; + + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +class ListedObject ; + +template <class T> +class List { + private: + + ListedObject<T>* plotLast ; + + public: + + typedef ListedObject<T>* iterator; + typedef const ListedObject<T>* const_iterator; + + // Constructor and destructor methods... + List ( ) ; + List ( const T& ) ; + List ( const List<T>& ) ; + List ( const List<T>&, const List<T>& ) ; + ~List ( ) ; + + // Overloaded operators... + List<T>& operator= ( const List<T>& ) ; + List<T>& operator+= ( const List<T>& ) ; + bool operator== ( const List<T>& ) const ; + bool operator!= ( const List<T>& ) const ; + + // Specification methods... + void clear ( ) ; + T& insert ( Listed(T)* ) ; + void remove ( Listed(T)* ) ; + T& add ( ) ; + T& push ( ) ; + void pop ( ) ; + Listed(T)* setFirst ( ) ; + Listed(T)* setNext ( Listed(T)* ) ; + + // Extraction methods... + const_iterator begin ( ) const { return getNext(NULL); } + const_iterator end ( ) const { return NULL; } + iterator& operator++ ( ) { *this=getNext(*this); return *this; } + int getCard ( ) const ; + Listed(T)* getFirst ( ) const ; + Listed(T)* getSecond( ) const ; + Listed(T)* getLast ( ) const ; + Listed(T)* getNext ( const Listed(T)* ) const ; + bool contains ( const T& ) const ; + bool isEmpty ( ) const ; + +/* // Input / output methods... */ +/* friend IStream operator>> ( pair<IStream,List<T>*> is_x, const char* psDlm ) { */ +/* IStream& is = is_x.first; */ +/* List<T>& x = *is_x.second; */ +/* if (IStream()!=is) */ +/* is = pair<IStream,T*>(is,&x.add())>>psDlm; */ +/* return is; */ +/* } */ +} ; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +class ListedObject : public T + { + friend class List<T> ; + + private: + + ListedObject<T>* plotNext ; + + public: + + const ListedObject<T>* next ( ) const { return plotNext; } + ListedObject ( ) { plotNext = NULL; } + ListedObject ( const ListedObject<T>& lot ) + { T::operator=(lot); } + ListedObject<T>& operator= ( const ListedObject<T>& lot ) + { T::operator=(lot); return(*this); } + operator T() { return *this; } + } ; + + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +List<T>::List ( ) + { + plotLast = NULL ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +List<T>::List ( const T& t ) + { + plotLast = NULL ; + + add() = t ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +List<T>::List ( const List<T>& lt ) + { + ListedObject<T>* pt ; + + plotLast = NULL ; + + foreach ( pt, lt ) + add() = *pt ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +List<T>::List ( const List<T>& lt1, const List<T>& lt2 ) + { + ListedObject<T>* pt ; + + plotLast = NULL ; + + foreach ( pt, lt1 ) + add() = *pt ; + foreach ( pt, lt2 ) + add() = *pt ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +List<T>::~List ( ) + { + clear(); + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +void List<T>::clear ( ) + { + ListedObject<T>* plot ; + ListedObject<T>* plot2 ; + if ( NULL != (plot = plotLast) ) + do { plot2 = plot->plotNext ; + ////fprintf(stderr,"list::destr %x\n",plot); + delete plot ; + } while ( plotLast != (plot = plot2) ) ; + plotLast = NULL ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +List<T>& List<T>::operator= ( const List<T>& lt ) + { + Listed(T)* pt ; + + this->~List ( ) ; + plotLast = NULL ; + + foreach ( pt, lt ) + add() = *pt ; + + return *this ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +List<T>& List<T>::operator+= ( const List<T>& lt ) + { + Listed(T)* pt ; + + foreach ( pt, lt ) + add() = *pt ; + + return *this ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +bool List<T>::operator== ( const List<T>& lt ) const + { + Listed(T)* pt1 ; + Listed(T)* pt2 ; + + for ( pt1 = getNext(NULL), pt2 = lt.getNext(NULL); + pt1 != NULL && pt2 != NULL ; + pt1 = getNext(pt1), pt2 = lt.getNext(pt2) ) + if ( !(*pt1 == *pt2) ) return false ; + + return ( pt1 == NULL && pt2 == NULL ) ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +bool List<T>::operator!= ( const List<T>& lt ) const + { + return !(*this == lt) ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +T& List<T>::insert ( Listed(T)* plotPrev ) + { + ListedObject<T>* plot = new ListedObject<T> ; + ////fprintf(stderr,"list::const %x\n",plot); + + if ( NULL != plotPrev ) + { + plot->plotNext = plotPrev->plotNext ; + plotPrev->plotNext = plot ; + if ( plotLast == plotPrev ) + plotLast = plot ; + } + else if ( NULL != plotLast ) + { + plot->plotNext = plotLast->plotNext ; + plotLast->plotNext = plot ; + } + else + { + plot->plotNext = plot ; + plotLast = plot ; + } + + return *plot ; + } + +//////////////////////////////////////////////////////////////////////////////// + +/* DON'T KNOW WHY THIS DOESN'T WORK +template <class T> +void List<T>::remove ( Listed(T)* plot ) + { + assert ( plot ); + assert ( plotLast ); + // If only one element... + if ( plot->plotNext == plot ) + { + assert ( plotLast == plot ); + plotLast = NULL; +fprintf(stderr,"list::delete1 %x\n",plot); + delete plot; + } + // If more than one element... + else + { + if ( plotLast == plot->plotNext ) plotLast = plot; + Listed(T)* plotTemp = plot->plotNext; + *plot = *(plot->plotNext); +fprintf(stderr,"list::delete2 %x\n",plotTemp); + delete plotTemp; + } + } +*/ + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +T& List<T>::add ( ) + { + ListedObject<T>* plot = new ListedObject<T> ; + ////fprintf(stderr,"list::add %x\n",plot); + + if ( NULL != plotLast ) + { + plot->plotNext = plotLast->plotNext ; + plotLast->plotNext = plot ; + plotLast = plot ; + } + else + { + plot->plotNext = plot ; + plotLast = plot ; + } + + return *plot ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +T& List<T>::push ( ) + { + ListedObject<T>* plot = new ListedObject<T> ; + ////fprintf(stderr,"list::push %x\n",plot); + + if ( NULL != plotLast ) + { + plot->plotNext = plotLast->plotNext ; + plotLast->plotNext = plot ; + } + else + { + plot->plotNext = plot ; + plotLast = plot ; + } + + return *plot ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +void List<T>::pop ( ) + { + ListedObject<T>* plot = plotLast->plotNext ; + + if ( plot->plotNext == plot ) + plotLast = NULL ; + else + plotLast->plotNext = plot->plotNext ; + + ////fprintf(stderr,"list::pop %x\n",plot); + delete plot ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +int List<T>::getCard ( ) const + { + Listed(T)* pt ; + int i = 0 ; + + foreach ( pt, *this ) + i++ ; + + return i ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +ListedObject<T>* List<T>::setFirst ( ) + { + return ( NULL != plotLast ) ? plotLast->plotNext : NULL ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +ListedObject<T>* List<T>::setNext ( ListedObject<T>* plot ) + { + return ( NULL == plot && NULL != plotLast ) ? plotLast->plotNext : + ( plot != plotLast ) ? plot->plotNext : NULL ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +ListedObject<T>* List<T>::getFirst ( ) const + { + return ( NULL != plotLast ) ? plotLast->plotNext : NULL ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +ListedObject<T>* List<T>::getSecond ( ) const + { + return getNext(getFirst()) ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +ListedObject<T>* List<T>::getLast ( ) const + { + return ( NULL != plotLast ) ? plotLast : NULL ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +ListedObject<T>* List<T>::getNext ( const ListedObject<T>* plot ) const + { + return ( NULL == plot && NULL != plotLast ) ? plotLast->plotNext : + ( plot != plotLast ) ? plot->plotNext : NULL ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +bool List<T>::contains ( const T& t ) const + { + ListedObject<T>* pt ; + + foreach ( pt, *this ) + if ( t == *pt ) return true ; + + return false ; + } + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +bool List<T>::isEmpty ( ) const + { + return ( NULL == plotLast ) ; + } + +#endif //_NL_LIST_ ///////////////////////////////////////////////////////////// + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-matrix.h b/contrib/synlm/hhmm/rvtl/include/nl-matrix.h new file mode 100644 index 000000000..c8c18f8e1 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-matrix.h @@ -0,0 +1,177 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +template<class I,class J,class P> +class SparseMatrix : public SimpleHash<I,SimpleHash<J,P> > { + public: + + typedef SimpleHash<I,SimpleHash<J,P> > Parent; + + //// Matrix / vector operator methods... + friend SparseMatrix<I,J,P> operator* ( const SparseMatrix<I,J,P>& a, const SparseMatrix<I,J,P>& b ) { + SparseMatrix mOut; + for ( typename Parent::const_iterator iit=a.begin(); iit!=a.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator kit=a.get(i).begin(); kit!=a.get(i).end(); kit++ ) { + I k = kit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=b.get(k).begin(); jit!=b.get(k).end(); jit++ ) { + I j = jit->first; + if ( a.get(i).get(k)!=0 && b.get(k).get(j)!=0 ) + mOut.set(i).set(j) += a.get(i).get(k) * b.get(k).get(j); + } + } + } + return mOut; + } + friend SparseMatrix<I,J,P> operator+ ( const SparseMatrix<I,J,P>& a, const SparseMatrix<I,J,P>& b ) { + SparseMatrix mOut; + for ( typename Parent::const_iterator iit=a.begin(); iit!=a.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=a.get(i).begin(); jit!=a.get(i).end(); jit++ ) { + I j = jit->first; + mOut.set(i).set(j) = a.get(i).get(j); + } + } + for ( typename Parent::const_iterator iit=b.begin(); iit!=b.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=b.get(i).begin(); jit!=b.get(i).end(); jit++ ) { + I j = jit->first; + mOut.set(i).set(j) += b.get(i).get(j); + } + } + return mOut; + } + friend SparseMatrix<I,J,P> operator- ( const SparseMatrix<I,J,P>& a, const SparseMatrix<I,J,P>& b ) { + SparseMatrix mOut; + for ( typename Parent::const_iterator iit=a.begin(); iit!=a.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=a.get(i).begin(); jit!=a.get(i).end(); jit++ ) { + I j = jit->first; + mOut.set(i).set(j) = a.get(i).get(j); + } + } + for ( typename Parent::const_iterator iit=b.begin(); iit!=b.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=b.get(i).begin(); jit!=b.get(i).end(); jit++ ) { + I j = jit->first; + mOut.set(i).set(j) -= b.get(i).get(j); + } + } + return mOut; + } + // Matrix + scalar operators... + friend SparseMatrix<I,J,P> operator+ ( const SparseMatrix<I,J,P>& a, const P& p ) { + SparseMatrix mOut; + for ( typename Parent::const_iterator iit=a.begin(); iit!=a.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=a.get(i).begin(); jit!=a.get(i).end(); jit++ ) { + I j = jit->first; + mOut.set(i).set(j) = a.get(i).get(j) + p; + } + } + return mOut; + } + friend SparseMatrix<I,J,P> operator- ( const SparseMatrix<I,J,P>& a, const P& p ) { + SparseMatrix mOut; + for ( typename Parent::const_iterator iit=a.begin(); iit!=a.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=a.get(i).begin(); jit!=a.get(i).end(); jit++ ) { + I j = jit->first; + mOut.set(i).set(j) = a.get(i).get(j) - p; + } + } + return mOut; + } + // Diagonal matrix of vector... + friend SparseMatrix<I,J,P> diag ( const SparseMatrix<I,J,P>& a ) { + SparseMatrix mOut; + for ( typename Parent::const_iterator iit=a.begin(); iit!=a.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=a.get(i).begin(); jit!=a.get(i).end(); jit++ ) { + I j = jit->first; + assert(j==0); // must be vector + mOut.set(i).set(i) += a.get(i).get(j); + } + } + return mOut; + } + // Scalar inf-norm (max) of matrix / vector... + P infnorm ( ) const { + P pOut = 0; // sparse matrix assumes some values are zero, so this is default infnorm. + for ( typename Parent::const_iterator iit=Parent::begin(); iit!=Parent::end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=Parent::get(i).begin(); jit!=Parent::get(i).end(); jit++ ) { + I j = jit->first; + if ( Parent::get(i).get(j) > pOut ) pOut = Parent::get(i).get(j); + } + } + return pOut; + } + + // Scalar one-norm (sum) of matrix / vector... + P onenorm ( ) const { + P sum=0; + for ( typename Parent::const_iterator iit=Parent::begin(); iit!=Parent::end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=Parent::get(i).begin(); jit!=Parent::get(i).end(); jit++ ) { + I j = jit->first; + sum += Parent::get(i).get(j); + } + } + return sum; + } + + //// Input / output methods... + friend pair<StringInput,SparseMatrix<I,J,P>*> operator>> ( StringInput si, SparseMatrix<I,J,P>& m ) { + return pair<StringInput,SparseMatrix<I,J,P>*>(si,&m); + } + friend StringInput operator>> ( pair<StringInput,SparseMatrix<I,J,P>*> si_m, const char* psD ) { + if (StringInput(NULL)==si_m.first) return si_m.first; + StringInput si; I i,j; P p; + si=si_m.first>>i>>" : ">>j>>" = ">>p>>psD; + if ( si!=NULL ) si_m.second->set(i).set(j) = p; + return si; + } + friend ostream& operator<< ( ostream& os, const SparseMatrix<I,J,P>& m ) { + int ctr=0; + for ( typename Parent::const_iterator iit=m.begin(); iit!=m.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=m.get(i).begin(); jit!=m.get(i).end(); jit++ ) { + I j = jit->first; + os<<((0==ctr++)?"":",")<<i<<":"<<j<<"="<<m.get(i).get(j); + } + } + return os; + } + friend String& operator<< ( String& str, const SparseMatrix<I,J,P>& m ) { + int ctr=0; + for ( typename Parent::const_iterator iit=m.begin(); iit!=m.end(); iit++ ) { + I i = iit->first; + for ( typename SimpleHash<J,P>::const_iterator jit=m.get(i).begin(); jit!=m.get(i).end(); jit++ ) { + I j = jit->first; + str<<((0==ctr++)?"":",")<<i<<j; + } + } + return str; + } +}; diff --git a/contrib/synlm/hhmm/rvtl/include/nl-mixture.h b/contrib/synlm/hhmm/rvtl/include/nl-mixture.h new file mode 100644 index 000000000..2da5aacb2 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-mixture.h @@ -0,0 +1,268 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_MIXTURE__ +#define _NL_MIXTURE__ + +#include "nl-randvar.h" +#include "nl-list.h" + +//////////////////////////////////////////////////////////////////////////////// +// +// Mixture Model +// +//////////////////////////////////////////////////////////////////////////////// + +template <template <class MY> class M,class Y,class C> +class Mixture2DModel : public Generic1DModel<Y,Prob> { + private: + // Static data members... + static bool bModeled; + // Data members... + CPT1DModel<C,Prob> mC; + SimpleHash<C,M<Y> > mY_giv_C; + public: + Mixture2DModel ( ) : mC() { } + Mixture2DModel ( const string& s ) : mC(s) { } + // Specification methods... + CPT1DModel<C,Prob>& setMixingModel ( ) { return mC; } + Prob& setComponentProb ( const C& c ) { return mC.setProb(c); } + M<Y>& setComponentModel ( const C& c ) { return mY_giv_C.set(c); } + // Extraction methods... + Prob getComponentProb ( const C& c ) { return mC.getProb(c); } + const M<Y>& getComponentModel ( const C& c ) { return mY_giv_C.get(c); } + Prob getProb ( const Y& y ) const { Prob pr=0.0; + C c; for ( bool b=c.setFirst(); b; b=c.setNext() ) + pr += ( mC.getProb(c) * mY_giv_C.get(c).getProb(y) ); + return pr; } + // Input / output methods... + bool readFields ( char* as[], int numF ) { return ( mC.readFields(as,numF) || mY_giv_C.set(C(as[1])).readFields(as+1,numF-1) ); } + void writeFields ( FILE* pf, string sPref ) const { mC.writeFields(pf,sPref); C c; for ( bool b=c.setFirst(); b; b=c.setNext() ) + mY_giv_C.get(c).writeFields(pf,sPref+" "+c.getString()); } +}; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +template <template <class MY> class M,class Y,class X,class C> +class Mixture3DModel : public Generic2DModel<Y,X,Prob> { + private: + // Static data members... + static bool bModeled; + // Data members... + CPT2DModel<C,X,Prob> mC_giv_X; + SimpleHash<Joint2DRV<X,C>,M<Y> > mY_giv_X_C; + public: + Mixture3DModel ( ) : mC_giv_X() { } + Mixture3DModel ( const string& s ) : mC_giv_X(s) { } + // Specification methods... + Prob& setComponentProb ( const X& x, const C& c ) { return mC_giv_X.setProb(c,x); } + M<Y>& setComponentModel ( const X& x, const C& c ) { return mY_giv_X_C.set(Joint2DRV<X,C>(x,c)); } + // Extraction methods... + Prob getProb ( const Y& y, const X& x ) const { Prob pr=0.0; + C c; for ( bool b=c.setFirst(); b; b=c.setNext() ) + pr += ( mC_giv_X.getProb(c,x) * mY_giv_X_C.get(Joint2DRV<X,C>(x,c)).getProb(y) ); + return pr; } + // Input / output methods... + bool readFields ( char* as[], int numF ) { return ( mC_giv_X.readFields(as,numF) || mY_giv_X_C.set(Joint2DRV<X,C>(X(as[1]),C(as[2]))).readFields(as+2,numF-2) ); } + void writeFields ( FILE* pf, string sPref ) const { mC_giv_X.writeFields(pf,sPref); + X x; for ( bool bx=x.setFirst(); bx; bx=x.setNext() ) { + C c; for ( bool b=c.setFirst(); b; b=c.setNext() ) { + mY_giv_X_C.get(Joint2DRV<X,C>(x,c)).writeFields(pf,sPref+" "+c.getString()); } } } +}; + +//////////////////////////////////////////////////////////////////////////////// +// +// Trainable Mixture Model +// +//////////////////////////////////////////////////////////////////////////////// + +template <template <class MY> class M,class Y,class C> +class TrainableMixture2DModel : public Mixture2DModel<M,Y,C> { +// private: +// LogPDFVal logpdfPrevDataAvg; + public: +// // Constructor / destructor methods... +// TrainableMixture2DModel ( ) : logpdfPrevDataAvg(log(0.0)) { } + // Specification methods... + void updateFields ( const List<Joint2DRV<Y,Prob> >&, const PDFVal, bool& ); + void train ( List<Joint2DRV<Y,Prob> >&, const int, const PDFVal ); +}; + +//////////////////////////////////////// +template <template <class MY> class M,class Y,class C> +void TrainableMixture2DModel<M,Y,C>::updateFields ( const List<Joint2DRV<Y,Prob> >& lyp, const PDFVal WEIGHT_LIMIT, bool& bShouldStop ) { + LogPDFVal logpdfData = 0.0; + CPT1DModel<C,Prob> mprPseudoEmpC; // pseudo-empirical prob marginal + SimpleHash<C,List<pair<const Y*,Prob> > > alypPseudoEmpY_giv_C; // pseudo-empirical prob conditional + + //// E step: create pseudo-annotated (expectation) corpus by re-distributing empirical probabilities over C, according to model parameters... + + C c; + for ( bool b=c.setFirst(); b; b=c.setNext() ) + setComponentModel(c).precomputeVarianceTerms(); + + // For each Y... + for ( const ListedObject<Joint2DRV<Y,Prob> >* pyp=lyp.getFirst(); pyp; pyp=lyp.getNext(pyp) ) { + const Y& y = pyp->getSub1(); // data value + const Prob& prEmpY = pyp->getSub2(); // empirical prob + + CPT1DModel<C,PDFVal> mpdfEstCY; // model-estimated pdf + PDFVal pdfEstY = 0.0; // model-estimated pdf marginal + + // For each C, compute model-estimated pdfs... + C c; for ( bool b=c.setFirst(); b; b=c.setNext() ) { + PDFVal pdfEstCY = getComponentProb(c) * getComponentModel(c).getProb(y); +// fprintf(stderr," ???%g %d %g\n",(double)pdfEstCY,(int)c,getComponentModel(c).getProb(y)); ////y.write(stderr); fprintf(stderr,"\n"); + mpdfEstCY.setProb(c) = pdfEstCY; + pdfEstY += pdfEstCY; // marginalize out C (marginal estimated pdf of each y) + } + + // Update total... + logpdfData += log(pdfEstY); // product of each datum (joint estimated pdf of training corpus) + //fprintf(stderr,"..%g %g\n",pdfEstY,logpdfData); + + // For each C, compute psuedo-empirical probs (model-estimated * empirical)... + for ( bool b=c.setFirst(); b; b=c.setNext() ) { + Prob prEstC_giv_Y = mpdfEstCY.getProb(c) / pdfEstY; // so these are normalized distribs over C + Prob prPseudoEmpCY = prEmpY * prEstC_giv_Y; + alypPseudoEmpY_giv_C.set(c).add() = pair<const Y*,Prob> ( &pyp->getSub1(), prPseudoEmpCY ); + mprPseudoEmpC.setProb(c) += prPseudoEmpCY; // marginalize out Y + } + } + // Renormalize sub-lists... + for ( bool b=c.setFirst(); b; b=c.setNext() ) + for ( ListedObject<pair<const Y*,Prob> >* pyp=alypPseudoEmpY_giv_C.set(c).setFirst(); pyp; pyp=alypPseudoEmpY_giv_C.set(c).setNext(pyp) ) + pyp->second /= mprPseudoEmpC.getProb(c); + + //// M step: update model parameters using MLE (relative frequency estimation) on pseudo-annotated corpus... + + for ( bool b=c.setFirst(); b; b=c.setNext() ) { + // If any change exceeds thresh, continue... + if ( bShouldStop && ( getComponentProb(c)-mprPseudoEmpC.getProb(c) > WEIGHT_LIMIT || + getComponentProb(c)-mprPseudoEmpC.getProb(c) < -WEIGHT_LIMIT ) ) bShouldStop = false; + // Update probs... + setComponentProb(c) = mprPseudoEmpC.getProb(c); + setComponentModel(c).setFields ( alypPseudoEmpY_giv_C.get(c) ); + //////getComponentModel(c).writeFields(stderr,c.getString()); + } + + //////Mixture2DModel<M,Y,C>::writeFields(stderr,""); + fprintf(stderr," log pdf data: %g\n",(double)logpdfData); +// // Quit if total prob doesn't improve enough... +// fprintf(stderr," log avg pdf: %g - %g < %g ORLY? %d\n", +// (double)logpdfDataAvg,(double)logpdfPrevDataAvg,(double)WEIGHT_LIMIT, +// ((logpdfDataAvg - logpdfPrevDataAvg) < WEIGHT_LIMIT) ); +// bShouldStop = ( (logpdfDataAvg - logpdfPrevDataAvg) < WEIGHT_LIMIT ); +// logpdfPrevDataAvg = logpdfDataAvg; +} + +//////////////////////////////////////// +template <template <class MY> class M,class Y,class C> +void TrainableMixture2DModel<M,Y,C>::train ( List<Joint2DRV<Y,Prob> >& lyp, const int EPOCH_LIMIT, const PDFVal WEIGHT_LIMIT ) { + + // Normalize model... + Mixture2DModel<M,Y,C>::setMixingModel().normalize(); // OUGHT NOT TO BE NECESSARY!!! + + // Normalize input... + PDFVal pdfNorm = 0.0; + for ( const ListedObject<Joint2DRV<Y,Prob> >* pyp=lyp.getFirst(); pyp; pyp=lyp.getNext(pyp) ) pdfNorm += pyp->getSub2(); + for ( ListedObject<Joint2DRV<Y,Prob> >* pyp=lyp.setFirst(); pyp; pyp=lyp.setNext(pyp) ) pyp->setSub2() /= pdfNorm; + + // Iterate over each epoch... + bool bShouldStop = true; + for ( int epoch=0; epoch<EPOCH_LIMIT; epoch++ ) { + fprintf(stderr,"---------- EPOCH %d/%d ----------\n",epoch,EPOCH_LIMIT); + clock_t start = clock(); + updateFields ( lyp, WEIGHT_LIMIT, bShouldStop ) ; + clock_t end = clock(); + fprintf ( stderr, "epoch time: %d ticks or %f seconds\n", end-start, ((double)end - start)/CLOCKS_PER_SEC ) ; + if (bShouldStop) break; + } +} + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +template <template <class MY> class M,class Y,class X,class C> +class TrainableMixture3DModel : public Generic2DModel<Y,X,C> { + private: + string sId; + SimpleHash<X,List<Joint2DRV<Y,Prob> > > alyp; + SimpleHash<X,TrainableMixture2DModel<M,Y,C> > am; + public: + // Constructor / destructor methods... + TrainableMixture3DModel ( string s ) : sId(s) { } + // Specification methods... + void train ( const int, const PDFVal ); + void train ( const List<Joint3DRV<X,Y,Prob> >&, const int, const PDFVal ); + // Extraction methods... + Prob getProb ( const Y& y, const X& x ) const { return am.get(x).getProb(y); } + // Input / output methods... + bool readData ( char* as[], int ); + bool readFields ( char* as[], int numF ) { return ( am.set(X(as[1])).readFields(as+1,numF-1) ); } + void writeFields ( FILE*, string sPref ); +}; + +//////////////////////////////////////// +template <template <class MY> class M,class Y,class X,class C> +void TrainableMixture3DModel<M,Y,X,C>::train ( const int EPOCH_LIMIT, const PDFVal WEIGHT_LIMIT ) { + // Update each subphone from list... + int ctr = 0; + X x; for ( bool b=x.setFirst(); b; b=x.setNext() ) { + if (OUTPUT_NOISY) fprintf(stderr,"==================== SUBMODEL %s (number %d) %d data pts ====================\n", + x.getString().c_str(),ctr++,alyp.get(x).getCard()); + am.set(x).train ( alyp.set(x), EPOCH_LIMIT, WEIGHT_LIMIT ); + } +} + +//////////////////////////////////////// +template <template <class MY> class M,class Y,class X,class C> +void TrainableMixture3DModel<M,Y,X,C>::train ( const List<Joint3DRV<X,Y,Prob> >& lxyp, const int EPOCH_LIMIT, const PDFVal WEIGHT_LIMIT ) { + // Chop list into phone-specific sub-lists... + ListedObject<Joint3DRV<X,Y,Prob> >* pxyp; + for ( pxyp=lxyp.getFirst(); pxyp; pxyp=lxyp.getNext(pxyp) ) + alyp.set(pxyp->getSub1()).add() = Joint2DRV<Y,Prob> ( pxyp->getSub2(), pxyp->getSub3() ); + // Update each subphone from list... + train(EPOCH_LIMIT,WEIGHT_LIMIT); +} + +//////////////////////////////////////// +template <template <class MY> class M,class Y,class X,class C> +bool TrainableMixture3DModel<M,Y,X,C>::readData ( char* as[], int numFields ) { + if ( /*as[0]!=sId+"dat" ||*/ numFields!=3 ) return false; + alyp.set(X(as[1])).add() = Joint2DRV<Y,Prob>(Y(as[2]),Prob(1.0)); + return true; +} + +//////////////////////////////////////// +template <template <class MY> class M,class Y,class X,class C> +void TrainableMixture3DModel<M,Y,X,C>::writeFields ( FILE* pf, string sPref ) { + X x; for ( bool b=x.setFirst(); b; b=x.setNext() ) { + am.get(x).writeFields(pf,sPref+" "+x.getString()); + } +} + + +#endif /*_NL_MIXTURE__*/ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-modelfile.h b/contrib/synlm/hhmm/rvtl/include/nl-modelfile.h new file mode 100644 index 000000000..dc6bec487 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-modelfile.h @@ -0,0 +1,126 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_MODEL_FILE__ +#define _NL_MODEL_FILE__ + +#include "nl-string.h" +#include "nl-iomacros.h" + +#include <netinet/in.h> + +static bool OUTPUT_QUIET = false; + +/////////////////////////////////////////////////////////////////////////////// + +void processModelFilePtr ( FILE* pf, bool rF(Array<char*>&) ) { + int i=0; int numFields=0; int c=' '; int line=1; + CONSUME_ALL(pf,c,WHITESPACE(c),line); // Get to first record + while ( c!=EOF ) { // For each record + if ( c=='#' ) CONSUME_ALL(pf, c, c!='\n' && c!='\0', line ) ; // If comment, consume + else { // If no comment, + Array<char*> aps(100); + String psBuff(1000); + CONSUME_STR ( pf, c, (c!='\n' && c!='\0'), psBuff, i, line ); + + char* psT=NULL; + for(int i=0;true;i++) { + char* z = strtok_r ( (0==i)?psBuff.c_array():NULL, " :=", &psT ); + if (!z) break; + aps[i]=z; + } + + if ( !rF(aps) ) // Try to process fields, else complain + fprintf( stderr, "\nERROR: %d %d-arg %s in line %d\n\n", numFields, aps.size(), aps[0], line); + } + CONSUME_ALL(pf,c,WHITESPACE(c),line); // Consume whitespace + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void processModelFile ( const char* ps, bool rF(Array<char*>&) ) { + FILE* pf; + if(!OUTPUT_QUIET) fprintf ( stderr, "Reading model file %s...\n", ps ) ; + if ( NULL == (pf=fopen(ps,"r")) ) // Complain if file not found + fprintf ( stderr, "\nERROR: file %s could not be opened.\n\n", ps ) ; + processModelFilePtr ( pf, rF ); + fclose(pf); + if(!OUTPUT_QUIET) fprintf ( stderr, "Model file %s loaded.\n", ps ) ; +} + +/////////////////////////////////////////////////////////////////////////////// + +void processModelSocket ( const int tSockfd, int& c, bool rF(Array<char*>&) ) { + int i=0; int numFields=0; int line=1; + CONSUME_ALL_SOCKET(tSockfd,c,WHITESPACE(c),line); // Get to first record + while ( c!='\0' && c!='\5' ) { // For each record + if ( c=='#' ) CONSUME_ALL_SOCKET(tSockfd, c, (c!='\n' && c!='\0' && c!='\5'), line ) ; // If comment, consume + else { // If no comment, + Array<char*> aps(100); + String psBuff(1000); + CONSUME_STR_SOCKET ( tSockfd, c, (c!='\n' && c!='\0' && c!='\5'), psBuff, i, line ); + ////cerr<<"|"<<psBuff.c_array()<<"|"<<endl; + + char* psT=NULL; + for(int i=0;true;i++) { + char* z = strtok_r ( (0==i)?psBuff.c_array():NULL, " :=", &psT ); + if (!z) break; + aps[i]=z; + } + + if ( !rF(aps) ) // Try to process fields, else complain + fprintf( stderr, "\nERROR: %d-arg %s in line %d\n\n", numFields, aps[0], line); + } + CONSUME_ALL_SOCKET(tSockfd,c,WHITESPACE(c),line); // Consume whitespace + } +} + +void processModelSocket ( const int tSockfd, bool rF(Array<char*>&) ) { + int c=' '; + processModelSocket ( tSockfd, c, rF ); +} + +/////////////////////////////////////////////////////////////////////////////// + +/* +void processModelString ( String& sBuff, bool rF(Array<char*>&) ) { + if ('#'!=sBuff[0]) { + Array<char*> aps(100); + char* psT=NULL; + for(int i=0;true;i++) { + char* z = strtok_r ( (0==i)?sBuff.c_array():NULL, " :=", &psT ); + if (!z) break; + aps[i]=z; + } + if ( !rF(aps) ) // Try to process fields, else complain + fprintf( stderr, "\nERROR: %d-arg %s in line %d\n\n", numFields, aps[0], line); + } +} +*/ + +/////////////////////////////////////////////////////////////////////////////// + +#endif //_NL_MODEL_FILE__ + + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-oblidtree.h b/contrib/synlm/hhmm/rvtl/include/nl-oblidtree.h new file mode 100644 index 000000000..24c82e313 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-oblidtree.h @@ -0,0 +1,239 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#include "nl-linsep.h" +/* +class ConvolvedComplexMelVec { + private: + ComplexMelVec vMain; + double dConvo; + public: + void set ( const ComplexMelVec& v ) { vMain=v; for(int i=0;i<ComplexMelVec::SIZE;i++)dConvo+=vMain.get(i)*vMain.get(i); } + static const int SIZE = ComplexMelVec::SIZE+1; + double& set ( int i ) { return (i<ComplexMelVec::SIZE)?vMain.set(i):dConvo; } + double get ( int i ) const { return (i<ComplexMelVec::SIZE)?vMain.get(i):dConvo; } + double& operator[] ( int i ) { return set(i); } + double operator[] ( int i ) const { return get(i); } + double operator* ( const ConvolvedComplexMelVec& v ) const { double d; for(int i=0;i<SIZE;i++)d+=get(i)*v.get(i); return d; } // inner product + friend pair<StringInput,ConvolvedComplexMelVec*> operator>> ( StringInput str, ConvolvedComplexMelVec& v ) { + return pair<StringInput,ConvolvedComplexMelVec*>(str,&v); } + friend StringInput operator>> ( pair<StringInput,ConvolvedComplexMelVec*> delimbuff, const char* ps ) { + StringInput str = delimbuff.first>>delimbuff.second->vMain>>",">>delimbuff.second->dConvo>>ps; + for(int i=0;i<ComplexMelVec::SIZE;i++)delimbuff.second->dConvo+=delimbuff.second->vMain.get(i)*delimbuff.second->vMain.get(i); + return str; } + friend ostream& operator<< ( ostream& os, const ConvolvedComplexMelVec& v ) { return os<<v.vMain<<","<<v.dConvo; } +}; + +class TrainingExample : public DelimitedJoint2DRV<psX,Psymb,psBar,ConvolvedComplexMelVec,psX> { }; +*/ + + +//////////////////////////////////////////////////////////////////////////////// + +class binuint { + private: + uint b; + public: + // Constructor / destructor methods... + binuint ( ) : b(0) { } + binuint ( uint i ) : b(i) { } + // Specification methods... + binuint& operator+= ( binuint i ) { b+=i.b; return *this; } + binuint& operator-= ( binuint i ) { b-=i.b; return *this; } + binuint& operator*= ( binuint i ) { b*=i.b; return *this; } + binuint& operator/= ( binuint i ) { b/=i.b; return *this; } + binuint& operator%= ( binuint i ) { b%=i.b; return *this; } + binuint& operator++ ( ) { b++; return *this; } + // Extractor methods... + binuint operator+ ( binuint i ) const { return (b+i.b); } + binuint operator- ( binuint i ) const { return (b-i.b); } + binuint operator* ( binuint i ) const { return (b*i.b); } + binuint operator/ ( binuint i ) const { return (b/i.b); } + binuint operator% ( binuint i ) const { return (b%i.b); } + bool operator== ( binuint i ) const { return (b==i.b); } + bool operator!= ( binuint i ) const { return (b!=i.b); } + bool operator< ( binuint i ) const { return (b<i.b); } + bool operator> ( binuint i ) const { return (b>i.b); } + bool operator<= ( binuint i ) const { return (b<=i.b); } + bool operator>= ( binuint i ) const { return (b>=i.b); } + size_t getHashKey ( ) const { return b; } + // Input / output methods... + friend StringInput operator>> ( StringInput si, binuint& i ) { + if(si==NULL) return si; + i.b=0; + for ( char c=si[0]; '0'<=c && c<='1'; ++si,c=si[0]) + { i.b=i.b*2+c-'0'; } + return si; } + friend ostream& operator<< ( ostream& os, binuint i ) { for(int e=uint(log2(i.b));e>=0;e--)os <<((i.b>>e)%2); return os; } + friend String& operator<< ( String& str, binuint i ) { for(int e=uint(log2(i.b));e>=0;e--)str<<((i.b>>e)%2); return str; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template<class YF,class M> +class ObliqueDTreeModel { + + private: + + ////=M////typedef SimpleLinSepModel<Q,X> SepModel; + //typedef int TreeAddr; + map<binuint,M> hag; + map<binuint,typename M::ProbType> hap; + + public: + + typedef typename M::InputTrainingExample InputTrainingExample; + + void train ( SubArray<SafePtr<const InputTrainingExample> >, const YF&, binuint=1 ); + + typename M::ProbType getProb ( binuint y, const typename M::CondVarType& ) const; + + friend pair<StringInput,ObliqueDTreeModel<YF,M>*> operator>> ( StringInput si, ObliqueDTreeModel<YF,M>& m ) { + return pair<StringInput,ObliqueDTreeModel<YF,M>*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,ObliqueDTreeModel<YF,M>*> si_m, const char* psD ) { + if (si_m.first==NULL) return si_m.first; + binuint branch=0; + StringInput si=si_m.first>>branch>>" "; + //cerr<<"?^?^? "<<si<<"\n"; + if ( si==NULL ) return si_m.first; + //fprintf(stderr," .. %x\n",delimbuff.second); + if ( '='==si[0] ) return si>>"= ">>si_m.second->hag[branch]>>psD; + else return si>>": 1 = ">>si_m.second->hap[branch]>>psD; + } + + friend ostream& operator<< ( ostream& os, const ObliqueDTreeModel<YF,M>& m ) { + return os<<pair<const String*,const ObliqueDTreeModel<YF,M>*>(&String(""),&m); } + friend pair< const String*, const ObliqueDTreeModel<YF,M>*> operator* ( const String& s, const ObliqueDTreeModel<YF,M>& m ) { + return pair<const String*,const ObliqueDTreeModel<YF,M>*>(&s,&m); } + friend ostream& operator<< ( ostream& os, const pair<const String*,const ObliqueDTreeModel<YF,M>*> s_m ) { + // For each tree branch... + binuint b=1; do { + // If non-terminal branch, write gradient... + if ( s_m.second->hag.find(b)!=s_m.second->hag.end() && s_m.second->hap.find(b)==s_m.second->hap.end() ) + os<<*s_m.first<<b<<" = " <<s_m.second->hag.find(b)->second<<"\n"; + // If terminal branch, write distrib... + else if ( s_m.second->hap.find(b)!=s_m.second->hap.end() ) + os<<*s_m.first<<b<<" : 1 = "<<s_m.second->hap.find(b)->second<<"\n"; + + // If non-terminal branch, recurse... + if ( s_m.second->hag.find(b)!=s_m.second->hag.end() ) b*=2; + // If terminal branch, pop odds then advance... + else { while(b%2==1)b/=2; ++b; } + } while ( b!=1 ); + //os<<"done.\n"; + return os; + } +}; + +//////////////////////////////////////// +template<class YF,class M> + void ObliqueDTreeModel<YF,M>::train ( SubArray<SafePtr<const InputTrainingExample> > ape, const YF& yfn, binuint branch ) { + + cerr<<" starting oblidtree branch "<<branch<<": "<<ape.size()<<" examples...\n"; + + //// find separator + + double iP=0.0,iN=0.0; + for ( unsigned int n=0; n<ape.size(); n++ ) { if (yfn(ape[n].getRef().first)) iP++; else iN++; } + //cerr<<" iN="<<iN<<" iP="<<iP<<" in this subtree\n"; + + //if ( iP>10 && iN>10 ) + hag[branch].train(ape,yfn,iP,iN); + + //// divide data + + unsigned int iSplitPt = ape.size(); + // For each example... + for ( unsigned int i=0; i<iSplitPt; i++ ) { + // Determine side of gradient... + bool yEst = hag[branch].classify(ape[i].getRef().second); + ////double yEst=0.0; + ////for ( int j=0; j<O::SIZE; j++ ) yEst += hag[branch][j]*ape[i].getRef().second[j]; + // Swap greater-than-one examples to end of array... + if ( yEst ) { ////( yEst >= 1.0 ) { + iSplitPt--; + SafePtr<const InputTrainingExample> peTemp = ape[i]; + ape[i] = ape[iSplitPt]; + ape[iSplitPt] = peTemp; + i--; + } + } + cerr<<" oblidtree branch "<<branch<<": split 0 < "<<iSplitPt<<" < "<<ape.size()<<" parent%pos="<<(iP/ape.size()); + if ( iSplitPt>0 && iSplitPt<ape.size() ) { + double pPos=0.0; + for ( unsigned int n=0; n<iSplitPt; n++ ) if(yfn(ape[n].getRef().first))pPos++; + cerr<<",left%pos=("<<pPos<<"/"<<iSplitPt; + pPos/=iSplitPt; + cerr<<")="<<pPos; + pPos=0.0; + for ( unsigned int n=iSplitPt; n<ape.size(); n++ ) if(yfn(ape[n].getRef().first))pPos++; + cerr<<",right%pos=("<<pPos<<"/"<<(ape.size()-iSplitPt); + pPos/=(ape.size()-iSplitPt); + cerr<<")="<<pPos; + } + cerr<<"\n"; + // Recurse on divided array: greater-than-ones at right... + if ( iP > 0 && iN > 0 && iSplitPt > 100 && ape.size()-iSplitPt > 100 ) { + train ( SubArray<SafePtr<const InputTrainingExample> >(ape,0,iSplitPt), yfn, branch*2 ); + train ( SubArray<SafePtr<const InputTrainingExample> >(ape,iSplitPt), yfn, branch*2+1 ); + } + else { + // Add prob... + hap[branch]=(double(iP)+.1)/(double(ape.size())+.1); + ////cout<<"terminal prob of "<<qTarget<<" at "<<branch<<" = "<<iP<<"/"<<ape.size()<<" = "<<hap[branch]<<"\n"; + } +} + +//////////////////////////////////////// +template<class YF,class M> +typename M::ProbType ObliqueDTreeModel<YF,M>::getProb ( binuint y, const typename M::CondVarType& o ) const { + + //const tr1::unordered_map<binuint,P>& hap = hhap.get(q); + //const double SLOPE = 20; + //// For each tree branch... + //Prob prWidth=1.0; Prob prQ=0.0; int b=1; Prob prL=0.0; Prob prR=0.0; map<binuint,double> widthcache; widthcache[b]=prWidth; + //do { + // // If nonterminal branch, push left... + // if ( hag.find(b)!=hag.end() ) { + // // Calc L/R probs given dist of o from separator g... + // prR=1.0/(1.0+exp(-(o*hag.find(b)->second-1.0)*SLOPE)); prL=1.0-prR; b*=2; widthcache[b]=prWidth*=prL; + // } + // // If terminal branch, pop all rights, then advance left to right... + // else if ( hap.find(b)!=hap.end() ) { + // prQ += prWidth * hap.find(b)->second.toDouble(); + // while(b%2==1){ b/=2; } + // prWidth=(b<=1)?0.0:widthcache[b/2]; b++; prR=(b<=1)?0.0:1.0/(1.0+exp(-(o*hag.find(b/2)->second-1.0)*SLOPE)); widthcache[b]=prWidth*=prR; + // } + //} while ( b!=1 ); + //return P(prQ); + + ////cout<<" :: "<<q<<" "<<(X)o<<"\n"; + binuint b=1; while ( hag.find(b)!=hag.end() ) { + ////cout<<" "<<b<<" "<<hag.find(b)->second<<"\n"; + //b=(o * hag.find(b)->second < 1.0) ? b*2 : b*2+1; + b = (hag.find(b)->second.classify(o)==false) ? b*2 : b*2+1; + } + //cout<<" SepTreeObsModel: "<<q<<" "<<b<<" "<<hhap.get(q).find(b)->second<<"\n"; + return (hap.find(b)==hap.end()) ? typename M::ProbType() : (y==0) ? typename M::ProbType(1.0-hap.find(b)->second.toDouble()) : hap.find(b)->second; +} + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-prob.h b/contrib/synlm/hhmm/rvtl/include/nl-prob.h new file mode 100644 index 000000000..76cf2fb57 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-prob.h @@ -0,0 +1,136 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_PROB__ +#define _NL_PROB__ + +#include "nl-safeids.h" +#include <math.h> +#include <limits.h> + +//////////////////////////////////////////////////////////////////////////////// + +typedef double PDFVal; +typedef double LogPDFVal; + +//////////////////////////////////////////////////////////////////////////////// + +class Prob { + private: + double gVal; + + public: + Prob ( ) { gVal = 0.0; } + Prob (double d) { gVal = d; } + Prob (const char* ps) { gVal = atof(ps); } + + operator double() const { return gVal; } + double toDouble() const { return gVal; } + Prob& operator+= ( const Prob p ) { gVal += p.gVal; return *this; } + Prob& operator-= ( const Prob p ) { gVal -= p.gVal; return *this; } + Prob& operator*= ( const Prob p ) { gVal *= p.gVal; return *this; } + Prob& operator/= ( const Prob p ) { gVal /= p.gVal; return *this; } + + friend ostream& operator<< ( ostream& os, const Prob& pr ) { return os<<pr.toDouble(); } + friend String& operator<< ( String& str, const Prob& pr ) { return str<<pr.toDouble(); } + friend pair<StringInput,Prob*> operator>> ( StringInput si, Prob& n ) { return pair<StringInput,Prob*>(si,&n); } + friend StringInput operator>> ( pair<StringInput,Prob*> si_n, const char* psDlm ) { + double d=0.0; StringInput si=si_n.first>>d>>psDlm; *si_n.second=Prob(d); return si; } +}; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// LogProb -- encapsulate min probability in sum operations +// +//////////////////////////////////////////////////////////////////////////////// + +//#define MIN_LOG_PROB (1-MAXINT) +#define MIN_LOG_PROB INT_MIN + +class LogProb : public Id<int> { + public: + // Constructor / destructor methods... + LogProb ( ) { set(MIN_LOG_PROB); } + LogProb ( int i ) { set(i); } + LogProb ( double d ) { set(int(100.0*log(d))); } + LogProb ( Prob d ) { set(int(100.0*log(d))); } + // Specification methods... + LogProb& operator+= ( const LogProb i ) { assert(false); return *this; } + LogProb& operator*= ( const LogProb i ) + { if((toInt() == MIN_LOG_PROB) || (i.toInt() == MIN_LOG_PROB)){ + set(MIN_LOG_PROB); + }else{ + // Correct underflow if result is greater than either addend... + int k=toInt()+i.toInt(); set((toInt()<=0 && i.toInt()<=0 && (k>i.toInt() || k>toInt())) ? MIN_LOG_PROB : k); + } + return *this; + } + LogProb& operator-= ( const LogProb i ) { assert(false); return *this; } + LogProb& operator/= ( const LogProb i ) + { if((toInt() == MIN_LOG_PROB) || (i.toInt() == MIN_LOG_PROB)){ + set(MIN_LOG_PROB); + }else{ + int k=toInt()-i.toInt(); set(k); + } + return *this; + } + + // Extraction methods... + bool operator==( const LogProb i ) const { return(i.toInt()==toInt()); } + bool operator!=( const LogProb i ) const { return(i.toInt()!=toInt()); } + LogProb operator+ ( const LogProb i ) const { assert(false); return *this; } // no support for addition in log mode! + LogProb operator- ( const LogProb i ) const { assert(false); return *this; } // no support for addition in log mode! + LogProb operator* ( const LogProb i ) const { + int k; + if((toInt() == MIN_LOG_PROB) || (i.toInt() == MIN_LOG_PROB)){ + k = MIN_LOG_PROB; + }else{ + k=toInt()+i.toInt(); + // Correct underflow if result is greater than either addend... + k = (toInt()<0 && i.toInt()<0 && (k>i.toInt() || k>toInt())) ? MIN_LOG_PROB : k; + } + return LogProb(k); + } + LogProb operator/ ( const LogProb i ) const { + int k; + if((toInt() == MIN_LOG_PROB) || (i.toInt() == MIN_LOG_PROB)){ + k = MIN_LOG_PROB; + }else{ + k = toInt()-i.toInt(); + // // Correct underflow if result is greater than either addend... + // k = (toInt()<0 && -i.toInt()<0 && (k>-i.toInt() || k>toInt())) ? MIN_LOG_PROB : k; + } + return LogProb(k); + } + Prob toProb() const { return exp(double(toInt())/100.0); } + double toDouble() const { return toProb().toDouble(); } +// operator double() const { return exp(toInt()/100.0); } + friend ostream& operator<< ( ostream& os, const LogProb& lp ) { return os<<lp.toInt(); } + friend String& operator<< ( String& str, const LogProb& lp ) { return str<<lp.toInt(); } + friend pair<StringInput,LogProb*> operator>> ( StringInput si, LogProb& n ) { return pair<StringInput,LogProb*>(si,&n); } + friend StringInput operator>> ( pair<StringInput,LogProb*> si_n, const char* psDlm ) { + double d=0.0; StringInput si=si_n.first>>d>>psDlm; *si_n.second=LogProb(d); return si; } +}; + +#endif /* _NL_PROB__ */ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-probmodel.h b/contrib/synlm/hhmm/rvtl/include/nl-probmodel.h new file mode 100644 index 000000000..2dcff7b30 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-probmodel.h @@ -0,0 +1,422 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_PROBMODEL__ +#define _NL_PROBMODEL__ + +#include "nl-prob.h" + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// XDModel +// +//////////////////////////////////////////////////////////////////////////////// + +template<class Y,class P> +class Generic1DModel { + public: + typedef Y RVType; + typedef P ProbType; + bool setFirst ( const Y& ) const { assert(false); return false; } // if you need it, you have to write it + bool setNext ( const Y& ) const { assert(false); return false; } // if you need it, you have to write it + P getProb ( const Y& ) const { assert(false); return 0.0; } // if you need it, you have to write it +}; + +//////////////////////////////////////////////////////////// + +template<class Y,class X1,class P> +class Generic2DModel { + public: + typedef Y RVType; + typedef X1 Dep1Type; + typedef P ProbType; + bool setFirst ( const Y& ) const { assert(false); return false; } // if you need it, you have to write it + bool setNext ( const Y& ) const { assert(false); return false; } // if you need it, you have to write it + bool setFirst ( const Y&, const X1& x1 ) const { assert(false); return false; } // if you need it, you have to write it + bool setNext ( const Y&, const X1& x1 ) const { assert(false); return false; } // if you need it, you have to write it + P getProb ( const Y&, const X1& x1 ) const { assert(false); return 0.0; } // if you need it, you have to write it +}; + +//////////////////////////////////////////////////////////// + +template<class Y,class X1,class X2,class P> +class Generic3DModel { + public: + typedef Y RVType; + typedef X1 Dep1Type; + typedef X2 Dep2Type; + typedef P ProbType; + bool setFirst ( const Y& ) const { assert(false); return false; } // if you need it, you have to write it + bool setNext ( const Y& ) const { assert(false); return false; } // if you need it, you have to write it + bool setFirst ( const Y&, const X1& x1, const X2& x2 ) const { assert(false); return false; } // if you need it, you have to write it + bool setNext ( const Y&, const X1& x1, const X2& x2 ) const { assert(false); return false; } // if you need it, you have to write it + P getProb ( const Y&, const X1& x1, const X2& x2 ) const { assert(false); return 0.0; } // if you need it, you have to write it +}; + +//////////////////////////////////////////////////////////// + +template<class Y,class X1,class X2,class X3,class P> +class Generic4DModel { + public: + typedef Y RVType; + typedef X1 Dep1Type; + typedef X2 Dep2Type; + typedef X3 Dep3Type; + typedef P ProbType; + bool setFirst ( const Y& ) const { assert(false); return false; } + bool setNext ( const Y& ) const { assert(false); return false; } + bool setFirst ( const Y&, const X1& x1, const X2& x2, const X3& x3 ) const { assert(false); return false; } + bool setNext ( const Y&, const X1& x1, const X2& x2, const X3& x3 ) const { assert(false); return false; } + P getProb ( const Y&, const X1& x1, const X2& x2, const X3& x3 ) const { assert(false); return 0.0; } +}; + +//////////////////////////////////////////////////////////// + +template<class Y,class X1,class X2,class X3,class X4,class P> +class Generic5DModel { + public: + typedef Y RVType; + typedef X1 Dep1Type; + typedef X2 Dep2Type; + typedef X3 Dep3Type; + typedef X4 Dep4Type; + typedef P ProbType; + bool setFirst ( const Y& ) const { assert(false); return false; } + bool setNext ( const Y& ) const { assert(false); return false; } + bool setFirst ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { assert(false); return false; } + bool setNext ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { assert(false); return false; } + P getProb ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { assert(false); return 0.0; } +}; + +//////////////////////////////////////////////////////////// + +template<class Y,class X1,class X2,class X3,class X4,class X5,class P> +class Generic6DModel { + public: + typedef Y RVType; + typedef X1 Dep1Type; + typedef X2 Dep2Type; + typedef X3 Dep3Type; + typedef X4 Dep4Type; + typedef X5 Dep5Type; + typedef P ProbType; + bool setFirst ( const Y& ) const { assert(false); return false; } + bool setNext ( const Y& ) const { assert(false); return false; } + bool setFirst ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { assert(false); return false; } + bool setNext ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { assert(false); return false; } + P getProb ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { assert(false); return 0.0; } +}; + +//////////////////////////////////////////////////////////// + +template<class Y,class X1,class X2,class X3,class X4,class X5,class X6,class P> +class Generic7DModel { + public: + typedef Y RVType; + typedef X1 Dep1Type; + typedef X2 Dep2Type; + typedef X3 Dep3Type; + typedef X4 Dep4Type; + typedef X5 Dep5Type; + typedef X6 Dep6Type; + typedef P ProbType; + bool setFirst ( const Y& ) const { assert(false); return false; } + bool setNext ( const Y& ) const { assert(false); return false; } + bool setFirst ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X5& x6 ) const { assert(false); return false; } + bool setNext ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X5& x6 ) const { assert(false); return false; } + P getProb ( const Y&, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5, const X5& x6 ) const { assert(false); return 0.0; } +}; + + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// ModeledXDRV +// +//////////////////////////////////////////////////////////////////////////////// + +template<class M, M& m> +class Modeled1DRV : public M::RVType { + public: + // Constructor / destructor methods... + Modeled1DRV ( ) { } + //Modeled1DRV ( int i ) { M::RVType::operator=(i); } + Modeled1DRV ( string& s ) { M::RVType::operator=(s); } + // Static specification methods... + static M& setModel ( ) { return m; } + // Static extraction methods... + static const M& getModel ( ) { return m; } + // Extraction methods... + typename M::ProbType getProb ( ) const { return m.getProb(*this); } +}; + +//////////////////////////////////////////////////////////// + +template<class M, M& m> +class Modeled2DRV : public M::RVType { + public: + typedef M Model; + // Constructor / destructor methods... + Modeled2DRV ( ) { } + // Modeled2DRV ( int i ) { M::RVType::operator=(i); } + Modeled2DRV ( const Modeled2DRV<M,m>& mrv ) : M::RVType(mrv) { } + Modeled2DRV ( const typename M::RVType& x ) { M::RVType::operator=(x); } + Modeled2DRV ( string& s ) { M::RVType::operator=(s); } + // Static specification methods... + static M& setModel ( ) { return m; } + // Static extraction methods... + static const M& getModel ( ) { return m; } + // Specification methods... + bool setFirst ( ) { return M::RVType::setFirst(); } + bool setNext ( ) { return M::RVType::setNext(); } + bool setFirst ( const typename M::Dep1Type& x1 ) { return m.setFirst(*this,x1); } + bool setFirstConsistent ( const typename M::Dep1Type& x1 ) { return m.setFirstConsistent(*this,x1); } + bool setNext ( const typename M::Dep1Type& x1 ) { return m.setNext (*this,x1); } + bool setSample ( const typename M::Dep1Type& x1 ) { return m.setSample(*this,x1); } + // Extraction methods... + typename M::ProbType getProb ( const typename M::Dep1Type& x1 ) const { return m.getProb(*this,x1); } +}; + +//////////////////////////////////////////////////////////// + +template<class M, M& m> +class DetermModeled2DRV : public Modeled2DRV<M,m> { + public: + DetermModeled2DRV ( const typename M::Dep1Type& x1 ) { m.setFirst(*this,x1); assert(LogProb(1.0)==m.getProb(*this,x1)); } +}; + +//////////////////////////////////////////////////////////// + +template<class M, M& m> +class Modeled3DRV : public M::RVType { + public: + // Constructor / destructor methods... + Modeled3DRV ( ) { } + //Modeled3DRV ( int i ) { M::RVType::operator=(i); } + Modeled3DRV ( string& s ) { M::RVType::operator=(s); } + // Static specification methods... + static M& setModel ( ) { return m; } + // Static extraction methods... + static const M& getModel ( ) { return m; } + // Specification methods... + bool setFirst ( ) { return M::RVType::setFirst(); } + bool setNext ( ) { return M::RVType::setNext(); } + bool setFirst ( const typename M::Dep1Type& x1, const typename M::Dep2Type& x2 ) { return m.setFirst(*this,x1,x2); } + bool setFirstConsistent ( const typename M::Dep1Type& x1, const typename M::Dep2Type& x2 ) { return m.setFirstConsistent(*this,x1,x2); } + bool setNext ( const typename M::Dep1Type& x1, const typename M::Dep2Type& x2 ) { return m.setNext (*this,x1,x2); } + bool setSample ( const typename M::Dep1Type& x1, const typename M::Dep2Type& x2 ) { return m.setSample(*this,x1,x2); } + // Extraction methods... + typename M::ProbType getProb ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2 ) const { return m.getProb(*this,x1,x2); } +}; + +//////////////////////////////////////////////////////////// + +template<class M, M& m> +class Modeled4DRV : public M::RVType { + public: + // Constructor / destructor methods... + Modeled4DRV ( ) { } + //Modeled4DRV ( int i ) { M::RVType::operator=(i); } + Modeled4DRV ( string& s ) { M::RVType::operator=(s); } + // Static specification methods... + static M& setModel ( ) { return m; } + // Static extraction methods... + static const M& getModel ( ) { return m; } + // Specification methods... + bool setFirst ( ) { return M::RVType::setFirst(); } + bool setNext ( ) { return M::RVType::setNext(); } + bool setFirst ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3 ) { return m.setFirst(*this,x1,x2,x3); } + bool setFirstConsistent ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3 ) { return m.setFirstConsistent(*this,x1,x2,x3); } + bool setNext ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3 ) { return m.setNext (*this,x1,x2,x3); } + bool setSample ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3 ) { return m.setSample(*this,x1,x2,x3); } + // Extraction methods... + typename M::ProbType getProb ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3 ) const { return m.getProb(*this,x1,x2,x3); } +}; + +//////////////////////////////////////////////////////////// + +template<class M, M& m> +class Modeled5DRV : public M::RVType { + public: + // Constructor / destructor methods... + Modeled5DRV ( ) { } + //Modeled5DRV ( int i ) { M::RVType::operator=(i); } + Modeled5DRV ( string& s ) { M::RVType::operator=(s); } + // Static specification methods... + static M& setModel ( ) { return m; } + // Static extraction methods... + static const M& getModel ( ) { return m; } + // Specification methods... + bool setFirst ( ) { return M::RVType::setFirst(); } + bool setNext ( ) { return M::RVType::setNext(); } + bool setFirst ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4 ) { return m.setFirst(*this,x1,x2,x3,x4); } + bool setFirstConsistent ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4 ) { return m.setFirstConsistent(*this,x1,x2,x3,x4); } + bool setNext ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4 ) { return m.setNext (*this,x1,x2,x3,x4); } + bool setSample (const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4 ) { return m.setSample(*this,x1,x2,x3,x4); } + // Extraction methods... + typename M::ProbType getProb ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4 ) const { return m.getProb(*this,x1,x2,x3,x4); } + +}; + +/////////////////////////////////////////////////////////////////////////////// +template<class M, M& m> +class Modeled6DRV : public M::RVType { + public: + // Constructor / destructor methods... + Modeled6DRV ( ) { } + //Modeled5DRV ( int i ) { M::RVType::operator=(i); } + Modeled6DRV ( string& s ) { M::RVType::operator=(s); } + // Static specification methods... + static M& setModel ( ) { return m; } + // Static extraction methods... + static const M& getModel ( ) { return m; } + // Specification methods... + bool setFirst ( ) { return M::RVType::setFirst(); } + bool setNext ( ) { return M::RVType::setNext(); } + bool setFirst ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5 ) { return m.setFirst(*this,x1,x2,x3,x4,x5); } + bool setFirstConsistent ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5 ) { return m.setFirstConsistent(*this,x1,x2,x3,x4,x5); } + bool setNext ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5 ) { return m.setNext (*this,x1,x2,x3,x4,x5); } + bool setSample (const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5 ) { return m.setSample(*this,x1,x2,x3,x4,x5); } + // Extraction methods... + typename M::ProbType getProb ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5 ) const { return m.getProb(*this,x1,x2,x3,x4,x5); } + +}; + +/////////////////////////////////////////////////////////////////////////////// +template<class M, M& m> +class Modeled7DRV : public M::RVType { + public: + // Constructor / destructor methods... + Modeled7DRV ( ) { } + //Modeled5DRV ( int i ) { M::RVType::operator=(i); } + Modeled7DRV ( string& s ) { M::RVType::operator=(s); } + // Static specification methods... + static M& setModel ( ) { return m; } + // Static extraction methods... + static const M& getModel ( ) { return m; } + // Specification methods... + bool setFirst ( ) { return M::RVType::setFirst(); } + bool setNext ( ) { return M::RVType::setNext(); } + bool setFirst ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5, + const typename M::Dep6Type& x6 ) { return m.setFirst(*this,x1,x2,x3,x4,x5,x6); } + bool setFirstConsistent ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5, + const typename M::Dep6Type& x6 ) { return m.setFirstConsistent(*this,x1,x2,x3,x4,x5,x6); } + bool setNext ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5, + const typename M::Dep6Type& x6 ) { return m.setNext (*this,x1,x2,x3,x4,x5,x6); } + bool setSample (const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5, + const typename M::Dep6Type& x6 ) { return m.setSample(*this,x1,x2,x3,x4,x5,x6); } + // Extraction methods... + typename M::ProbType getProb ( const typename M::Dep1Type& x1, + const typename M::Dep2Type& x2, + const typename M::Dep3Type& x3, + const typename M::Dep4Type& x4, + const typename M::Dep5Type& x5, + const typename M::Dep6Type& x6 ) const { return m.getProb(*this,x1,x2,x3,x4,x5,x6); } + +}; + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// PrecomputedModeledXDRV +// +//////////////////////////////////////////////////////////////////////////////// + +template<class M, M& m, class S> +class PrecomputedModeled2DRV : public Modeled2DRV<M,m> { + private: + SafeArray1D<typename M::Dep1Type,typename M::ProbType> apr; + void precompute ( ) { apr.init(M::Dep1Type::getDomain().getSize()); + typename M::Dep1Type x1; + for(bool b=x1.setFirst(); b; b=x1.setNext()) apr.set(x1)=m.getProb(*this,x1); } + public: + typename M::ProbType getProb ( const typename M::Dep1Type& x1 ) const { return apr.get(x1); } + typename M::RVType set ( const S s ) { M::RVType::set(s); precompute(); return *this; } +}; +//template<class M, M& m, class S> SafeArray1D<typename M::Dep1Type,typename M::ProbType> PrecomputedModeled2DRV<M,m,S>::apr; + + +#endif /* _NL_PROBMODEL__ */ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-racpt.h b/contrib/synlm/hhmm/rvtl/include/nl-racpt.h new file mode 100644 index 000000000..5d1502f1f --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-racpt.h @@ -0,0 +1,332 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + + +#ifndef _NL_RACPT__ +#define _NL_RACPT__ + + +template<class K, class P> +class GenericRACPTModel : public SimpleHash<K,P> { + private: + typedef SimpleHash<K,P> HKP; +// typedef typename SimpleHash<Y,P>::const_iterator IYP; + //HKYP h; + + public: + //typedef Y RVType; + //typedef BaseIterVal<std::pair<IYP,IYP>,Y> IterVal; + //typedef typename HKYP::const_iterator const_key_iterator; + + bool contains ( const K& k ) const { + return ( SimpleHash<K,P>::contains(k) ); + } + +/* + P getProb ( const IterVal& ikyp, const K& k ) const { + if ( ikyp.iter.first == ikyp.iter.second ) { cerr<<"ERROR: no iterator to fix probability: "<<k<<endl; return P(); } + return ( ikyp.iter.first->second ); + } +*/ + + P getProb ( const K& k ) const { + return SimpleHash<K,P>::get(k); + } + P& setProb ( const K& k ) { + return SimpleHash<K,P>::set(k); + } + +/* + void normalize ( ) { + for ( typename HKYP::const_iterator ik=HKYP::begin(); ik!=HKYP::end(); ik++ ) { + K k=ik->first; + P p=P(); + IterVal y; + for(bool by=setFirst(y,k); by; by=setNext(y,k)) + p+=getProb(y,k); + if (p!=P()) + for(bool by=setFirst(y,k); by; by=setNext(y,k)) + setProb(y,k)/=p; + } + } +*/ +/* + void transmit ( int tSockfd, const char* psId ) const { + for ( typename HKYP::const_iterator ik=HKYP::begin(); ik!=HKYP::end(); ik++ ) { + K k=ik->first; + IterVal y; + // For each non-zero probability in model... + for ( bool b=setFirst(y,k); b; b=setNext(y,k) ) { + //if ( getProb(y,k) != P() ) { + String str(1000); + str<<psId<<" "<<k<<" : "<<y<<" = "<<getProb(y,k).toDouble()<<"\n"; + if ( send(tSockfd,str.c_array(),str.size()-1,MSG_EOR) != int(str.size()-1) ) + {cerr<<"ERROR writing to socket\n";exit(0);} + //} + } + } + } +*/ + void dump ( ostream& os, const char* psId ) const { + for ( typename HKP::const_iterator ik=HKP::begin(); ik!=HKP::end(); ik++ ) { + K k=ik->first; + os << psId<<" "<<k<<" = "<<getProb(k).toDouble()<<endl; + +// IterVal y; +// for ( bool b=setFirst(y,k); b; b=setNext(y,k) ) +// os<<psId<<" "<<k<<" : "<<y<<" = "<<getProb(y,k).toDouble()<<"\n"; + } + } + void subsume ( GenericRACPTModel<K,P>& m ) { + for ( typename HKP::const_iterator ik=m.HKP::begin(); ik!=m.HKP::end(); ik++ ) { + K k=ik->first; + setProb(k) = m.getProb(k); +// IterVal y; +// for ( bool b=m.setFirst(y,k); b; b=m.setNext(y,k) ) +// setProb(y,k) = m.getProb(y,k); + } + } + void clear ( ) { SimpleHash<K,P>::clear(); } + + friend pair<StringInput,GenericRACPTModel<K,P>*> operator>> ( StringInput si, GenericRACPTModel<K,P>& m ) { + return pair<StringInput,GenericRACPTModel<K,P>*>(si,&m); } + + friend StringInput operator>> ( pair<StringInput,GenericRACPTModel<K,P>*> delimbuff, const char* psD ) { + K k; + StringInput si,si2,si3; + GenericRACPTModel<K,P>& m = *delimbuff.second; + si=delimbuff.first; + if ( si==NULL ) return si; + + // Kill the colon since we're treating the whole thing as the condition + char * str = si.c_str(); + char * p = strchr(str, ':'); + if(p){ + p[0] = ' '; + } + si=str; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>k>>" "; + while((si2=si>>" ")!=NULL)si=si2; + si=si>>"= "; + while((si2=si>>" ")!=NULL)si=si2; + return (si!=NULL) ? si>>m.setProb(k)>>psD : si; + } +}; + + +template<class Y, class P> +class RandAccCPT1DModel : public GenericRACPTModel<MapKey1D<Y>,P> { + public: +// typedef typename GenericCPTModel<Y,MapKey1D<Unit>,P>::IterVal IterVal; + + bool contains ( const Y& y ) const { + return GenericRACPTModel<MapKey1D<Y>,P>::contains ( MapKey1D<Y>(y) ); + } +/* + P getProb ( const IterVal& ixyp ) const { + return GenericCPTModel<MapKey1D<Y>,P>::getProb ( ixyp, MapKey1D<Y>(Y()) ); + } +*/ + P getProb ( const Y& y ) const { + return GenericRACPTModel<MapKey1D<Y>,P>::getProb ( MapKey1D<Y>(y) ); + } +/* +P& setProb ( const Y& y ) { + cerr << "setProb called on racpt1d" << endl; + return GenericRACPTModel<MapKey1D<Y>,P>::setProb ( MapKey1D<Y>(y) ); + } +*/ + /* + bool readFields ( Array<char*>& aps ) { + if ( 3==aps.size() ) { + GenericRACPTModel<MapKey1D<Y>,P>::setProb ( MapKey1D<Y>(aps[1]) ) = atof(aps[2]); + return true; + } + return false; + } + */ +}; + + +//////////////////// +template<class Y, class X1, class P> +class RandAccCPT2DModel : public GenericRACPTModel<MapKey2D<X1,Y>,P> { + public: + + // This stuff only for deterministic 'Determ' models... +// typedef X1 Dep1Type; +// typedef P ProbType; +// MapKey1D<X1> condKey; + + bool contains ( const Y& y, const X1& x1 ) const { +// MapKey2D<X1,Y> temp = MapKey2D<X1,Y>(x1,y); + return GenericRACPTModel<MapKey2D<X1,Y>,P>::contains ( MapKey2D<X1,Y>(x1,y) ); + } + + P getProb ( const Y& y, const X1& x1 ) const { + return GenericRACPTModel<MapKey2D<X1,Y>,P>::getProb ( MapKey2D<X1,Y>(x1,y) ); + } + +/* + P& setProb ( const Y& y, const X1& x1 ) { + cerr << "setProb called on racpt2d" << endl; + return GenericRACPTModel<MapKey2D<Y,X1>,P>::setProb ( MapKey2D<Y,X1>(y,x1) ); + } +*/ + +}; + + +//////////////////// +template<class Y, class X1, class X2, class P> +class RandAccCPT3DModel : public GenericRACPTModel<MapKey3D<X1,X2,Y>,P> { + public: + + bool contains ( const Y& y, const X1& x1, const X2& x2 ) const { + return GenericRACPTModel<MapKey3D<X1,X2,Y>,P>::contains ( MapKey3D<X1,X2,Y>(x1,x2,y) ); + } + + P getProb ( const Y& y, const X1& x1, const X2& x2 ) const { + return GenericRACPTModel<MapKey3D<X1,X2,Y>,P>::getProb ( MapKey3D<X1,X2,Y>(x1,x2,y) ); + } +/* + P& setProb ( const Y& y, const X1& x1, const X2& x2 ) { + return GenericRACPTModel<MapKey3D<X1,X2,Y>,P>::setProb ( MapKey3D<Y,X1,X2>(x1,x2,y) ); + } +*/ +}; + +/* +//////////////////// +template<class Y, class X1, class X2, class X3, class P> +class CPT4DModel : public GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P> { + public: + typedef typename GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::IterVal IterVal; + + bool setFirst ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::setFirst ( ixyp, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + bool setNext ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::setNext ( ixyp, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + bool contains ( const Y& y, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::contains ( y, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::contains ( MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + P getProb ( const IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::getProb ( ixyp, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + P getProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3 ) const { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::getProb ( y, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3 ) { + return GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::setProb ( y, MapKey3D<X1,X2,X3>(x1,x2,x3) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 6==aps.size() ) { + GenericCPTModel<Y,MapKey3D<X1,X2,X3>,P>::setProb ( Y(aps[4]), MapKey3D<X1,X2,X3>(aps[1],aps[2],aps[3]) ) = atof(aps[5]); + return true; + } + return false; + } +}; + + +//////////////////// +template<class Y, class X1, class X2, class X3, class X4, class P> +class CPT5DModel : public GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P> { + public: + typedef typename GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::IterVal IterVal; + + bool setFirst ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::setFirst ( ixyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + bool setNext ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::setNext ( ixyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + bool contains ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::contains ( y, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::contains ( MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + P getProb ( const IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::getProb ( ixyp, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + P getProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) const { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::getProb ( y, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4 ) { + return GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::setProb ( y, MapKey4D<X1,X2,X3,X4>(x1,x2,x3,x4) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 7==aps.size() ) { + GenericCPTModel<Y,MapKey4D<X1,X2,X3,X4>,P>::setProb ( Y(aps[5]), MapKey4D<X1,X2,X3,X4>(aps[1],aps[2],aps[3],aps[4]) ) = atof(aps[6]); + return true; + } + return false; + } +}; + + +//////////////////// +template<class Y, class X1, class X2, class X3, class X4, class X5, class P> +class RACPT6DModel : public GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P> { + public: + typedef typename GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::IterVal IterVal; + + bool setFirst ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::setFirst ( ixyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + bool setNext ( IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::setNext ( ixyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + bool contains ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::contains ( y, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + bool contains ( const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::contains ( MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + P getProb ( const IterVal& ixyp, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::getProb ( ixyp, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + P getProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) const { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::getProb ( y, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + P& setProb ( const Y& y, const X1& x1, const X2& x2, const X3& x3, const X4& x4, const X5& x5 ) { + return GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::setProb ( y, MapKey5D<X1,X2,X3,X4,X5>(x1,x2,x3,x4,x5) ); + } + bool readFields ( Array<char*>& aps ) { + if ( 8==aps.size() ) { + GenericCPTModel<Y,MapKey5D<X1,X2,X3,X4,X5>,P>::setProb ( Y(aps[6]), MapKey5D<X1,X2,X3,X4,X5>(aps[1],aps[2],aps[3],aps[4],aps[5]) ) = atof(aps[7]); + return true; + } + return false; + } +}; + +*/ +#endif //_NL_RACPT__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-randvar.h b/contrib/synlm/hhmm/rvtl/include/nl-randvar.h new file mode 100644 index 000000000..66cc0b8f2 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-randvar.h @@ -0,0 +1,593 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_RAND_VAR__ +#define _NL_RAND_VAR__ + +#include <math.h> +#include <string> +#include "nl-string.h" +#include "nl-safeids.h" +#include "nl-stringindex.h" +#include "nl-prob.h" +#include "nl-hash.h" + +//////////////////////////////////////////////////////////////////////////////// + +template <class A, class B, class C> +class trip { + public: + trip ( ) { } + trip ( A& a, B& b, C& c ) : first(a), second(b), third(c) { } + A first; + B second; + C third; + friend ostream& operator<< ( ostream& os, const trip<A,B,C>& a ) { return os<<a.first<<","<<a.second<<","<<a.third; } +}; + +template <class A, class B, class C, class D> +class quad { + public: + quad ( ) { } + quad ( A& a, B& b, C& c, D& d ) : first(a), second(b), third(c), fourth(d) { } + A first; + B second; + C third; + D fourth; + friend ostream& operator<< ( ostream& os, const quad<A,B,C,D>& a ) { return os<<a.first<<","<<a.second<<","<<a.third<<","<<a.fourth; } +}; + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// DiscreteDomainRV template -- creates RV with a distinct set of values for domain T (unique class) +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////// +template <class T> +class DiscreteDomain : public StringIndex { + public: + typedef T ValType; + int MAX_SIZE ; + DiscreteDomain ( ) : StringIndex() { } + DiscreteDomain ( int i ) : StringIndex() { MAX_SIZE=i; } + int addIndex ( const char* ps ) { int i=StringIndex::addIndex(ps); assert(i==T(i)); return i; } +}; + +//////////////////////////////////////////////////////////// +template <class T, DiscreteDomain<T>& domain> +class DiscreteDomainRV : public Id<T> { + private: + + static String strTemp; + + public: + + typedef DiscreteDomainRV<T,domain> BaseType; + + static const int NUM_VARS = 1; + + //////////////////// + template<class P> + class ArrayDistrib : public Array<pair<DiscreteDomainRV<T,domain>,P> > { + }; + + //////////////////// + template<class P> + class ArrayIterator : public pair<SafePtr<const ArrayDistrib<P> >,Id<int> > { + public: + static const int NUM_ITERS = NUM_VARS; + operator DiscreteDomainRV<T,domain>() const { return ArrayIterator<P>::first.getRef().get(ArrayIterator<P>::second.toInt()).first; } + //const DiscreteDomainRV<T,domain>& toRV() { return ArrayIterator<P>::first.getRef().get(ArrayIterator<P>::second.toInt()).first; } + bool end ( ) const { return ( ArrayIterator<P>::second >= ArrayIterator<P>::first.getRef().getSize() ); } + ArrayIterator<P>& operator++ ( ) { ++ArrayIterator<P>::second; return *this; } + }; + + // Static extraction methods... + static const DiscreteDomain<T>& getDomain ( ) { return domain; } + + // Constructor / destructor methods... + DiscreteDomainRV ( ) { Id<T>::set(0); } + DiscreteDomainRV ( int i ) { Id<T>::set(i); } + DiscreteDomainRV ( const char* ps ) { assert(ps!=NULL); Id<T>::set(domain.addIndex(ps)); } + + // Specification methods... + template<class P> + DiscreteDomainRV<T,domain>& setVal ( const ArrayIterator<P>& it ) { *this=it; return *this; } + bool setFirst ( ) { Id<T>::set(0); return isValid(); } + bool setNext ( ) { Id<T>::setNext(); if (!isValid()){Id<T>::set(0); return false;} return true; } + + // Extraction methods... + bool isValid ( ) const { return *this<domain.getSize(); } //return (this->Id<T>::operator<(domain.getSize())); } + int getIndex ( ) const { return Id<T>::toInt(); } // DO NOT DELETE THIS METHOD!!!!!!!!!! + string getString ( ) const { return domain.getString(Id<T>::toInt()); } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const DiscreteDomainRV<T,domain>& rv ) { return os<<rv.getString(); } + friend String& operator<< ( String& str, const DiscreteDomainRV<T,domain>& rv ) { return str<<rv.getString(); } + friend pair<StringInput,DiscreteDomainRV<T,domain>*> operator>> ( const StringInput ps, DiscreteDomainRV<T,domain>& rv ) { return pair<StringInput,DiscreteDomainRV<T,domain>*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,DiscreteDomainRV<T,domain>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + ////assert(*delimbuff.second<domain.getSize()); + int j=0; + StringInput psIn = delimbuff.first; + if(psDlm[0]=='\0') { *delimbuff.second=psIn.c_str(); return psIn+strlen(psIn.c_str()); } + for(int i=0;psIn[i]!='\0';i++) { + if(psIn[i]==psDlm[j]) j++; + else j=0; + strTemp[i]=psIn[i]; + if(j==int(strlen(psDlm))) { strTemp[i+1-j]='\0'; /*delimbuff.second->set(domain.addIndex(psIn.c_str()));*/ *delimbuff.second=strTemp.c_array(); return psIn+i+1;} + } + return NULL; //psIn; + } +}; +template <class T, DiscreteDomain<T>& domain> +String DiscreteDomainRV<T,domain>::strTemp ( 100 ); + + +/* DON'T COMMENT BACK IN!!! THIS HAS BEEN MOVED TO nl-refrv.h!!!!!! +//////////////////////////////////////////////////////////// +template <class T> +class RefRV : public Id<const T*> { + public: + + typedef RefRV<T> BaseType; + + static const int NUM_VARS = 1; + static const T DUMMY; + + //////////////////// + template<class P> + class ArrayDistrib : public Array<pair<RefRV<T>,P> > { + }; + + //////////////////// + template<class P> + class ArrayIterator : public pair<SafePtr<const ArrayDistrib<P> >,Id<int> > { + public: + static const int NUM_ITERS = NUM_VARS; + operator RefRV<T>() const { return ArrayIterator<P>::first.getRef().get(ArrayIterator<P>::second.toInt()).first; } + //const DiscreteDomainRV<T,domain>& toRV() { return ArrayIterator<P>::first.getRef().get(ArrayIterator<P>::second.toInt()).first; } + }; + + // Constructor / destructor methods... + RefRV ( ) { Id<const T*>::set(NULL); } + RefRV ( const T& t ) { Id<const T*>::set(&t); } + + // Specification methods... + template<class P> + RefRV<T>& setVal ( const ArrayIterator<P>& it ) { *this=it; return *this; } + + // Extraction methods... + const T& getRef ( ) const { return (Id<const T*>::toInt()==NULL) ? DUMMY : *(static_cast<const T*>(Id<const T*>::toInt())); } + + // Input / output methods.. + friend ostream& operator<< ( ostream& os, const RefRV<T>& rv ) { return os <<&rv.getRef(); } //{ return os<<rv.getRef(); } + friend String& operator<< ( String& str, const RefRV<T>& rv ) { return str<<"addr"<<(long int)(void*)&rv.getRef(); } //{ return str<<rv.getRef(); } + friend pair<StringInput,RefRV<T>*> operator>> ( const StringInput ps, RefRV<T>& rv ) { return pair<StringInput,RefRV<T>*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,RefRV<T>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + return NULL; //psIn; + } +}; +template <class T> const T RefRV<T>::DUMMY; +*/ + + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// Joint2DRV +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////// +template<class V1,class V2> +class Joint2DRV { + + public: + + V1 first; + V2 second; + + // Constructor / destructor methods... + Joint2DRV ( ) { } + Joint2DRV ( const V1& v1, const V2& v2 ) { first=v1; second=v2; } + + // Extraction methods... + size_t getHashKey ( ) const { size_t k=rotLeft(first.getHashKey(),3); k^=second.getHashKey(); + /*fprintf(stderr," (%d) %d ^& %d = %d\n",sizeof(*this),x1.getHashKey(),x2.getHashKey(),k);*/ return k; } + bool operator< ( const Joint2DRV<V1,V2>& j ) const { return ( (first<j.first) || + (first==j.first && second<j.second) ); } + bool operator== ( const Joint2DRV<V1,V2>& j ) const { return ( first==j.first && second==j.second ); } + bool operator!= ( const Joint2DRV<V1,V2>& j ) const { return ( !(first==j.first && second==j.second) ); } +}; + + +//////////////////////////////////////////////////////////// +template<char* SD1,class V1,char* SD2,class V2,char* SD3> +class DelimitedJoint2DRV : public Joint2DRV<V1,V2> { + + public: + + static const int NUM_VARS = V1::NUM_VARS + V2::NUM_VARS; + + //////////////////// + template<class P> + class ArrayIterator : public pair<typename V1::template ArrayIterator<P>, typename V2::template ArrayIterator<P> > { + public: + // static const int NUM_ITERS = (typename V1::template ArrayIterator<P>)::NUM_ITERS + (typename V2::template ArrayIterator<P>)::NUM_ITERS; + static const int NUM_ITERS = NUM_VARS; +// DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& set ( DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& rv ) const { first.set(rv.first=first); rv.second=second; return rv; } + friend ostream& operator<< ( ostream& os, const ArrayIterator<P>& rv ) { + return os<<SD1<<rv.first<<SD2<<rv.second<<SD3; } + }; + + // Constructor / destructor methods... + DelimitedJoint2DRV ( ) : Joint2DRV<V1,V2>() { } + DelimitedJoint2DRV ( const V1& v1, const V2& v2 ) : Joint2DRV<V1,V2>(v1,v2) { } + DelimitedJoint2DRV ( char* ps ) : Joint2DRV<V1,V2>() { ps>>*this>>"\0"; } + DelimitedJoint2DRV ( const char* ps ) : Joint2DRV<V1,V2>() { strdup(ps)>>*this>>"\0"; } //DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>(strdup(ps)) { } + + // Specification methods... + template<class P> + DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& setVal ( const ArrayIterator<P>& it ) { + Joint2DRV<V1,V2>::first.setVal(it.first); Joint2DRV<V1,V2>::second.setVal(it.second); return *this; } + + // Extraction methods... + bool operator==(const DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& vv) const { return Joint2DRV<V1,V2>::operator==(vv); } + bool operator< (const DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& vv) const { return Joint2DRV<V1,V2>::operator<(vv); } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& rv ) { return os<<SD1<<rv.first<<SD2<<rv.second<<SD3; } + friend String& operator<< ( String& str, const DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& rv ) { return str<<SD1<<rv.first<<SD2<<rv.second<<SD3; } + friend IStream operator>> ( pair<IStream,DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& x = *is_x.second; + // Propagate fail... + if ( IStream()==is ) return is; + // Use last delimiter only if not empty (otherwise it will immediately trivially match)... + return ( (SD3[0]=='\0') ? is>>SD1>>x.first>>SD2>>x.second>>psDlm + : is>>SD1>>x.first>>SD2>>x.second>>SD3>>psDlm ); + } + + // OBSOLETE! + friend pair<StringInput,DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>*> operator>> ( StringInput ps, DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>& rv ) { return pair<StringInput,DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,DelimitedJoint2DRV<SD1,V1,SD2,V2,SD3>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + return ( (SD3[0]=='\0') ? delimbuff.first>>SD1>>delimbuff.second->first>>SD2>>delimbuff.second->second>>psDlm + : delimbuff.first>>SD1>>delimbuff.second->first>>SD2>>delimbuff.second->second>>SD3>>psDlm ); + } +}; + + + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// Joint3DRV +// +//////////////////////////////////////////////////////////////////////////////// + +template<class V1,class V2,class V3> +class Joint3DRV { + + public: + + V1 first; + V2 second; + V3 third; + + // Constructor / destructor methods... + Joint3DRV ( ) { } + Joint3DRV ( const V1& v1, const V2& v2, const V3& v3 ) { first=v1; second=v2; third=v3; } + + /* + // Specification methods... + bool operator< ( const Joint3DRV<V1,V2,V3>& j ) const { + return ( (x1<j.x1) || + (x1==j.x1 && x2<j.x2) || + (x1==j.x1 && x2==j.x2 && x3<j.x3) ) ; + } + */ + + // Extraction methods... + size_t getHashKey ( ) const { size_t k=rotLeft(first.getHashKey(),3); k^=second.getHashKey(); k=rotLeft(k,3); k^=third.getHashKey(); + /*fprintf(stderr," (%d) %d ^& %d = %d\n",sizeof(*this),x1.getHashKey(),x2.getHashKey(),k);*/ return k; } +// bool operator< ( const Joint2DRV<V1,V2>& j ) const { return ( (first<j.first) || +// (first==j.first && second<j.second) ); } + bool operator== ( const Joint3DRV<V1,V2,V3>& j ) const { return ( first==j.first && second==j.second && third==j.third ); } + bool operator!= ( const Joint3DRV<V1,V2,V3>& j ) const { return ( !(first==j.first && second==j.second && third==j.third) ); } +}; + +//////////////////////////////////////////////////////////// +template<char* SD1,class V1,char* SD2,class V2,char* SD3,class V3,char* SD4> +class DelimitedJoint3DRV : public Joint3DRV<V1,V2,V3> { + + public: + + static const int NUM_VARS = V1::NUM_VARS + V2::NUM_VARS + V3::NUM_VARS; + + //////////////////// + template<class P> + class ArrayIterator : public trip<typename V1::template ArrayIterator<P>, typename V2::template ArrayIterator<P>, typename V3::template ArrayIterator<P> > { + public: + // static const int NUM_ITERS = (typename V1::template ArrayIterator<P>)::NUM_ITERS + (typename V2::template ArrayIterator<P>)::NUM_ITERS; + static const int NUM_ITERS = NUM_VARS; + friend ostream& operator<< ( ostream& os, const ArrayIterator<P>& rv ) { + return os<<SD1<<rv.first<<SD2<<rv.second<<SD3<<rv.third<<SD4; } + }; + + // Constructor / destructor methods... + DelimitedJoint3DRV ( ) : Joint3DRV<V1,V2,V3>() { } + DelimitedJoint3DRV ( const V1& v1, const V2& v2, const V3& v3 ) : Joint3DRV<V1,V2,V3>(v1,v2,v3) { } + DelimitedJoint3DRV ( char* ps ) : Joint3DRV<V1,V2,V3>() { ps>>*this>>"\0"; } + DelimitedJoint3DRV ( const char* ps ) : Joint3DRV<V1,V2,V3>() { strdup(ps)>>*this>>"\0"; } + + // Specification methods... + template<class P> + DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>& setVal ( const ArrayIterator<P>& it ) { + Joint3DRV<V1,V2,V3>::first.setVal(it.first); Joint3DRV<V1,V2,V3>::second.setVal(it.second); Joint3DRV<V1,V2,V3>::third.setVal(it.third); return *this; } + + // Extraction methods... + bool operator==(const DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>& vvv) const { return Joint3DRV<V1,V2,V3>::operator==(vvv); } + bool operator< (const DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>& vvv) const { return Joint3DRV<V1,V2,V3>::operator< (vvv); } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>& rv ) { return os<<SD1<<rv.first<<SD2<<rv.second<<SD3<<rv.third<<SD4; } + friend String& operator<< ( String& str, const DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>& rv ) { return str<<SD1<<rv.first<<SD2<<rv.second<<SD3<<rv.third<<SD4; } + friend pair<StringInput,DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>*> operator>> ( StringInput ps, DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>& rv ) { + return pair<StringInput,DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,DelimitedJoint3DRV<SD1,V1,SD2,V2,SD3,V3,SD4>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + return ( (SD4[0]=='\0') ? delimbuff.first>>SD1>>delimbuff.second->first>>SD2>>delimbuff.second->second>>SD3>>delimbuff.second->third>>psDlm + : delimbuff.first>>SD1>>delimbuff.second->first>>SD2>>delimbuff.second->second>>SD3>>delimbuff.second->third>>SD4>>psDlm ); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +// +// Joint4DRV +// +//////////////////////////////////////////////////////////////////////////////// + +template<class V1,class V2,class V3, class V4> +class Joint4DRV { + + public: + + V1 first; + V2 second; + V3 third; + V4 fourth; + + // Constructor / destructor methods... + Joint4DRV ( ) { } + Joint4DRV ( const V1& v1, const V2& v2, const V3& v3, const V4& v4 ) { first=v1; second=v2; third=v3; fourth=v4;} + // Extraction methods... + size_t getHashKey ( ) const { size_t k=rotLeft(first.getHashKey(),3); k^=second.getHashKey(); k=rotLeft(k,3); k^=third.getHashKey();k^=fourth.getHashKey(); + /*fprintf(stderr," (%d) %d ^& %d = %d\n",sizeof(*this),x1.getHashKey(),x2.getHashKey(),k);*/ return k; } +// bool operator< ( const Joint2DRV<V1,V2>& j ) const { return ( (first<j.first) || +// (first==j.first && second<j.second) ); } + bool operator== ( const Joint4DRV<V1,V2,V3,V4>& j ) const { return ( first==j.first && second==j.second && third==j.third && fourth==j.fourth ); } + bool operator!= ( const Joint4DRV<V1,V2,V3,V4>& j ) const { return ( !(first==j.first && second==j.second && third==j.third && fourth==j.fourth) ); } +}; + +//////////////////////////////////////////////////////////// +template<char* SD1,class V1,char* SD2,class V2,char* SD3,class V3,char* SD4,class V4, char* SD5> +class DelimitedJoint4DRV : public Joint4DRV<V1,V2,V3,V4> { + + public: + + static const int NUM_VARS = V1::NUM_VARS + V2::NUM_VARS + V3::NUM_VARS+ V4::NUM_VARS; + + //////////////////// + template<class P> + class ArrayIterator : public quad<typename V1::template ArrayIterator<P>, typename V2::template ArrayIterator<P>, typename V3::template ArrayIterator<P> , typename V4::template ArrayIterator<P> > { + public: + // static const int NUM_ITERS = (typename V1::template ArrayIterator<P>)::NUM_ITERS + (typename V2::template ArrayIterator<P>)::NUM_ITERS; + static const int NUM_ITERS = NUM_VARS; + friend ostream& operator<< ( ostream& os, const ArrayIterator<P>& rv ) { + return os<<SD1<<rv.first<<SD2<<rv.second<<SD3<<rv.third<<SD4<<rv.fourth<<SD5; } + }; + + // Constructor / destructor methods... + DelimitedJoint4DRV ( ) : Joint4DRV<V1,V2,V3,V4>() { } + DelimitedJoint4DRV ( const V1& v1, const V2& v2, const V3& v3, const V4& v4 ) : Joint4DRV<V1,V2,V3,V4>(v1,v2,v3,v4) { } + DelimitedJoint4DRV ( char* ps ) : Joint4DRV<V1,V2,V3,V4>() { ps>>*this>>"\0"; } + DelimitedJoint4DRV ( const char* ps ) : Joint4DRV<V1,V2,V3,V4>() { strdup(ps)>>*this>>"\0"; } + + // Specification methods... + template<class P> + DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>& setVal ( const ArrayIterator<P>& it ) { + Joint4DRV<V1,V2,V3,V4>::first.setVal(it.first); + Joint4DRV<V1,V2,V3,V4>::second.setVal(it.second); + Joint4DRV<V1,V2,V3,V4>::third.setVal(it.third); + Joint4DRV<V1,V2,V3,V4>::fourth.setVal(it.fourth); + return *this; + } + + // Extraction methods... + bool operator==(const DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>& vvvv) const { return Joint4DRV<V1,V2,V3,V4>::operator==(vvvv); } + bool operator< (const DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>& vvvv) const { return Joint4DRV<V1,V2,V3,V4>::operator< (vvvv); } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>& rv ) { return os<<SD1<<rv.first<<SD2<<rv.second<<SD3<<rv.third<<SD4<<rv.fourth<<SD5; } + friend String& operator<< ( String& str, const DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>& rv ) { return str<<SD1<<rv.first<<SD2<<rv.second<<SD3<<rv.third<<SD4<<rv.fourth<<SD5; } + friend pair<StringInput,DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>*> operator>> ( StringInput ps, DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>& rv ) { + return pair<StringInput,DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,DelimitedJoint4DRV<SD1,V1,SD2,V2,SD3,V3,SD4,V4,SD5>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + return ( (SD5[0]=='\0') ? delimbuff.first>>SD1>>delimbuff.second->first>>SD2>>delimbuff.second->second>>SD3>>delimbuff.second->third>>SD4>>delimbuff.second->fourth>>psDlm + : delimbuff.first>>SD1>>delimbuff.second->first>>SD2>>delimbuff.second->second>>SD3>>delimbuff.second->third>>SD4>>delimbuff.second->fourth>>SD5>>psDlm ); + } +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// JointArrayRV<T,I> +// +//////////////////////////////////////////////////////////////////////////////// + +template <int I, class T> +class JointArrayRV { + private: + // Data members... + T at[I]; + public: + typedef T ElementType; + + /* + // Constructor / destructor methods... + JointArrayRV ( ) { } + JointArrayRV ( const T& t ) { for(int i=0;i<I;i++) at[i]=t; } + */ + + // Static extraction methods... + static const int SIZE = I; + static const int getSize ( ) { return I; } + + // Specification methods... + T& set (int i) { assert(0<=i); assert(i<I); return at[i]; } + + // Extraction methods... + const T& get (int i) const { assert(NULL!=this); assert(0<=i); assert(i<I); return at[i]; } + bool operator< ( const JointArrayRV<I,T>& a ) const { + int i; + for ( i=0; at[i]==a.at[i] && i<I; i++ ) ; + return ( i<I && at[i]<a.at[i] ) ; + } + bool operator== ( const JointArrayRV<I,T>& a ) const { + int i; + for ( i=0; at[i]==a.at[i] && i<I; i++ ) ; + return ( i==I ) ; + } + size_t getHashKey ( ) const { size_t k=0; for(int i=0;i<I;i++){k=rotLeft(k,3); k^=get(i).getHashKey(); } return k; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <int I, char* SD, class T> +class DelimitedJointArrayRV : public JointArrayRV<I,T> { + public: + + static const int NUM_VARS = T::NUM_VARS * I; + + //////////////////// + template<class P> + class ArrayIterator : public StaticSafeArray<I,typename T::template ArrayIterator<P> > { + public: + static const int NUM_ITERS = NUM_VARS; + // static const int NUM_ITERS = (typename T::template ArrayIterator<P>)::NUM_ITERS * I; + friend ostream& operator<< ( ostream& os, const ArrayIterator<P>& rv ) { for(int i=0;i<I;i++) os<<((i==0)?"":SD)<<rv.get(i); return os; } + }; + + // Specification methods... + template<class P> + DelimitedJointArrayRV<I,SD,T>& setVal ( const ArrayIterator<P>& it ) { + for(int i=0;i<I;i++) JointArrayRV<I,T>::set(i).setVal(it.get(i)); return *this; } + + // Extraction methods... + bool operator==(const DelimitedJointArrayRV<I,SD,T>& a) const { return JointArrayRV<I,T>::operator==(a); } + bool operator< (const DelimitedJointArrayRV<I,SD,T>& a) const { return JointArrayRV<I,T>::operator<(a); } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const DelimitedJointArrayRV<I,SD,T>& a ) { for(int i=0;i<I;i++) os<<((i==0)?"":SD)<<a.get(i); return os; } + friend String& operator<< ( String& str, const DelimitedJointArrayRV<I,SD,T>& a ) { for(int i=0;i<I;i++)str<<((i==0)?"":SD)<<a.get(i); return str; } + friend IStream operator>> ( pair<IStream,DelimitedJointArrayRV<I,SD,T>*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + DelimitedJointArrayRV<I,SD,T>& x = *is_x.second; + if (IStream()==is) return IStream(); + for(int i=0;i<I;i++) + is = pair<IStream,T*>(is,&x.set(i))>>((i<I-1)?SD:psDlm); + return is; + } + + // OBSOLETE! + friend pair<StringInput,DelimitedJointArrayRV<I,SD,T>*> operator>> ( StringInput ps, DelimitedJointArrayRV<I,SD,T>& a ) { return pair<StringInput,DelimitedJointArrayRV<I,SD,T>*>(ps,&a); } + friend StringInput operator>> ( pair<StringInput,DelimitedJointArrayRV<I,SD,T>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + StringInput psIn = delimbuff.first; + for(int i=0;i<I;i++) + psIn = pair<StringInput,T*>(psIn,&delimbuff.second->set(i))>>((i<I-1)?SD:psDlm); + return psIn; + } +}; + + +/////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// History<T,N> +// +//////////////////////////////////////////////////////////////////////////////// + +template <int N,class T> +class History { + private: + // Data members... + StaticSafeArray<N,T> at; + public: + // Constructor / destructor methods... + History ( ) { } + History ( char* ps ) { ps>>*this>>"\0"; } + /* + History ( char* ps ) { read(ps); } + */ +// History ( const string& s ) { read(s.c_str()); } + // Specification methods... + void advanceHistory(const T& t) { for(int i=N-1;i>0;i--)at.set(i)=at.get(i-1); at.set(0)=t; } + T& advanceHistory() { for(int i=N-1;i>0;i--)at.set(i)=at.get(i-1); return at.set(0); } + T& setBack(int i) { return at.set(i); } + // Extraction methods... + const T& getBack(int i) const { assert(i>=0); assert(i<N); return at.get(i); } + // Input / output methods... + /* + void read ( char* ps, const ReaderContext& rc=ReaderContext() ) { char* psT; for(int i=0;i<N;i++){char* z=strtok_r((0==i)?ps:NULL,";",&psT); assert(z); at.set(i).read(z);} } + //at.set(i).read(strtok_r((0==i)?ps:NULL,";",&psT)); } + */ + + friend ostream& operator<< ( ostream& os, const History<N,T>& a ) { for(int i=0;i<N;i++)os<<((i==0)?"":";")<<a.getBack(i); return os; } + friend pair<StringInput,History<N,T>*> operator>> ( StringInput ps, History<N,T>& a ) { return pair<StringInput,History<N,T>*>(ps,&a); } + friend StringInput operator>> ( pair<StringInput,History<N,T>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + StringInput psIn = delimbuff.first; + for(int i=0;i<N;i++) + psIn = pair<StringInput,T*>(psIn,&delimbuff.second->setBack(i))>>((i<N-1)?";":psDlm); + return psIn; + } + + /* + void write ( FILE* pf ) const { for(int i=0;i<N;i++) {fprintf(pf,(0==i)?"":";"); at.get(i).write(pf);} } + */ +}; + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + + +#endif //_NL_RAND_VAR__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-refrv.h b/contrib/synlm/hhmm/rvtl/include/nl-refrv.h new file mode 100644 index 000000000..1e4eca4fd --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-refrv.h @@ -0,0 +1,74 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////// + //template <class T> +template<class T, map<T,T>& domain> +class RefRV : public Id<const T*> { + public: + + typedef RefRV<T,domain> BaseType; + + static const int NUM_VARS = 1; + static const T DUMMY; + + //////////////////// + template<class P> + class ArrayDistrib : public Array<pair<RefRV<T,domain>,P> > { + }; + + //////////////////// + template<class P> + class ArrayIterator : public pair<SafePtr<const ArrayDistrib<P> >,Id<int> > { + public: + static const int NUM_ITERS = NUM_VARS; + operator RefRV<T,domain>() const { return ArrayIterator<P>::first.getRef().get(ArrayIterator<P>::second.toInt()).first; } + //const DiscreteDomainRV<T,domain>& toRV() { return ArrayIterator<P>::first.getRef().get(ArrayIterator<P>::second.toInt()).first; } + }; + + // Constructor / destructor methods... + RefRV ( ) { Id<const T*>::set(NULL); } + //RefRV ( int i ) { Id<const T*>::set(i); } + //RefRV ( const T& t ) { Id<const T*>::set(&t); } + RefRV ( const T& t ) { if(domain.find(t)==domain.end()) *(const_cast<T*>(Id<const T*>::set(&domain[t]).toInt())) = t; + else Id<const T*>::set(&domain[t]); } + + // Specification methods... + template<class P> + RefRV<T,domain>& setVal ( const ArrayIterator<P>& it ) { *this=it; return *this; } + //T& setRef ( ) { return Id<const T*>::setRef(); } + + // Extraction methods... + const T& getRef ( ) const { return (Id<const T*>::toInt()==NULL) ? DUMMY : *(static_cast<const T*>(Id<const T*>::toInt())); } + static map<T,T>& setDomain ( ) { return domain; } + + // Input / output methods.. + friend ostream& operator<< ( ostream& os, const RefRV<T,domain>& rv ) { return os <<&rv.getRef(); } //{ return os<<rv.getRef(); } + friend String& operator<< ( String& str, const RefRV<T,domain>& rv ) { return str<<"addr"<<(long int)(void*)&rv.getRef(); } //{ return str<<rv.getRef(); } + friend pair<StringInput,RefRV<T,domain>*> operator>> ( const StringInput ps, RefRV<T,domain>& rv ) { return pair<StringInput,RefRV<T,domain>*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,RefRV<T,domain>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + return NULL; //psIn; + } +}; +template <class T, map<T,T>& domain> const T RefRV<T,domain>::DUMMY; diff --git a/contrib/synlm/hhmm/rvtl/include/nl-safeids.h b/contrib/synlm/hhmm/rvtl/include/nl-safeids.h new file mode 100644 index 000000000..50837c366 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-safeids.h @@ -0,0 +1,503 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_SAFE_IDS__ +#define _NL_SAFE_IDS__ + +#include "nl-const.h" +#include "nl-stringindex.h" +//#include "nl-string.h" +#include "nl-stream.h" + +#include <iostream> +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// StaticSafeArray<T,I> +// +//////////////////////////////////////////////////////////////////////////////// + +template <int I, class T> +class StaticSafeArray { + private: + // Data members... + T at[I]; + public: + typedef T ElementType; + template<class P> class Iterator : public StaticSafeArray<I,typename T::template Iterator<P> > { + public: + //operator StaticSafeArray<I,T>() const { } + }; + // Constructor / destructor methods... + StaticSafeArray ( ) { } + StaticSafeArray ( const T& t ) { for(int i=0;i<I;i++) at[i]=t; } + StaticSafeArray ( const StaticSafeArray<I,T>& a ) { for(int i=0;i<I;i++) at[i]=a.at[i]; } + // Static extraction methods... + static const int SIZE = I; + static const int getSize ( ) { return I; } + // Math methods... + StaticSafeArray<I,T> operator+ ( const StaticSafeArray<I,T>& a ) const { StaticSafeArray<I,T> aOut; for(int i=0;i<I;i++) aOut.at[i]=at[i]+a.at[i]; return aOut; } + StaticSafeArray<I,T> operator- ( const StaticSafeArray<I,T>& a ) const { StaticSafeArray<I,T> aOut; for(int i=0;i<I;i++) aOut.at[i]=at[i]-a.at[i]; return aOut; } + //StaticSafeArray<I,T> operator+ ( const T& t ) const { StaticSafeArray<I,T> aOut; for(int i=0;i<I;i++) aOut.at[i]=at[i]+t; return aOut; } + //StaticSafeArray<I,T> operator- ( const T& t ) const { StaticSafeArray<I,T> aOut; for(int i=0;i<I;i++) aOut.at[i]=at[i]-t; return aOut; } + // Specification methods... + T& set (int i) { assert(0<=i); assert(i<I); return at[i]; } + T& operator[](int i) { assert(0<=i); assert(i<I); return at[i]; } + // Extraction methods... + const T& get (int i) const { assert(NULL!=this); assert(0<=i); assert(i<I); return at[i]; } + const T& operator[](int i) const { assert(NULL!=this); assert(0<=i); assert(i<I); return at[i]; } + bool operator< ( const StaticSafeArray<I,T>& a ) const { + int i; + for ( i=0; at[i]==a.at[i] && i<I; i++ ) ; + return ( i<I && at[i]<a.at[i] ) ; + } + bool operator== ( const StaticSafeArray<I,T>& a ) const { + int i; + for ( i=0; at[i]==a.at[i] && i<I; i++ ) ; + return ( i==I ) ; + } + size_t getHashKey ( ) const { size_t k=0; for(int i=0;i<I;i++){k=rotLeft(k,3); k^=get(i).getHashKey(); } return k; } +// friend ostream& operator<< ( ostream& os, const StaticSafeArray<I,T>& a ) { for(int i=0;i<I;i++) os<<((i==0)?"":",")<<a.get(i); return os; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <int I, char* SD, class T> +class DelimitedStaticSafeArray : public StaticSafeArray<I,T> { + public: + DelimitedStaticSafeArray ( ) : StaticSafeArray<I,T>() { } + DelimitedStaticSafeArray ( const T& t ) : StaticSafeArray<I,T>(t) { } + DelimitedStaticSafeArray ( const StaticSafeArray<I,T>& a ) : StaticSafeArray<I,T>(a) { } + + bool operator==(const DelimitedStaticSafeArray<I,SD,T>& a) const { return StaticSafeArray<I,T>::operator==(a); } + bool operator< (const DelimitedStaticSafeArray<I,SD,T>& a) const { return StaticSafeArray<I,T>::operator<(a); } + + friend ostream& operator<< ( ostream& os, const DelimitedStaticSafeArray<I,SD,T>& a ) { for(int i=0;i<I;i++) os<<((i==0)?"":SD)<<a.get(i); return os; } + friend String& operator<< ( String& str, const DelimitedStaticSafeArray<I,SD,T>& a ) { for(int i=0;i<I;i++)str<<((i==0)?"":SD)<<a.get(i); return str; } + friend IStream operator>> ( pair<IStream,DelimitedStaticSafeArray<I,SD,T>*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + DelimitedStaticSafeArray<I,SD,T>& x = *is_x.second; + if (IStream()==is) return IStream(); + for(int i=0;i<I;i++) + is = pair<IStream,T*>(is,&x.set(i))>>((i<I-1)?SD:psDlm); + return is; + } + + + // OBSOLETE! + friend pair<StringInput,DelimitedStaticSafeArray<I,SD,T>*> operator>> ( StringInput ps, DelimitedStaticSafeArray<I,SD,T>& a ) { return pair<StringInput,DelimitedStaticSafeArray<I,SD,T>*>(ps,&a); } + friend StringInput operator>> ( pair<StringInput,DelimitedStaticSafeArray<I,SD,T>*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + StringInput psIn = delimbuff.first; + for(int i=0;i<I;i++) + psIn = pair<StringInput,T*>(psIn,&delimbuff.second->set(i))>>((i<I-1)?SD:psDlm); + return psIn; + } + + /* + void read ( char* ps ) { char* psT; for(int i=0;i<I;i++){char* z=strtok_r((0==i)?ps:NULL,SD,&psT); assert(z); + StaticSafeArray<I,T>::set(i).read(z);} } + void read ( char* ps, ReaderContext& rc ) { read(ps); } + void write ( FILE* pf ) const { for(int i=0;i<I;i++){fprintf(pf,(0==i)?"":SD); StaticSafeArray<I,T>::get(i).write(pf);} } + void write ( FILE* pf, ReaderContext& rc ) const { write(pf); } + */ +}; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Delimited pair +// +//////////////////////////////////////////////////////////////////////////////// + +template<char* SD1,class V1,char* SD2,class V2,char* SD3> +class DelimitedPair : public pair<V1,V2> { + public: + // Constructor / destructor methods... + DelimitedPair ( ) : pair<V1,V2>() { } + DelimitedPair ( const V1& v1, const V2& v2 ) : pair<V1,V2>(v1,v2) { } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const DelimitedPair<SD1,V1,SD2,V2,SD3>& rv ) { return os<<SD1<<rv.first<<SD2<<rv.second<<SD3; } + friend String& operator<< ( String& str, const DelimitedPair<SD1,V1,SD2,V2,SD3>& rv ) { return str<<SD1<<rv.first<<SD2<<rv.second<<SD3; } + friend IStream operator>> ( pair<IStream,DelimitedPair<SD1,V1,SD2,V2,SD3>*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + DelimitedPair<SD1,V1,SD2,V2,SD3>& x = *is_x.second; + // Propagate fail... + if ( IStream()==is ) return is; + IStream is1 = (is>>SD1>>x.first>>SD2); + IStream is2 = (is>>SD1>>x.first>>SD2>>x.second>>psDlm); + // Use last delimiter only if not empty (otherwise it will immediately trivially match)... + return ( (SD3[0]=='\0') ? is>>SD1>>x.first>>SD2>>x.second>>psDlm + : is>>SD1>>x.first>>SD2>>x.second>>SD3>>psDlm ); + } +}; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Id -- generalize int to any symbol value +// +//////////////////////////////////////////////////////////////////////////////// + +template<class T> +class Id { + private: + // Data members... + T val; + public: + typedef T IntType; + // Constructor / destructor methods... + Id () { val=0; } + Id (T t) { val=t; } +// Id (int i) { val=i; } + Id (const Id& i) { val=i.val; } + // Specification methods... + Id& operator= (const T& i) { val=i; return *this; } +// Id& operator= (int i) { val=i; return *this; } + Id& operator= (const Id& i) { val=i.val; return *this; } + Id& operator+= (const Id& i) { val+=i.val; return *this; } + Id& operator++ ( ) { ++val; return *this; } + Id& setFirst ( ) { val=0; return *this; } + Id& setFirst (const Id& i) { val=0; return *this; } + Id& setNext ( ) { ++val; return *this; } + Id& setNext (const Id& i) { ++val; return *this; } + // Extraction methods... + //int getHashConst ( const int m ) const { return (sizeof(T)%m); } + //int getHashKey ( const int m ) const { return (((int)*this)%m); } + size_t getHashKey ( ) const { return (size_t)val; } + bool operator== ( const Id& i ) const { return val==i.val; } + bool operator!= ( const Id& i ) const { return val!=i.val; } + bool operator< ( const Id& i ) const { return val<i.val; } + bool operator<= ( const Id& i ) const { return val<=i.val; } + bool operator> ( const Id& i ) const { return val>i.val; } + bool operator>= ( const Id& i ) const { return val>=i.val; } + Id operator+ ( const Id& i ) const { return Id(val+i.val); } + Id operator- ( const Id& i ) const { return Id(val-i.val); } + bool operator== ( const T t ) const { return val==t; } + bool operator< ( const T t ) const { return val<t; } + bool operator<= ( const T t ) const { return val<=t; } + bool operator> ( const T t ) const { return val>t; } + bool operator>= ( const T t ) const { return val>=t; } + Id operator+ ( const T t ) const { return Id(val+t); } + Id operator- ( const T t ) const { return Id(val-t); } +// operator T() const { return val; }; + + T toInt ( ) const { return val; } + Id<T> set (T t) { val=t; return *this; } +// void setInt (int i) { val=i; } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const Id<T>& i ) { return os<<i.val; } + + /* + void write ( FILE* pf, const ReaderContext& rc=ReaderContext() ) const { (sizeof(T)<sizeof(long long)) ? fprintf(pf,"%d",val) : fprintf(pf,"%lld",val); } + */ +} ; + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Safe pointers.. +// +//////////////////////////////////////////////////////////////////////////////// + +template <class R> +class SafePtr { + private: + R* pr; + static const R rDummy; + public: + SafePtr<R> ( ) : pr(NULL) { } + SafePtr<R> ( R& r ) : pr(&r) { } + bool operator== ( const SafePtr<R>& spr ) const { return(pr==spr.pr); } + bool operator!= ( const SafePtr<R>& spr ) const { return(!(pr==spr.pr)); } + //friend void delete ( SafePtr<R>& sp ) { delete sp.pr; } + void del ( ) { delete pr; pr=NULL; } + //R& setRefV ( ) { assert(pr); return *pr; } + R& setRef ( ) { assert(pr); return (pr!=NULL) ? *pr : rDummy; } + const R& getRef ( ) const { return (pr!=NULL) ? *pr : rDummy; } +}; +template <class R> +const R SafePtr<R>::rDummy = R(); + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// Safe static array templates for small arrays... +// +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +class Doub { + private: + T at[2]; + public: + T& set ( int i ) { assert(0<=i&&i<2); return at[i]; } + const T& get ( int i ) const { assert(0<=i&&i<2); return at[i]; } +}; + +template <class T> +class Trip { + private: + T at[3]; + public: + T& set ( int i ) { assert(0<=i&&i<3); return at[i]; } + const T& get ( int i ) const { assert(0<=i&&i<3); return at[i]; } +}; + +template <class T> +class Quad { + private: + T at[4]; + public: + T& set ( int i ) { assert(0<=i&&i<4); return at[i]; } + const T& get ( int i ) const { assert(0<=i&&i<4); return at[i]; } +}; + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// +// SafeArrays -- prevents overflow memory leaks +// +//////////////////////////////////////////////////////////////////////////////// + +template <class X1, class T> +class SafeArray1D { + private: + // Data members; + int xSize; + T* at; + public: + // Constructor / destructor methods... + ~SafeArray1D( ) { delete[] at; } + SafeArray1D ( ) { xSize=0; at=NULL; } + SafeArray1D (int x) { xSize=x; at=new T[x]; } + SafeArray1D (int x, const T& t) { xSize=x; at=new T[x]; + for(int i=0;i<x;++i) at[i]=t; } + SafeArray1D ( const SafeArray1D<X1,T>& sat ) { delete[] at; xSize=sat.xSize; at=new T[xSize]; + for(int i=0;i<xSize;i++) at[i]=sat.at[i]; } + // Specification methods... + SafeArray1D& operator= ( const SafeArray1D<X1,T>& sat ) + { delete[] at; xSize=sat.xSize; at=new T[xSize]; + for(int i=0;i<xSize;i++) at[i]=sat.at[i]; return *this; } + void init ( int x ) { delete[] at; xSize=x; at=new T[x];} + void init ( int x, const T& t ) { delete[] at; xSize=x; at=new T[x]; + for(int i=0;i<x;++i) at[i]=t; } + void reset() { delete[] at; xSize=0; at=NULL; } + T& set ( const X1& x ) { assert(x.toInt()>=0); assert(x.toInt()<xSize); + return at[x.toInt()];} + // Extraction methods... + int getSize ( ) const { return xSize; } + const T& get ( const X1& x ) const { assert(x.toInt()>=0); assert(x.toInt()<xSize); + return at[x.toInt()];} +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class X1, class X2, class T> +class SafeArray2D { + private: + // Data members; + int xSize; + int ySize; + T* at; + public: + // Constructor / destructor methods... + ~SafeArray2D( ) { delete[] at; } + SafeArray2D ( ) { xSize=0; ySize=0; at=NULL; } + SafeArray2D (int x, int y) { xSize=x; ySize=y; at=new T[x*y]; } + SafeArray2D (int x, int y, const T& t) { xSize=x; ySize=y; at=new T[x*y]; + for(int i=0;i<x*y;++i) at[i]=t; } + //SafeArray2D ( const SafeArray2D& a ) { xSize=a.xSize; ySize=a.ySize; + // for(int i=0;i<xSize;i++) for(int j=0;j<ySize;j++) at[i*ySize+j]=a.at[i*ySize+j]; } + // Specification methods... + SafeArray2D& operator= ( const SafeArray2D<X1,X2,T>& sat ) + { delete[] at; xSize=sat.xSize; ySize=sat.ySize; at=new T[xSize*ySize]; + for(int i=0;i<xSize*ySize;i++) at[i]=sat.at[i]; return *this; } + void init ( int x,int y ) { delete[] at; xSize=x; ySize=y; at=new T[x*y];} + void init ( int x,int y,const T& t ) { delete[] at; xSize=x; ySize=y; at=new T[x*y]; + for(int i=0;i<x*y;++i) at[i]=t; } + void reset() { delete[] at; xSize=0; ySize=0; at=NULL; } + T& set ( const X1& x,const X2& y) { assert(at!=NULL); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); assert(y.toInt()<ySize); + return at[x.toInt()*ySize + y.toInt()];} + // Extraction methods... + const T& get (const X1& x,const X2& y) const { assert(at!=NULL); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); +//this assert failed when compile without -DNDEBUG (needed for debugging). Have to figure out why before adding this assert back in +//assert(y.toInt()<ySize); + return at[x.toInt()*ySize + y.toInt()];} + int getxSize( ) const { return xSize; } + int getySize( ) const { return ySize; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class X1, class X2, class X3, class T> +class SafeArray3D { + private: + // Data members; + int xSize; + int ySize; + int zSize; + T* at; + public: + // Constructor / destructor methods... + ~SafeArray3D() { delete[] at; } + SafeArray3D() { xSize=0; ySize=0; zSize=0; at=NULL; } + SafeArray3D(int x,int y,int z) { xSize=x; ySize=y; zSize=z; at=new T[x*y*z];} + SafeArray3D(int x,int y,int z,const T& t){ xSize=x; ySize=y; zSize=z; at=new T[x*y*z]; + for(int i=0;i<x*y*z;++i) at[i]=t; } + // Specification methods... + SafeArray3D& operator= ( const SafeArray3D<X1,X2,X3,T>& sat ) + { delete[] at; xSize=sat.xSize; ySize=sat.ySize; zSize=sat.zSize; + at=new T[xSize*ySize*zSize]; + for(int i=0;i<xSize*ySize*zSize;i++) at[i]=sat.at[i]; return *this; } + void init(int x,int y,int z) { delete[] at; xSize=x; ySize=y; zSize=z; at=new T[x*y*z]; } + void init(int x,int y,int z,const T& t){ delete[] at; xSize=x; ySize=y; zSize=z; at=new T[x*y*z]; + for(int i=0;i<x*y*z;++i) at[i]=t; } + void reset() { delete[] at; xSize=0; ySize=0; zSize=0; at=NULL; } + T& set(const X1& x,const X2& y,const X3& z) + { assert(at!=NULL); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); assert(y.toInt()<ySize); + assert(z.toInt()>=0); assert(z.toInt()<zSize); + return at[(x.toInt()*ySize+y.toInt())*zSize+z.toInt()];} + // Extraction methods... + const T& get(const X1& x,const X2& y,const X3& z) const + { assert(at!=NULL); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); assert(y.toInt()<ySize); + assert(z.toInt()>=0); assert(z.toInt()<zSize); + return at[(x.toInt()*ySize+y.toInt())*zSize+z.toInt()];} +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class X1, class X2, class X3, class X4, class T> +class SafeArray4D { + private: + // Data members; + int wSize; + int xSize; + int ySize; + int zSize; + T* at; + public: + // Constructor / destructor methods... + ~SafeArray4D( ) { delete[] at; } + SafeArray4D ( ) { wSize=0; xSize=0; ySize=0; zSize=0; at=NULL; } + SafeArray4D(int w, int x,int y,int z) + { wSize=w; xSize=x; ySize=y; zSize=z; at=new T[w*x*y*z];} + SafeArray4D(int w, int x,int y,int z,const T& t) + { wSize=w; xSize=x; ySize=y; zSize=z; at=new T[w*x*y*z]; + for(int i=0;i<w*x*y*z;++i) at[i]=t; } + // Specification methods... + SafeArray4D& operator= ( const SafeArray4D<X1,X2,X3,X4,T>& sat ) + { delete[] at; wSize=sat.wSize; xSize=sat.xSize; ySize=sat.ySize; + zSize=sat.zSize; at=new T[wSize*xSize*ySize*zSize]; + for(int i=0;i<wSize*xSize*ySize*zSize;i++) at[i]=sat.at[i]; return *this; } + void init (int w,int x,int y,int z) + { delete[] at; wSize=w; xSize=x; ySize=y; zSize=z; at=new T[w*x*y*z]; } + void init (int w,int x,int y,int z,const T& t) + { delete[] at; wSize=w; xSize=x; ySize=y; zSize=z; at=new T[w*x*y*z]; + for(int i=0;i<w*x*y*z;++i) at[i]=t; } + void reset() { delete[] at; wSize=0; xSize=0; ySize=0; zSize=0; at=NULL; } + T& set(const X1& w,const X2& x,const X3& y,const X4& z) + { assert(at!=NULL); + assert(w.toInt()>=0); assert(w.toInt()<wSize); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); assert(y.toInt()<ySize); + assert(z.toInt()>=0); assert(z.toInt()<zSize); + return at[((w.toInt()*xSize+x.toInt())*ySize+y.toInt())*zSize+z.toInt()];} + // Extraction methods... + const T& get(const X1& w,const X2& x,const X3& y,const X4& z) const + { assert(at!=NULL); + assert(w.toInt()>=0); assert(w.toInt()<wSize); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); assert(y.toInt()<ySize); + assert(z.toInt()>=0); assert(z.toInt()<zSize); + return at[((w.toInt()*xSize+x.toInt())*ySize+y.toInt())*zSize+z.toInt()];} +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class X1, class X2, class X3, class X4, class X5, class T> +class SafeArray5D { + private: + // Data members; + int vSize; + int wSize; + int xSize; + int ySize; + int zSize; + T* at; + public: + // Constructor / destructor methods... + ~SafeArray5D( ) { delete[] at; } + SafeArray5D ( ) { vSize=0; wSize=0; xSize=0; ySize=0; zSize=0; at=NULL; } + SafeArray5D(int v,int w, int x,int y,int z) + { vSize=v; wSize=w; xSize=x; ySize=y; zSize=z; at=new T[v*w*x*y*z];} + SafeArray5D(int v,int w, int x,int y,int z,const T& t) + { vSize=v; wSize=w; xSize=x; ySize=y; zSize=z; at=new T[v*w*x*y*z]; + for(int i=0;i<v*w*x*y*z;++i) at[i]=t; } + // Specification methods... + SafeArray5D& operator= ( const SafeArray5D<X1,X2,X3,X4,X5,T>& sat ) + { delete[] at; vSize=sat.vSize; wSize=sat.wSize; xSize=sat.xSize; + ySize=sat.ySize; zSize=sat.zSize; at=new T[vSize*wSize*xSize*ySize*zSize]; + for(int i=0;i<vSize*wSize*xSize*ySize*zSize;i++) at[i]=sat.at[i]; return *this; } + void init(int v,int w,int x,int y,int z) + { delete[] at; vSize=v; wSize=w; xSize=x; ySize=y; zSize=z; at=new T[v*w*x*y*z]; } + void init(int v,int w,int x,int y,int z,const T& t) + { delete[] at; vSize=v; wSize=w; xSize=x; ySize=y; zSize=z; at=new T[v*w*x*y*z]; + for(int i=0;i<v*w*x*y*z;++i) at[i]=t; } + void reset() { delete[] at; vSize=0; wSize=0; xSize=0; ySize=0; zSize=0; at=NULL; } + T& set(const X1& v,const X2& w,const X3& x,const X4& y,const X5& z) + { assert(at!=NULL); + assert(v.toInt()>=0); assert(v.toInt()<vSize); + assert(w.toInt()>=0); assert(w.toInt()<wSize); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); assert(y.toInt()<ySize); + assert(z.toInt()>=0); assert(z.toInt()<zSize); + return at[(((v.toInt()*wSize+w.toInt())*xSize+x.toInt())*ySize+y.toInt())*zSize+z.toInt()];} + // Extraction methods... + const T& get(const X1& v,const X2& w,const X3& x,const X4& y,const X5& z) const + { assert(at!=NULL); + assert(v.toInt()>=0); assert(v.toInt()<vSize); + assert(w.toInt()>=0); assert(w.toInt()<wSize); + assert(x.toInt()>=0); assert(x.toInt()<xSize); + assert(y.toInt()>=0); assert(y.toInt()<ySize); + assert(z.toInt()>=0); assert(z.toInt()<zSize); + return at[(((v.toInt()*wSize+w.toInt())*xSize+x.toInt())*ySize+y.toInt())*zSize+z.toInt()];} +}; + +//////////////////////////////////////////////////////////////////////////////// + + +#endif //_NL_SAFE_IDS__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-stream.h b/contrib/synlm/hhmm/rvtl/include/nl-stream.h new file mode 100644 index 000000000..8f743e12b --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-stream.h @@ -0,0 +1,240 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_STREAM__ +#define _NL_STREAM__ + +#include "nl-string.h" + + +//////////////////////////////////////////////////////////////////////////////// + +class IStreamSource { + private: + + // Data members... + String sBuff; // string buffer, to allow re-reading in disjunctions + unsigned int iAvail; // index of where string buffer begins; subtract from indices in IStream to get buffer index + FILE* pfSrc; // file pointer + + public: + + IStreamSource ( FILE* pf ) : sBuff(256), iAvail(0), pfSrc(pf) { } + IStreamSource ( const char* psz ) : sBuff(psz), iAvail(0), pfSrc() { } + + char* c_array ( ) { return sBuff.c_array(); } + char get ( unsigned int i ) { assert(i-iAvail<((unsigned int)(-1))/2); // NOTE: PROBLEM IF ANY ISTREAM FALLS BEHIND BY MORE THAN MAXINT CHARACTERS! + int c=0; while(i-iAvail>=sBuff.size()-1){sBuff.add()=((c=getc(pfSrc))==EOF)?'\0':c;} return sBuff.get(i-iAvail); } + char& set ( unsigned int i ) { assert(i-iAvail<((unsigned int)(-1))/2); // NOTE: PROBLEM IF ANY ISTREAM FALLS BEHIND BY MORE THAN MAXINT CHARACTERS! + int c=0; while(i-iAvail>=sBuff.size()-1){sBuff.add()=((c=getc(pfSrc))==EOF)?'\0':c;} return sBuff.set(i-iAvail); } + FILE* getFile ( ) { return pfSrc; } + + // Output method... + friend ostream& operator<< ( ostream& os, const IStreamSource& iss ) { return os<<"'"<<iss.sBuff<<"',"<<iss.iAvail<<","<<iss.pfSrc; } + + void compress ( ) { iAvail+=strlen(sBuff.c_array()); sBuff=String(256); } // NOTE: NOT TESTED WITH FILES EXCEEDING MAXINT CHARACTERS! (BUT SHOULD WORK) +}; + + +//////////////////////////////////////////////////////////////////////////////// + +class IStream { + private: + + IStreamSource* psrc; // pointer to source of stream, which contains buffer and file pointer + unsigned int iIndex; // stream index; subtract psrc->iAvail to get index in psrc->sBuff + + char get ( unsigned int i ) { return psrc->get(i); } + char& set ( unsigned int i ) { return psrc->set(i); } + char* c_array ( unsigned int i=0 ) { return &psrc->set(i); } + + public: + + IStream ( ) : psrc(NULL), iIndex(0) { } + IStream ( IStreamSource& src, unsigned int i=0 ) : psrc(&src), iIndex(i) { } + IStream ( const IStream& is, unsigned int i ) : psrc(is.psrc), iIndex(i) { } + + // Specification methods... + IStream& operator= ( const IStream& is ) { psrc=is.psrc; iIndex=is.iIndex; return *this; } + + // Extraction methods... + bool operator== ( const IStream& is ) const { return (psrc==is.psrc && iIndex==is.iIndex); } + bool operator!= ( const IStream& is ) const { return (psrc!=is.psrc || iIndex!=is.iIndex); } + IStreamSource* getSource() { return psrc; } + + // Output method... + friend ostream& operator<< ( ostream& os, const IStream& is ) { return os<<is.iIndex<<","<<is.psrc<<","<<*is.psrc; } + + // Match single char... + friend IStream operator>> ( IStream is, char& c ) { + // Propagate fail... + if (IStream()==is) return IStream(); + c=is.get(is.iIndex); + return ('\0'==c) ? IStream() : IStream(is,is.iIndex+1); + } + + // Match string delimiter... + friend IStream operator>> ( IStream is, const char* psDlm ) { + // Propagate fail... + if (IStream()==is) { return IStream(); } + unsigned int i; + // Read in characters until fail or match delimiter... + for (i=0; is.get(i+is.iIndex)!='\0' && psDlm[i]!='\0'; i++) + if (is.get(i+is.iIndex)!=psDlm[i]) return IStream(); + return (psDlm[i]=='\0') ? IStream(is,i+is.iIndex) : IStream(); + } + + // Match anything else followed by zero-terminated string delimiter... + template<class X> friend pair<IStream,X*> operator>> ( IStream is, X& x ) { return pair<IStream,X*>(is,&x); } + template<class X> friend IStream operator>> ( pair<IStream,X*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + X& x = *is_x.second; + // Propagate fail... + if (IStream()==is) { return IStream(); } + unsigned int d = strlen(psDlm); + // Read in characters until fail or match delimiter... + for (unsigned int i=is.iIndex; is.get(i)!='\0'; i++) { + // Try to match delimiter ending at current point in stream... + if ( i-is.iIndex+1>=d ) { + unsigned int j; + for (j=0; j<d && is.get(i+1+j-d)==psDlm[j]; j++); + ////cerr<<"-->i="<<i<<",j="<<j<<",d="<<d<<",delim='"<<psDlm<<"',curr='"<<is.get(i)<<"' "<<is<<"\n"; + if (j==d) { + is.set(i+1-d)='\0'; x=X(is.c_array(is.iIndex)); is.set(i+1-d)=psDlm[0]; + return IStream(is,i+1); + } + } + } + return IStream(); + } + + // Match integer followed by zero-terminated string delimiter... + friend IStream operator>> ( pair<IStream,int*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + int& x = *is_x.second; + // Propagate fail... + if (IStream()==is) return IStream(); + unsigned int d = strlen(psDlm); + // Read in characters until fail or match delimiter... + for (unsigned int i=is.iIndex; is.get(i)!='\0'; i++) { + // Try to match delimiter ending at current point in stream... + if ( i+1>=is.iIndex+d ) { + unsigned int j; + for (j=0; j<d && is.get(i+1+j-d)==psDlm[j]; j++); + if (j==d) { + is.set(i+1-d)='\0'; x=atoi(is.c_array(is.iIndex)); is.set(i+1-d)=psDlm[0]; + return IStream(is,i+1); + } + } + } + return IStream(); + } + + // Match unsigned int followed by zero-terminated string delimiter... + friend IStream operator>> ( pair<IStream,unsigned int*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + unsigned int& x = *is_x.second; + // Propagate fail... + if (IStream()==is) return IStream(); + unsigned int d = strlen(psDlm); + // Read in characters until fail or match delimiter... + for (unsigned int i=is.iIndex; is.get(i)!='\0'; i++) { + // Try to match delimiter ending at current point in stream... + if ( i+1>=is.iIndex+d ) { + unsigned int j; + for (j=0; j<d && is.get(i+1+j-d)==psDlm[j]; j++); + if (j==d) { + is.set(i+1-d)='\0'; x=atoi(is.c_array(is.iIndex)); is.set(i+1-d)=psDlm[0]; + return IStream(is,i+1); + } + } + } + return IStream(); + } + + // Match float followed by zero-terminated string delimiter... + friend IStream operator>> ( pair<IStream,float*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + float& x = *is_x.second; + // Propagate fail... + if (IStream()==is) return IStream(); + unsigned int d = strlen(psDlm); + // Read in characters until fail or match delimiter... + for (unsigned int i=is.iIndex; is.get(i)!='\0'; i++) { + // Try to match delimiter ending at current point in stream... + if ( i+1>=is.iIndex+d ) { + unsigned int j; + for (j=0; j<d && is.get(i+1+j-d)==psDlm[j]; j++); + if (j==d) { + is.set(i+1-d)='\0'; x=atof(is.c_array(is.iIndex)); is.set(i+1-d)=psDlm[0]; + return IStream(is,i+1); + } + } + } + return IStream(); + } + + // Match double followed by zero-terminated string delimiter... + friend IStream operator>> ( pair<IStream,double*> is_x, const char* psDlm ) { + IStream& is = is_x.first; + double& x = *is_x.second; + // Propagate fail... + if (IStream()==is) return IStream(); + unsigned int d = strlen(psDlm); + // Read in characters until fail or match delimiter... + for (unsigned int i=is.iIndex; is.get(i)!='\0'; i++) { + // Try to match delimiter ending at current point in stream... + if ( i+1>=is.iIndex+d ) { + unsigned int j; + for (j=0; j<d && is.get(i+1+j-d)==psDlm[j]; j++); + if (j==d) { + is.set(i+1-d)='\0'; x=atof(is.c_array(is.iIndex)); is.set(i+1-d)=psDlm[0]; + return IStream(is,i+1); + } + } + } + return IStream(); + } + + // Match void pointer followed by zero-terminated string delimiter... + friend IStream operator>> ( pair<IStream,void**> is_x, const char* psDlm ) { + IStream& is = is_x.first; + // Propagate fail... + if (IStream()==is) return IStream(); + unsigned int d = strlen(psDlm); + // Read in characters until fail or match delimiter... + for (unsigned int i=is.iIndex; is.get(i)!='\0'; i++) { + // Try to match delimiter ending at current point in stream... + if ( i+1>=is.iIndex+d ) { + unsigned int j; + for (j=0; j<d && is.get(i+1+j-d)==psDlm[j]; j++); + if (j==d) return IStream(is,i+1); + } + } + return IStream(); + } +}; + + +#endif //_NL_STREAM__ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-string.h b/contrib/synlm/hhmm/rvtl/include/nl-string.h new file mode 100644 index 000000000..ce3f68aa3 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-string.h @@ -0,0 +1,213 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_STRING__ +#define _NL_STRING__ + +#include "nl-array.h" + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <iostream> +#include <string> +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// + + +// OBSOLETE! +class StringInput { + private: + char* psIn; + static char* PS_EOS; + public: + + static const StringInput SI_EOS; + static Array<char> strTemp; + + StringInput ( ) : psIn(NULL) { } + StringInput ( char* ps ) : psIn(ps) { } + ////StringInput ( String& s ) : psIn(s.c_array()) { } + + char& operator[] ( int i ) { assert(psIn); return psIn[i]; } + const char operator[] ( int i ) const { assert(psIn); return psIn[i]; } + StringInput& operator++ ( ) { assert(psIn); psIn++; return *this; } + StringInput operator+ ( int i ) { assert(psIn); return StringInput(psIn+i); } + const StringInput operator+ ( int i ) const { assert(psIn); return StringInput(psIn+i); } + bool operator== ( const StringInput& si ) const { return psIn==si.psIn; } + bool operator!= ( const StringInput& si ) const { return psIn!=si.psIn; } + + //operator bool() { return psIn!=NULL; } + + char* c_str() { assert(psIn); return psIn; } + + friend ostream& operator<< ( ostream& os, const StringInput& si ) { return os<<si.psIn; } + + friend StringInput operator>> ( StringInput psIn, const char* psDlm ) { + if (StringInput(NULL)==psIn) return psIn; + int i; + for (i=0; psIn[i]!='\0' && psDlm[i]!='\0'; i++) + if(psIn[i]!=psDlm[i]) return StringInput(NULL); //psIn; + return (psDlm[i]!='\0') ? StringInput(NULL) : (psIn[i]!='\0') ? psIn+i : SI_EOS; + } + + friend pair<StringInput,int*> operator>> ( StringInput ps, int& n ) { return pair<StringInput,int*>(ps,&n); } + friend StringInput operator>> ( pair<StringInput,int*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + ///int i; for(i=0;psIn[i]!='\0';i++) if(psIn[i]==psDlm[i]) return psIn; return psIn+i; + int j=0; + StringInput psIn = delimbuff.first; + if(psDlm[0]=='\0') { *delimbuff.second=atoi(psIn.c_str()); return psIn+strlen(psIn.c_str()); } + for(int i=0;psIn[i]!='\0';i++) { + if(psIn[i]==psDlm[j]) j++; + else j=0; + strTemp[i]=psIn[i]; + if(j==int(strlen(psDlm))) { strTemp[i+1-j]='\0'; *delimbuff.second=atoi(strTemp.c_array()); return psIn+i+1;} + } + return NULL; //psIn; + } + + friend pair<StringInput,unsigned int*> operator>> ( StringInput ps, unsigned int& n ) { return pair<StringInput,unsigned int*>(ps,&n); } + friend StringInput operator>> ( pair<StringInput,unsigned int*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + ///int i; for(i=0;psIn[i]!='\0';i++) if(psIn[i]==psDlm[i]) return psIn; return psIn+i; + int j=0; + StringInput psIn = delimbuff.first; + if(psDlm[0]=='\0') { *delimbuff.second=atoi(psIn.c_str()); return psIn+strlen(psIn.c_str()); } + for(int i=0;psIn[i]!='\0';i++) { + if(psIn[i]==psDlm[j]) j++; + else j=0; + strTemp[i]=psIn[i]; + if(j==int(strlen(psDlm))) { strTemp[i+1-j]='\0'; *delimbuff.second=atoi(strTemp.c_array()); return psIn+i+1;} + } + return NULL; //psIn; + } + + friend pair<StringInput,double*> operator>> ( StringInput ps, double& d ) { return pair<StringInput,double*>(ps,&d); } + friend StringInput operator>> ( pair<StringInput,double*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + ///int i; for(i=0;psIn[i]!='\0';i++) if(psIn[i]==psDlm[i]) return psIn; return psIn+i; + int j=0; + StringInput psIn = delimbuff.first; + if(psDlm[0]=='\0') { *delimbuff.second=atof(psIn.c_str()); return psIn+strlen(psIn.c_str()); } + for(int i=0;psIn[i]!='\0';i++) { + if(psIn[i]==psDlm[j]) j++; + else j=0; + strTemp[i]=psIn[i]; + if(j==int(strlen(psDlm))) { strTemp[i+1-j]='\0'; *delimbuff.second=atof(strTemp.c_array()); return psIn+i+1;} + } + return NULL; //psIn; + } +}; +char* StringInput::PS_EOS = strdup("<EOS>"); +const StringInput StringInput::SI_EOS = StringInput::PS_EOS; +Array<char> StringInput::strTemp ( 100 ); + + +/* +template<class T> +pair<StringInput,T*> operator>> ( StringInput si, T& t ) { + return pair<StringInput,T*>(si,&t); +} +*/ + +/* +template<class T> +class DelimiterBuffer { + //private: + public: + char* psIn; + T& tVar; + public: + DelimiterBuffer<T> ( char* ps, T& t ) : psIn(ps), tVar(t) { } + //char* operator>> ( const char* psDlm ) { + // int j=0; + // for(int i=0;psIn[i]!='\0';i++) { + // if(psIn[i]==psDlm[j]) j++; + // if(j==strlen(psDlm)) { psIn[i+1-j]='\0'; psIn>>tVar; return psIn+i;} + // } + // return psIn; + //} +}; +*/ + + +//////////////////////////////////////////////////////////////////////////////// + +class String : public Array<char> { + private: + static Array<char> strTemp; + public: + // Constructor / destructor methods... + String ( ) : Array<char>(1) { operator[](0)='\0'; } + String ( const String& str ) : Array<char>(str) { } + explicit String ( unsigned int i ) : Array<char>(i) { operator[](0)='\0'; } + explicit String ( const char* ps ) : Array<char>(strlen(ps)) { for(unsigned int i=0; ps[i]!='\0'; i++) Array<char>::add()=ps[i]; Array<char>::add()='\0'; } + // Specification methods... + String& ensureCapacity ( unsigned int i ) { return static_cast<String&> ( Array<char>::ensureCapacity(i) ); } + String& clear ( ) { Array<char>::clear(); operator[](0)='\0'; return *this; } + char& add ( ) { Array<char>::add()='\0'; return set(size()-2); } + friend String& operator<< ( String& str, double d ) { str.addReserve(20); uint32_t i=str.size(); + uint32_t j=sprintf(str.c_array()+i-1,"%g",d); assert(j<20); + str[i+j-1]='\0'; return str; } //str[i+j]=str[i+j]; return str; } + friend String& operator<< ( String& str, const string& s ) { return str<<s.c_str(); } // NOTE: SECOND ARG IS STDLIB STRING! + friend String& operator<< ( String& str, const String& s ) { return str<<s.c_array(); } + friend String& operator<< ( String& str, const char* ps ) { unsigned int j=str.size()-1; unsigned int i=0; + for(i=0;ps[i]!='\0';i++) str[j+i]=ps[i]; + str[j+i]='\0'; return str; } + Array<char*>& split ( Array<char*>& aps, const char* psDlm ) { aps.clear(); char* psT=NULL; + for(int i=0;true;i++) + { char* z=strtok_r( (0==i)?c_array():NULL, psDlm, &psT ); + if (!z) break; + aps[i]=z; } + return aps; } + // Input / output methods... + friend ostream& operator<< ( ostream& os, const String& str ) { return os<<str.c_array(); } + + //friend explicit operator int ( const String& s ) { return atoi(s.c_array()); } + + friend pair<StringInput,String*> operator>> ( const StringInput ps, String& s ) { return pair<StringInput,String*>(ps,&s); } + friend StringInput operator>> ( pair<StringInput,String*> delimbuff, const char* psDlm ) { + if (StringInput(NULL)==delimbuff.first) return delimbuff.first; + ////assert(*delimbuff.second<domain.getSize()); + int j=0; + StringInput psIn = delimbuff.first; + if(psDlm[0]=='\0') { *delimbuff.second=String(psIn.c_str()); return psIn+strlen(psIn.c_str()); } + for(int i=0;psIn[i]!='\0';i++) { + if(psIn[i]==psDlm[j]) j++; + else j=0; + strTemp[i]=psIn[i]; + if(j==int(strlen(psDlm))) { strTemp[i+1-j]='\0'; *delimbuff.second=String(strTemp.c_array()); return psIn+i+1;} + } + return NULL; //psIn; + } +}; +Array<char> String::strTemp ( 100 ); + + + + +#endif //_NL_STRING__ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-stringindex.h b/contrib/synlm/hhmm/rvtl/include/nl-stringindex.h new file mode 100644 index 000000000..22931f081 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-stringindex.h @@ -0,0 +1,61 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _STRING_INDEX__ +#define _STRING_INDEX__ + +#include <cassert> +#include <string> +using std::string; +#include <map> +using std::map; + +class StringIndex{ + + private: + + // Data members... + map <string, int> msi; + map <int, string> mis; + int maxIndex; + + public: + + // Constructor / destructor methods... + StringIndex() { maxIndex=0; } + + // Specification methods... + int addIndex (const char* ps) { assert(ps!=NULL); return addIndex(string(ps)); } + int addIndex (const string& s) { if(msi.end()==msi.find(s)){msi[s]=maxIndex;mis[maxIndex]=s;maxIndex++;} return msi[s]; } + void clear ( ) { msi.clear(); mis.clear(); maxIndex=0; } + // Extraction methods... + int getSize ( ) const { return maxIndex; } + int getIndex (const char* ps) const { assert(ps!=NULL); return getIndex(string(ps)); } + int getIndex (const string& s) const { assert(msi.end()!=msi.find(s)); + return msi.find(s)->second; } + const string& getString (int i) const { assert(mis.end()!=mis.find(i)); + return mis.find(i)->second; } +}; + + +#endif // _STRING_INDEX__ diff --git a/contrib/synlm/hhmm/rvtl/include/nl-tetrahex.h b/contrib/synlm/hhmm/rvtl/include/nl-tetrahex.h new file mode 100644 index 000000000..d77e4f471 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-tetrahex.h @@ -0,0 +1,56 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +/*********************************************** + * nl-tetrahex.h + * a little header with some base conversion stuff + * so that we can represent base 16, 32 or 64 with + * one character. + * 0, 1, 2, ..., 9, A, B, ..., Y, Z, a, b, ..., z, {, | + * 0 1 2 9 10 11 34 35 36 37 61 62 63 + ***********************************************/ + +#include <cassert> + +#ifndef NL_TETRAHEX +#define NL_TETRAHEX + +char intToTetraHex(int i) { + assert(i < 64 && i >= 0); + if(i < 10) return (i + '0'); + if(i > 35) return (i + 'a' - 36); + return (i + 'A' - 10); +} + +int tetraHexToInt(char c) { + if(c < '0') + fprintf(stderr, "Bad c: %d\n",(int)c); + assert(c >= '0'); + if(c - '0' < 10) return (int)(c-'0'); + assert(c >= 'A'); + if(c - 'A' < 26) return (int)(c-'A'+10); + assert(c >= 'a' && c < '}'); + return (int)(c-'a'+36); +} + +#endif diff --git a/contrib/synlm/hhmm/rvtl/include/nl-timer.h b/contrib/synlm/hhmm/rvtl/include/nl-timer.h new file mode 100644 index 000000000..3fa7c5387 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-timer.h @@ -0,0 +1,52 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_TIMER__ +#define _NL_TIMER__ + +#include <sys/time.h> + +class Timer { + private: + struct timeval kept; + struct timeval beg; + public: + Timer ( ) { gettimeofday(&beg,NULL); kept.tv_sec=0; kept.tv_usec=0; } + void start ( ) { gettimeofday(&beg,NULL); } + void pause ( ) { + struct timeval now; gettimeofday(&now,NULL); + kept.tv_sec += now.tv_sec - beg.tv_sec; + kept.tv_usec += (now.tv_usec - beg.tv_usec)%1000000; + kept.tv_sec += int((now.tv_usec - beg.tv_usec)/1000000); + } + double elapsed ( ) { // in milliseconds. + return (double(kept.tv_sec)*1000.0 + double(kept.tv_usec)/1000.0); + //struct timeval end; gettimeofday(&end,NULL); + //double beg_time_s = (double) beg.tv_sec + (double) ((double)beg.tv_usec / 1000000.0); + //double end_time_s = (double) end.tv_sec + (double) ((double)end.tv_usec / 1000000.0); + //return ( (end_time_s - beg_time_s) * 1000.0 ); + } +}; + +#endif //_NL_TIMER__ + diff --git a/contrib/synlm/hhmm/rvtl/include/nl-tree.h b/contrib/synlm/hhmm/rvtl/include/nl-tree.h new file mode 100644 index 000000000..102c5c0b3 --- /dev/null +++ b/contrib/synlm/hhmm/rvtl/include/nl-tree.h @@ -0,0 +1,43 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +template<class B, class T> +class Tree : public T { + private: + // Data members... + SimpleHash<B,Tree<B,T>*> apt; + static const Tree<B,T> tDummy; + public: + // Constructor / destructor methods... + ~Tree ( ) { for(typename SimpleHash<B,Tree<B,T>*>::iterator i=apt.begin(); i!=apt.end(); i++) delete i->second; } + Tree ( ) { } +// Tree ( const Tree<T>& t ) { ptL = (t.ptL) ? new Tree<T>(*t.ptL) : NULL; +// ptR = (t.ptR) ? new Tree<T>(*t.ptR) : NULL; } + // Extraction methods... + const bool isTerm ( ) const { return (apt.empty()); } + const Tree<B,T>& getBranch ( const B& b ) const { return (apt.find(b)!=apt.end()) ? *apt.find(b)->second : tDummy; } + // Specification methods... + Tree<B,T>& setBranch ( const B& b ) { if (apt.find(b)==apt.end()) apt[b]=new Tree<B,T>(); return *apt[b]; } +}; +template<class B, class T> const Tree<B,T> Tree<B,T>::tDummy;// = Tree<B,T>(); + diff --git a/contrib/synlm/hhmm/wsjparse/include/HHMMLangModel-gf.h b/contrib/synlm/hhmm/wsjparse/include/HHMMLangModel-gf.h new file mode 100644 index 000000000..3987d6969 --- /dev/null +++ b/contrib/synlm/hhmm/wsjparse/include/HHMMLangModel-gf.h @@ -0,0 +1,405 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#include "nl-cpt.h" +#include "TextObsVars.h" + +char psX[]=""; +char psSlash[]="/"; +char psComma[]=","; +char psSemi[]=";"; +char psSemiSemi[]=";;"; +char psDashDiamondDash[]="-<>-"; +char psTilde[]="~"; +//char psBar[]="|"; +char psLBrace[]="{"; +char psRBrace[]="}"; +char psLangle[]="<"; +char psRangle[]=">"; +char psLbrack[]="["; +char psRbrack[]="]"; + +const char* BEG_STATE = "-/-;-/-;-/-;-/-;-"; +const char* END_STATE = "eos/eos;-/-;-/-;-/-;-"; + + +//////////////////////////////////////////////////////////////////////////////// +// +// Random Variables +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////// Simple Variables + +//// B: boolean +DiscreteDomain<char> domB; +class B : public DiscreteDomainRV<char,domB> { + public: + B ( ) : DiscreteDomainRV<char,domB> ( ) { } + B ( const char* ps ) : DiscreteDomainRV<char,domB> ( ps ) { } +}; +const B B_0 ("0"); +const B B_1 ("1"); + + +//// D: depth (input only, to HHMM models)... +DiscreteDomain<char> domD; +class D : public DiscreteDomainRV<char,domD> { + public: + D ( ) : DiscreteDomainRV<char,domD> ( ) { } + D ( int i ) : DiscreteDomainRV<char,domD> ( i ) { } + D ( const char* ps ) : DiscreteDomainRV<char,domD> ( ps ) { } +}; +const D D_0("0"); +const D D_1("1"); +const D D_2("2"); +const D D_3("3"); +const D D_4("4"); +const D D_5("5"); + + +//// G: grammatical constituent category +DiscreteDomain<int> domG; +class G : public DiscreteDomainRV<int,domG> { + private: + static SimpleHash<G,B> hIsTerm; + void calcDetModels ( string s ) { + if (!hIsTerm.contains(*this)) { + hIsTerm.set(*this) = (('A'<=s.c_str()[0] && s.c_str()[0]<='Z') || s.find('_')!=string::npos) ? B_0 : B_1; + } + } + public: + G ( ) : DiscreteDomainRV<int,domG> ( ) { } + template<class P> + G ( const G::ArrayIterator<P>& it ) { setVal(it); } + G ( const char* ps ) : DiscreteDomainRV<int,domG> ( ps ) { calcDetModels(ps); } + B isTerm ( ) const { return hIsTerm.get(*this); } + friend pair<StringInput,G*> operator>> ( StringInput si, G& g ) { return pair<StringInput,G*>(si,&g); } + friend StringInput operator>> ( pair<StringInput,G*> si_g, const char* psD ) { + if ( si_g.first == NULL ) return NULL; + StringInput si=si_g.first>>(DiscreteDomainRV<int,domG>&)*si_g.second>>psD; + si_g.second->calcDetModels(si_g.second->getString()); return si; } +}; +SimpleHash<G,B> G::hIsTerm; +const G G_NIL("-"); +const G G_SUB("-"); // G_SUB = G_NIL +const G G_TOP("DISC"); +const G G_RST("REST"); + +typedef G C; + + +//// A: added feature tags for underspec cats +DiscreteDomain<char> domA; +class A : public DiscreteDomainRV<char,domA> { + public: + A ( ) : DiscreteDomainRV<char,domA> ( ) { } + A ( const char* ps ) : DiscreteDomainRV<char,domA> ( ps ) { } +}; +const A A_NIL ("-"); + + +//////////////////////////////////////// Formally Joint Variables Implemented as Simple Variables + +//// Rd: final-state (=FGA)... +DiscreteDomain<int> domRd; +class Rd : public DiscreteDomainRV<int,domRd> { + private: + static SimpleHash<Rd,B> hToB; + static SimpleHash<Rd,G> hToG; + static SimpleHash<G,Rd> hFromG; + void calcDetModels ( string s ) { + if (!hToB.contains(*this)) { + size_t i=s.find(','); + assert(i!=string::npos); + hToB.set(*this) = B(s.substr(0,i).c_str()); + } + if (!hToG.contains(*this)) { + size_t i=s.find(','); + assert(i!=string::npos); + hToG.set(*this) = G(s.substr(i+1).c_str()); + if ( '1'==s[0] ) + hFromG.set(G(s.substr(i+1).c_str())) = *this; + } + } + public: + Rd ( ) : DiscreteDomainRV<int,domRd> ( ) { } + Rd ( const DiscreteDomainRV<int,domRd>& rv ) : DiscreteDomainRV<int,domRd> ( rv ) { } + Rd ( const char* ps ) : DiscreteDomainRV<int,domRd> ( ps ) { calcDetModels(ps); } + Rd ( const G& g ) { *this = hFromG.get(g); } + B getB ( ) const { return hToB.get(*this); } + G getG ( ) const { return hToG.get(*this); } + static Rd getRd ( G g ) { return hFromG.get(g); } + friend pair<StringInput,Rd*> operator>> ( StringInput si, Rd& m ) { return pair<StringInput,Rd*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,Rd*> si_m, const char* psD ) { + if ( si_m.first == NULL ) return NULL; + StringInput si=si_m.first>>(DiscreteDomainRV<int,domRd>&)*si_m.second>>psD; + si_m.second->calcDetModels(si_m.second->getString()); return si; } +}; +SimpleHash<Rd,B> Rd::hToB; +SimpleHash<Rd,G> Rd::hToG; +SimpleHash<G,Rd> Rd::hFromG; +const Rd Rd_INC("0,-"); // BOT +const Rd Rd_SUB("1,-"); // TOP + + +//////////////////////////////////////// Formally and Implementationally Joint Variables + +//// Sd: store element... +class Sd : public DelimitedJoint2DRV<psX,G,psSlash,G,psX> { + typedef DelimitedJoint2DRV<psX,G,psSlash,G,psX> Parent; + public: + Sd ( ) : Parent() { } + template<class P> + Sd ( const Sd::ArrayIterator<P>& it ) { setVal(it); } + Sd ( const G& gia, const G& giw ) : Parent(gia,giw) { } + const G& getAct ( ) const { return first; } + const G& getAwa ( ) const { return second; } + template<class P> class ArrayIterator : public Parent::ArrayIterator<P> { + public: + G::ArrayIterator<P>& setAct ( ) { return Parent::ArrayIterator<P>::first; } + G::ArrayIterator<P>& setAwa ( ) { return Parent::ArrayIterator<P>::second; } + }; + friend pair<StringInput,Sd*> operator>> ( StringInput si, Sd& sd ) { return pair<StringInput,Sd*>(si,&sd); } + friend StringInput operator>> ( pair<StringInput,Sd*> si_sd, const char* psD ) { + if ( si_sd.first == NULL ) return NULL; + StringInput si = si_sd.first>>*(Parent*)si_sd.second>>psD; + return si; + } +}; +const Sd Sd_TOP(G_TOP,G_RST); +const Sd Sd_SUB(G_SUB,G_SUB); + + +//// R: collection of syntactic variables at all depths in each `reduce' phase... +typedef DelimitedJointArrayRV<4,psSemi,Rd> R; + + +//// S: collection of syntactic variables at all depths in each `shift' phase... +class S : public DelimitedJoint2DRV<psX,DelimitedJointArrayRV<4,psSemi,Sd>,psSemi,G,psX> { + public: + operator G() const { return ( ( (second != G_SUB) ? second : + (first.get(3)!=Sd_SUB) ? first.get(3).second : + (first.get(2)!=Sd_SUB) ? first.get(2).second : + (first.get(1)!=Sd_SUB) ? first.get(1).second : + first.get(0).second ) ); } + bool compareFinal ( const S& s ) const { return(*this==s); } +}; + + +//// Y: the set of all (marginalized) reduce and (modeled) shift variables in the HHMM... +class Y : public DelimitedJoint2DRV<psX,R,psDashDiamondDash,S,psX> +{ public: + operator R() const {return first;} + operator S() const {return second;} +}; + + +//////////////////////////////////////////////////////////////////////////////// +// +// Models +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////// "Wrapper" models for individual RVs... + +//// Model of Rd given D and Rd and Sd (from above) and Sd (from previous) +class RdModel { + private: + HidVarCPT4DModel<Rd,D,G,G,LogProb> mRd; // Reduction model: F giv D, G (active cat from prev), G (awaited cat from above) (assume prev awa = reduc) + static const HidVarCPT1DModel<Rd,LogProb> mRd_INC; // Fixed Rd_INC model. + static const HidVarCPT1DModel<Rd,LogProb> mRd_SUB; // Fixed Rd_SUB model. + public: + //static bool Rd_ROOT_OBS; + LogProb setIterProb ( Rd::ArrayIterator<LogProb>& rd, const D& d, const Rd& rdD, const Sd& sdP, const Sd& sdU, bool b1, int& vctr ) const { + LogProb pr; + if ( rdD==Rd_SUB && (sdP.getAwa()==G_SUB) ) { + // _/sub 1,sub (bottom) case... + pr = mRd_SUB.setIterProb(rd,vctr); + } + else if ( rdD==Rd_SUB && sdP.getAwa().isTerm()==B_1 ) { + // _/term 1,sub (middle) case... + pr = mRd.setIterProb(rd,d,sdU.getAwa(),sdP.getAct(),vctr); + if ( vctr<-1 && pr==LogProb() ) cerr<<"\nERROR: no condition F "<<d<<" "<<sdU.getAwa()<<" "<<sdP.getAct()<<"\n\n"; + } + else { + // _/noterm or 0,_ (top) case, otherwise... + pr = mRd_INC.setIterProb(rd,vctr); + } + // Iterate again if result doesn't match root observation... + if ( vctr>=-1 && d==D_1 && b1!=(Rd(rd).getB()==B_1) ) pr=LogProb(); + //cerr<<" Rd "<<d<<" "<<rdD<<" "<<sdP<<" "<<sdU<<" : "<<rd<<" = "<<pr<<" ("<<vctr<<")\n"; + return pr; + } + friend pair<StringInput,RdModel*> operator>> ( StringInput si, RdModel& m ) { return pair<StringInput,RdModel*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,RdModel*> si_m, const char* psD ) { + StringInput si; + return ( (si=si_m.first>>"F ">>si_m.second->mRd>>psD)!=NULL ) ? si : StringInput(NULL); + } +}; +const HidVarCPT1DModel<Rd,LogProb> RdModel::mRd_INC(Rd_INC); +const HidVarCPT1DModel<Rd,LogProb> RdModel::mRd_SUB(Rd_SUB); + + +//// Model of Sd given D and Rd and Rd and Sd(from prev) and Sd(from above) +class SdModel { + public: + HidVarCPT3DModel<G,D,G,LogProb> mGe; // Expansion model of G given D, G (from above) + private: + HidVarCPT4DModel<G,D,G,G,LogProb> mGtaa; // Active transition model of G (active) given D, G (above awa), G (from reduction) + HidVarCPT4DModel<G,D,G,G,LogProb> mGtaw; // Active transition model of G (awaited) given D, G (prev act), G (from reduction) + HidVarCPT4DModel<G,D,G,G,LogProb> mGtww; // Awaited transition model of G (awaited) given D, G (prev awa), G (reduction below) + static const HidVarCPT1DModel<G,LogProb> mG_SUB; // Fixed G_SUB model + static HidVarCPT2DModel<G,G,LogProb> mG_CPY; // Cached G_CPY model -- WARNING: STATIC NON-CONST is not thread safe! + public: + LogProb setIterProb ( Sd::ArrayIterator<LogProb>& sd, const D& d, const Rd& rdD, const Rd& rd, const Sd& sdP, const Sd& sdU, int& vctr ) const { + LogProb pr,p; + if (rdD.getB()!=B_0) { + if (rd.getB()!=B_0 || rd.getG()!=G_NIL) { //if (rd!=Rd_INC) { + if (rd.getB()==B_1) { + if (sdU.getAwa().isTerm()==B_1 || sdU==Sd_SUB) { + // 1,g 1,g (expansion to sub) case: + pr = mG_SUB.setIterProb(sd.setAct(),vctr); + pr *= mG_SUB.setIterProb(sd.setAwa(),vctr); + } + else { + // 1,g 1,g (expansion) case: + pr = p = mGe.setIterProb(sd.setAct() ,d,sdU.getAwa(),vctr); + if ( vctr<-1 && p==LogProb() ) cerr<<"\nERROR: no condition Ge "<<d<<" "<<sdU.getAwa()<<"\n\n"; + if ( !mG_CPY.contains(G(sd.setAct())) ) mG_CPY.setProb(G(sd.setAct()),G(sd.setAct()))=1.0; + pr *= p = mG_CPY.setIterProb(sd.setAwa(),G(sd.setAct()),vctr); + } + } + else { + // 1,_ 0,g (active transition following reduction) case: + pr = p = mGtaa.setIterProb(sd.setAct(),d,sdU.getAwa(),rd.getG(),vctr); + if ( vctr<-1 && p==LogProb() ) cerr<<"\nERROR: no condition Gtaa "<<d<<" "<<sdU.getAwa()<<" "<<rd.getG()<<" ("<<rd<<")\n\n"; + pr *= p = mGtaw.setIterProb(sd.setAwa(),d,G(sd.setAct()),rd.getG(),vctr); + if ( vctr<-1 && p==LogProb() ) cerr<<"\nERROR: no condition Gtaw "<<d<<" "<<G(sd.setAct())<<" "<<rd.getG()<<"\n\n"; + } + } + else { + // 1,g 0,- (awaited transition without reduction) case: + if ( !mG_CPY.contains(sdP.getAct()) ) mG_CPY.setProb(sdP.getAct(),sdP.getAct())=1.0; + pr = p = mG_CPY.setIterProb(sd.setAct(),sdP.getAct(),vctr); + pr *= p = mGtww.setIterProb(sd.setAwa(),d,sdP.getAwa(),rdD.getG(),vctr); + if ( vctr<-1 && p==LogProb() ) cerr<<"\nERROR: no condition Gtww "<<d<<" "<<sdP.getAwa()<<" "<<rdD.getG()<<"\n\n"; + } + } + else { + // 0,- _ (copy) case: + if ( !mG_CPY.contains(sdP.getAct()) ) mG_CPY.setProb(sdP.getAct(),sdP.getAct() )=1.0; + pr = p = mG_CPY.setIterProb(sd.setAct(), sdP.getAct(), vctr); + if ( !mG_CPY.contains(sdP.getAwa()) ) mG_CPY.setProb(sdP.getAwa(),sdP.getAwa())=1.0; + pr *= p = mG_CPY.setIterProb(sd.setAwa(),sdP.getAwa(),vctr); + } + //cerr<<" Sd "<<d<<" "<<rdD<<" "<<rd<<" "<<sdP<<" "<<sdU<<" : "<<sd<<" = "<<pr<<" ("<<vctr<<")\n"; + return pr; + } + friend pair<StringInput,SdModel*> operator>> ( StringInput si, SdModel& m ) { return pair<StringInput,SdModel*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,SdModel*> si_m, const char* psD ) { + StringInput si; + return ( (si=si_m.first>>"Ge " >>si_m.second->mGe >>psD)!=NULL || + (si=si_m.first>>"Gtaa ">>si_m.second->mGtaa>>psD)!=NULL || + (si=si_m.first>>"Gtaw ">>si_m.second->mGtaw>>psD)!=NULL || + (si=si_m.first>>"Gtww ">>si_m.second->mGtww>>psD)!=NULL ) ? si : StringInput(NULL); + } +}; +const HidVarCPT1DModel<G,LogProb> SdModel::mG_SUB(G_SUB); +HidVarCPT2DModel<G,G,LogProb> SdModel::mG_CPY; + + +//////////////////////////////////////// Joint models... + +//////////////////// Reduce phase... + +//// Model of R given S +class RModel : public SingleFactoredModel<RdModel> { + public: + LogProb setIterProb ( R::ArrayIterator<LogProb>& r, const S& sP, bool b1, int& vctr ) const { + const RdModel& mRd = getM1(); + LogProb pr; + pr = mRd.setIterProb ( r.set(4-1), 4, Rd(sP.second) , sP.first.get(4-1), sP.first.get(3-1), b1, vctr ); + pr *= mRd.setIterProb ( r.set(3-1), 3, Rd(r.get(4-1)), sP.first.get(3-1), sP.first.get(2-1), b1, vctr ); + pr *= mRd.setIterProb ( r.set(2-1), 2, Rd(r.get(3-1)), sP.first.get(2-1), sP.first.get(1-1), b1, vctr ); + pr *= mRd.setIterProb ( r.set(1-1), 1, Rd(r.get(2-1)), sP.first.get(1-1), Sd_TOP , b1, vctr ); + return pr; + } +}; + + +//////////////////// Shift phase... + +//// Model of S given R and S +class SModel : public SingleFactoredModel<SdModel> { + private: + static const HidVarCPT1DModel<G,LogProb> mG_SUB; + public: + LogProb setIterProb ( S::ArrayIterator<LogProb>& s, const R::ArrayIterator<LogProb>& r, const S& sP, int& vctr ) const { + const SdModel& mSd = getM1(); + LogProb pr,p; + pr = mSd.setIterProb ( s.first.set(1-1), 1, Rd(r.get(2-1)), Rd(r.get(1-1)), sP.first.get(1-1), Sd_TOP , vctr ); + pr *= mSd.setIterProb ( s.first.set(2-1), 2, Rd(r.get(3-1)), Rd(r.get(2-1)), sP.first.get(2-1), Sd(s.first.set(1-1)), vctr ); + pr *= mSd.setIterProb ( s.first.set(3-1), 3, Rd(r.get(4-1)), Rd(r.get(3-1)), sP.first.get(3-1), Sd(s.first.set(2-1)), vctr ); + pr *= mSd.setIterProb ( s.first.set(4-1), 4, Rd(sP.second) , Rd(r.get(4-1)), sP.first.get(4-1), Sd(s.first.set(3-1)), vctr ); + if ( G(s.first.set(4-1).second)!=G_SUB && + G(s.first.set(4-1).second).isTerm()!=B_1 ) { + pr *= p = mSd.mGe.setIterProb (s.second, 5, G(s.first.set(4-1).second), vctr ); + if ( vctr<-1 && p==LogProb() ) cerr<<"\nERROR: no condition Ge 5 "<<G(s.first.set(4-1).second)<<"\n\n"; + } else { + pr *= mG_SUB.setIterProb ( s.second, vctr ); + } + ////cerr<<" G "<<5<<" "<<G(sd4.second)<<" : "<<g<<" = "<<pr<<" ("<<vctr<<")\n"; + return pr; + } +}; +const HidVarCPT1DModel<G,LogProb> SModel::mG_SUB(G_SUB); + + +//////////////////// Overall... + +//// Model of Y=R,S given S +class YModel : public DoubleFactoredModel<RModel,SModel> { + public: + typedef Y::ArrayIterator<LogProb> IterVal; + S& setTrellDat ( S& s, const Y::ArrayIterator<LogProb>& y ) const { + s.setVal(y.second); + return s; + } + R setBackDat ( const Y::ArrayIterator<LogProb>& y ) const { + R r; + for(int i=0;i<4;i++) + r.set(i)=Rd(y.first.get(i)); + return r; + } + LogProb setIterProb ( Y::ArrayIterator<LogProb>& y, const S& sP, const X& x, bool b1, int& vctr ) const { + const RModel& mR = getM1(); + const SModel& mS = getM2(); + LogProb pr; + pr = mR.setIterProb ( y.first, sP, b1, vctr ); + if ( LogProb()==pr ) return pr; + pr *= mS.setIterProb ( y.second, y.first, sP, vctr ); + return pr; + } + void update ( ) const { } +}; diff --git a/contrib/synlm/hhmm/wsjparse/include/TextObsModel.h b/contrib/synlm/hhmm/wsjparse/include/TextObsModel.h new file mode 100644 index 000000000..5e277e857 --- /dev/null +++ b/contrib/synlm/hhmm/wsjparse/include/TextObsModel.h @@ -0,0 +1,173 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#include "nl-cpt.h" +#include "nl-dtree.h" +#include "TextObsVars.h" + +//////////////////////////////////////////////////////////////////////////////// +// +// Models +// +//////////////////////////////////////////////////////////////////////////////// + + +/* WS: DO *NOT* BRING THIS BACK AND DELETE MY X MODEL !!!!!!!!!!!!!!! + +//// Preterminal (POS) given constituent category models... +typedef HidVarCPT2DModel<P,C,LogProb> PgivCModel; + + +//// Generative model of word given tag... +class WModel { + private: + TrainableDTree2DModel<P,W,LogProb> modPgivWdt; + + RandAccCPT2DModel<P,W,LogProb> modPgivWs; + RandAccCPT1DModel<P,LogProb> modP; + RandAccCPT1DModel<W,LogProb> modW; + + public: + //LogProb getProb ( const W& w, const HidVarCPT1DModel<P,LogProb>::IterVal& p ) const { + LogProb getProb ( const W& w, const P::ArrayIterator<LogProb>& p ) const { + assert(modP.getProb(p)!=LogProb()); + LogProb pr = ( ( modW.contains(w) ? modPgivWs.getProb(p,w) : modPgivWdt.getProb(p,w) ) + * LogProb(-1000) / modP.getProb(p) ); + if(!modW.contains(w)){ + cerr<<" w: "<<w<<" p: "<<p<<" modPgivWdt.getProb(p,w) : "<<modPgivWdt.getProb(p,w) <<endl; + } + return pr; + } + void writeFields ( FILE* pf, string sPref ) { modPgivWdt.writeFields(pf,sPref); } + friend pair<StringInput,WModel*> operator>> ( StringInput si, WModel& m ) { return pair<StringInput,WModel*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,WModel*> delimbuff, const char* psD ) { + StringInput si; + return ( (si=delimbuff.first>>"W " >>delimbuff.second->modW >>psD)!=NULL || + (si=delimbuff.first>>"Pw " >>delimbuff.second->modPgivWs >>psD)!=NULL || + (si=delimbuff.first>>"PwDT ">>delimbuff.second->modPgivWdt>>psD)!=NULL || + (si=delimbuff.first>>"P " >>delimbuff.second->modP >>psD)!=NULL ) ? si : StringInput(NULL); + } +}; + + +//// Wrapper class for model +class OModel { + + private: + + PgivCModel modPgivC; + WModel modWgivP; + + public: + + class DistribModeledWgivC { + private: + SimpleHash<C,Prob> hcpCache; + W wW; + public: + DistribModeledWgivC& set ( const W& w, const OModel& m ) { wW=w; m.calcProb(*this,w); return *this; } + void clear ( ) { hcpCache.clear(); } + Prob& setProb ( const C& c ) { return hcpCache.set(c); } + LogProb getProb ( const C& c ) const { return LogProb(hcpCache.get(c)); } + W getW ( ) const { return wW; } + }; + + typedef DistribModeledWgivC RandVarType; + + + + void calcProb ( OModel::RandVarType& o, const W& w ) const { + o.clear(); + for ( PgivCModel::const_iterator iter = modPgivC.begin(); iter!=modPgivC.end(); iter++ ) { + C c = iter->first.getX1(); + P::ArrayIterator<LogProb> p; + int aCtr=-1; + //for ( bool bp=modPgivC.setIterProb(p,c,aCtr); bp; bp=modPgivC.setIterProb(p,c,aCtr=0) ) { + for (LogProb pr=modPgivC.setIterProb(p,c,aCtr); pr!=LogProb(); pr = modPgivC.setIterProb(p,c,aCtr=0) ){ + o.setProb(c) += modPgivC.getProb(p,c).toProb() * modWgivP.getProb(w,p).toProb(); + } + + } + } + + LogProb getProb ( const OModel::RandVarType& o, const C& c ) const { return o.getProb(c); } + + friend pair<StringInput,OModel*> operator>> ( StringInput si, OModel& m ) { return pair<StringInput,OModel*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,OModel*> delimbuff, const char* psD ) { + StringInput si; + return ( (si=delimbuff.first>>"Pc ">>delimbuff.second->modPgivC>>psD)!=NULL || + (si=delimbuff.first>> delimbuff.second->modWgivP>>psD)!=NULL ) ? si : StringInput(NULL); + } + + void writeFields ( FILE* pf, string sPref ) { modWgivP.writeFields(pf,sPref); } +}; +*/ + + +class XModel { + + private: + + // Data members... + HidVarCPT2DModel<P,C,Prob> modPgivC; + TrainableDTree2DModel<P,X,Prob> modPgivXdt; + RandAccCPT2DModel<P,W,Prob> modPgivW; + RandAccCPT1DModel<P,Prob> modP; + RandAccCPT1DModel<W,Prob> modW; + + public: + + typedef X RandVarType; + + bool unknown( const X& x ) const { + W w(x); + return w == W_UNK; + } + + // Extraction methods... + LogProb getProb ( const X& x, const C& c ) const { + Prob pr=0.0; + W w(x); + if ( w != W_UNK ) { + for ( P::ArrayIterator<Prob> p = modPgivC.begin(c); !p.end(); ++p ) { + pr += modPgivC.getProb(p,c) * modPgivW.getProb(p,w) * modW.getProb(w) / modP.getProb(p); + } + } else { + for ( P::ArrayIterator<Prob> p = modPgivC.begin(c); !p.end(); ++p ) { + pr += modPgivC.getProb(p,c) * modPgivW.getProb(p,w) * modW.getProb(w) / modP.getProb(p); + } + } + return LogProb(pr); + } + + // Input/output methods... + friend pair<StringInput,XModel*> operator>> ( StringInput si, XModel& m ) { return pair<StringInput,XModel*>(si,&m); } + friend StringInput operator>> ( pair<StringInput,XModel*> si_m, const char* psD ) { + StringInput si; + return ( (si=si_m.first>>"Pc " >>si_m.second->modPgivC >>psD)!=NULL || + (si=si_m.first>>"W " >>si_m.second->modW >>psD)!=NULL || + (si=si_m.first>>"Pw " >>si_m.second->modPgivW >>psD)!=NULL || + (si=si_m.first>>"PwDT ">>si_m.second->modPgivXdt>>psD)!=NULL || + (si=si_m.first>>"P " >>si_m.second->modP >>psD)!=NULL ) ? si : StringInput(NULL); + } +}; diff --git a/contrib/synlm/hhmm/wsjparse/include/TextObsVars.h b/contrib/synlm/hhmm/wsjparse/include/TextObsVars.h new file mode 100644 index 000000000..c32a6cbc1 --- /dev/null +++ b/contrib/synlm/hhmm/wsjparse/include/TextObsVars.h @@ -0,0 +1,110 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see <http://www.gnu.org/licenses/>. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _TEXT_OBS_VARS_ +#define _TEXT_OBS_VARS_ + +#include "nl-randvar.h" + +//////////////////////////////////////////////////////////////////////////////// +// +// Random Variables +// +//////////////////////////////////////////////////////////////////////////////// + +//// P: part of speech category... +DiscreteDomain<short> domainP; +typedef DiscreteDomainRV<short,domainP> P; + +//// L: letter... +DiscreteDomain<char> domainLt; +typedef DiscreteDomainRV<char,domainLt> Lt; + +//// X: observed word (array of letters, arranged last to first)... +//typedef StaticSafeArray<5,Lt> X; +DiscreteDomain<int> domainX; +class X : public DiscreteDomainRV<int,domainX>, public StaticSafeArray<5,Lt> { + public: + X ( ) { } + X ( const char* ps ) : DiscreteDomainRV<int,domainX>(ps) { + char psTemp[2]="-"; int n=strlen(ps); + for(int i=0;i<5;i++) { + psTemp[0]=(i<n)?ps[n-i-1]:'_'; + //cout<<"!!!!!!!!!!!!!!!!!!"<<psTemp<<endl; + StaticSafeArray<5,Lt>::set(i)=Lt(psTemp); + } + } + friend pair<StringInput,X*> operator>> ( const StringInput ps, X& rv ) { return pair<StringInput,X*>(ps,&rv); } + friend StringInput operator>> ( pair<StringInput,X*> si_x, const char* psDlm ) { + if(si_x.first==NULL)return si_x.first; String s; StringInput si=si_x.first>>s>>psDlm; *si_x.second=s.c_array(); return si; } + bool operator== ( const X& x ) const { return DiscreteDomainRV<int,domainX>::operator==(x); } + size_t getHashKey ( ) const { return DiscreteDomainRV<int,domainX>::getHashKey(); } +}; + +//// W: subset of words with reliable statistics for POS model +DiscreteDomain<int> domW; +class W : public DiscreteDomainRV<int,domW> { + private: + static SimpleHash<X,W> hXtoW; + void calcDetModels ( string s ) { if (!hXtoW.contains(X(s.c_str()))) hXtoW.set(X(s.c_str())) = *this; } + public: + static const W W_UNK; + W ( ) : DiscreteDomainRV<int,domW> ( ) { } + W ( const DiscreteDomainRV<int,domW>& rv ) : DiscreteDomainRV<int,domW>(rv) { } + W ( const char* ps ) : DiscreteDomainRV<int,domW> ( ps ) { calcDetModels(ps); } + //C ( string s ) : DiscreteDomainRV<int,domC> ( s ) { calcDetModels(s); } + W ( const X& x ) { *this = (hXtoW.contains(x)) ? hXtoW.get(x) : W_UNK; } + friend pair<StringInput,W*> operator>> ( StringInput si, W& x ) { return pair<StringInput,W*>(si,&x); } + friend StringInput operator>> ( pair<StringInput,W*> si_x, const char* psD ) { + if ( si_x.first == NULL ) return NULL; + StringInput si=si_x.first>>(DiscreteDomainRV<int,domW>&)*si_x.second>>psD; + si_x.second->calcDetModels(si_x.second->getString()); return si; } +}; +SimpleHash<X,W> W::hXtoW; +const W W::W_UNK ("unk"); +const W W_UNK = W::W_UNK; + +//// H: subset of words within threshhold of head words used for clustering +DiscreteDomain<int> domH; +class H : public DiscreteDomainRV<int,domH> { + private: + static SimpleHash<X,H> hXtoH; + void calcDetModels ( string s ) { if (!hXtoH.contains(X(s.c_str()))) hXtoH.set(X(s.c_str())) = *this; } + public: + static const H H_UNK; + H ( ) : DiscreteDomainRV<int,domH> ( ) { } + H ( const DiscreteDomainRV<int,domH>& rv ) : DiscreteDomainRV<int,domH>(rv) { } + H ( const char* ps ) : DiscreteDomainRV<int,domH> ( ps ) { calcDetModels(ps); } + //C ( string s ) : DiscreteDomainRV<int,domC> ( s ) { calcDetModels(s); } + H ( const X& x ) { *this = (hXtoH.contains(x)) ? hXtoH.get(x) : H_UNK; } + friend pair<StringInput,H*> operator>> ( StringInput si, H& x ) { return pair<StringInput,H*>(si,&x); } + friend StringInput operator>> ( pair<StringInput,H*> si_x, const char* psD ) { + if ( si_x.first == NULL ) return NULL; + StringInput si=si_x.first>>(DiscreteDomainRV<int,domH>&)*si_x.second>>psD; + si_x.second->calcDetModels(si_x.second->getString()); return si; } +}; +SimpleHash<X,H> H::hXtoH; +const H H::H_UNK ("unk"); +const H H_UNK = H::H_UNK; + +#endif //_TEXT_OBS_VARS_ diff --git a/contrib/web/bin/daemon.pl b/contrib/web/bin/daemon.pl new file mode 100644 index 000000000..8e6a08739 --- /dev/null +++ b/contrib/web/bin/daemon.pl @@ -0,0 +1,59 @@ +#!/usr/bin/perl -w +use warnings; +use strict; +$|++; + +# file: daemon.pl + +# Herve Saint-Amand +# Universitaet des Saarlandes +# Tue May 13 19:45:31 2008 + +# This script starts Moses to run in the background, so that it can be used by +# the CGI script. It spawns the Moses process, then binds itself to listen on +# some port, and when it gets a connection, reads it line by line, feeds those +# to Moses, and sends back the translation. + +# You can either run one instance of this on your Web server, or, if you have +# the hardware setup for it, run several instances of this, then configure +# translate.cgi to connect to these. + +#------------------------------------------------------------------------------ +# includes + +use IO::Socket::INET; +use IPC::Open2; + +#------------------------------------------------------------------------------ +# constants, global vars, config + +my $MOSES = '/local/herves/moses/moses-irst'; +my $MOSES_INI = '/local/herves/moses/fr-en/moses.ini.2'; + +die "usage: daemon.pl <hostname> <port>" unless (@ARGV == 2); +my $LISTEN_HOST = shift; +my $LISTEN_PORT = shift; + +#------------------------------------------------------------------------------ +# main + +# spawn moses +my ($MOSES_IN, $MOSES_OUT); +my $pid = open2 ($MOSES_OUT, $MOSES_IN, $MOSES, '-f', $MOSES_INI, '-t'); + +# open server socket +my $server_sock = new IO::Socket::INET + (LocalAddr => $LISTEN_HOST, LocalPort => $LISTEN_PORT, Listen => 1) + || die "Can't bind server socket"; + +while (my $client_sock = $server_sock->accept) { + while (my $line = <$client_sock>) { + print $MOSES_IN $line; + $MOSES_IN->flush (); + print $client_sock scalar <$MOSES_OUT>; + } + + $client_sock->close (); +} + +#------------------------------------------------------------------------------ diff --git a/contrib/web/bin/detokenizer.perl b/contrib/web/bin/detokenizer.perl new file mode 100644 index 000000000..4d53aeddd --- /dev/null +++ b/contrib/web/bin/detokenizer.perl @@ -0,0 +1,114 @@ +#!/usr/bin/perl -w + +# Sample De-Tokenizer +# written by Josh Schroeder, based on code by Philipp Koehn +# +# This script is now outdated, use scripts/tokenizer/detokenizer.perl instead + +# This added by Herve Saint-Amand for compatibility with translate.cgi +$|++; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +use strict; + +my $language = "en"; +my $QUIET = 0; +my $HELP = 0; + +while (@ARGV) { + $_ = shift; + /^-l$/ && ($language = shift, next); + /^-q$/ && ($QUIET = 1, next); + /^-h$/ && ($HELP = 1, next); +} + +if ($HELP) { + print "Usage ./detokenizer.perl (-l [en|de|...]) < tokenizedfile > detokenizedfile\n"; + exit; +} +if (!$QUIET) { + print STDERR "Detokenizer Version 1.0\n"; + print STDERR "Language: $language\n"; +} + +while(<STDIN>) { + if (/^<.+>$/ || /^\s*$/) { + #don't try to detokenize XML/HTML tag lines + print $_; + } + else { + print &detokenize($_); + } +} + +sub detokenize { + my($text) = @_; + chomp($text); + $text = " $text "; + + my $word; + my $i; + my @words = split(/ /,$text); + $text = ""; + my %quoteCount = ("\'"=>0,"\""=>0); + my $prependSpace = " "; + for ($i=0;$i<(scalar(@words));$i++) { + if ($words[$i] =~ /^[\p{IsSc}\(\[\{\¿\¡]+$/) { + #perform right shift on currency and other random punctuation items + $text = $text.$prependSpace.$words[$i]; + $prependSpace = ""; + } elsif ($words[$i] =~ /^[\,\.\?\!\:\;\\\%\}\]\)]+$/){ + #perform left shift on punctuation items + $text=$text.$words[$i]; + $prependSpace = " "; + } elsif (($language eq "en") && ($i>0) && ($words[$i] =~ /^[\'][\p{IsAlpha}]/) && ($words[$i-1] =~ /[\p{IsAlnum}]$/)) { + #left-shift the contraction for English + $text=$text.$words[$i]; + $prependSpace = " "; + } elsif (($language eq "fr") && ($i<(scalar(@words)-2)) && ($words[$i] =~ /[\p{IsAlpha}][\']$/) && ($words[$i+1] =~ /^[\p{IsAlpha}]/)) { + #right-shift the contraction for French + $text = $text.$prependSpace.$words[$i]; + $prependSpace = ""; + } elsif ($words[$i] =~ /^[\'\"]+$/) { + #combine punctuation smartly + if (($quoteCount{$words[$i]} % 2) eq 0) { + if(($language eq "en") && ($words[$i] eq "'") && ($i > 0) && ($words[$i-1] =~ /[s]$/)) { + #single quote for posesssives ending in s... "The Jones' house" + #left shift + $text=$text.$words[$i]; + $prependSpace = " "; + } else { + #right shift + $text = $text.$prependSpace.$words[$i]; + $prependSpace = ""; + $quoteCount{$words[$i]} = $quoteCount{$words[$i]} + 1; + + } + } else { + #left shift + $text=$text.$words[$i]; + $prependSpace = " "; + $quoteCount{$words[$i]} = $quoteCount{$words[$i]} + 1; + + } + + } else { + $text=$text.$prependSpace.$words[$i]; + $prependSpace = " "; + } + } + + # clean up spaces at head and tail of each line as well as any double-spacing + $text =~ s/ +/ /g; + $text =~ s/\n /\n/g; + $text =~ s/ \n/\n/g; + $text =~ s/^ //g; + $text =~ s/ $//g; + + #add trailing break + $text .= "\n" unless $text =~ /\n$/; + + return $text; +} + diff --git a/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.de b/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.de new file mode 100644 index 000000000..c24f2080f --- /dev/null +++ b/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.de @@ -0,0 +1,325 @@ +#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
+#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
+
+#any single upper case letter followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
+#usually upper case letters are initials in a name
+#no german words end in single lower-case letters, so we throw those in too.
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+
+
+#Roman Numerals. A dot after one of these is not a sentence break in German.
+I
+II
+III
+IV
+V
+VI
+VII
+VIII
+IX
+X
+XI
+XII
+XIII
+XIV
+XV
+XVI
+XVII
+XVIII
+XIX
+XX
+i
+ii
+iii
+iv
+v
+vi
+vii
+viii
+ix
+x
+xi
+xii
+xiii
+xiv
+xv
+xvi
+xvii
+xviii
+xix
+xx
+
+#Titles and Honorifics
+Adj
+Adm
+Adv
+Asst
+Bart
+Bldg
+Brig
+Bros
+Capt
+Cmdr
+Col
+Comdr
+Con
+Corp
+Cpl
+DR
+Dr
+Ens
+Gen
+Gov
+Hon
+Hosp
+Insp
+Lt
+MM
+MR
+MRS
+MS
+Maj
+Messrs
+Mlle
+Mme
+Mr
+Mrs
+Ms
+Msgr
+Op
+Ord
+Pfc
+Ph
+Prof
+Pvt
+Rep
+Reps
+Res
+Rev
+Rt
+Sen
+Sens
+Sfc
+Sgt
+Sr
+St
+Supt
+Surg
+
+#Misc symbols
+Mio
+Mrd
+bzw
+v
+vs
+usw
+d.h
+z.B
+u.a
+etc
+Mrd
+MwSt
+ggf
+d.J
+D.h
+m.E
+vgl
+I.F
+z.T
+sogen
+ff
+u.E
+g.U
+g.g.A
+c.-Ã -d
+Buchst
+u.s.w
+sog
+u.ä
+Std
+evtl
+Zt
+Chr
+u.U
+o.ä
+Ltd
+b.A
+z.Zt
+spp
+sen
+SA
+k.o
+jun
+i.H.v
+dgl
+dergl
+Co
+zzt
+usf
+s.p.a
+Dkr
+Corp
+bzgl
+BSE
+
+#Number indicators
+# add #NUMERIC_ONLY# after the word if it should ONLY be non-breaking when a 0-9 digit follows it
+No
+Nos
+Art
+Nr
+pp
+ca
+Ca
+
+#Ordinals are done with . in German - "1." = "1st" in English
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
diff --git a/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.el b/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.el new file mode 100644 index 000000000..7bb3d490a --- /dev/null +++ b/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.el @@ -0,0 +1,2 @@ +# for now, just include the Greek equivalent of "Mr."
+κ
diff --git a/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.en b/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.en new file mode 100644 index 000000000..7e7a8ce2e --- /dev/null +++ b/contrib/web/bin/nonbreaking_prefixes/nonbreaking_prefix.en @@ -0,0 +1,107 @@ +#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
+#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
+
+#any single upper case letter followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
+#usually upper case letters are initials in a name
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+
+#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
+Adj
+Adm
+Adv
+Asst
+Bart
+Bldg
+Brig
+Bros
+Capt
+Cmdr
+Col
+Comdr
+Con
+Corp
+Cpl
+DR
+Dr
+Drs
+Ens
+Gen
+Gov
+Hon
+Hr
+Hosp
+Insp
+Lt
+MM
+MR
+MRS
+MS
+Maj
+Messrs
+Mlle
+Mme
+Mr
+Mrs
+Ms
+Msgr
+Op
+Ord
+Pfc
+Ph
+Prof
+Pvt
+Rep
+Reps
+Res
+Rev
+Rt
+Sen
+Sens
+Sfc
+Sgt
+Sr
+St
+Supt
+Surg
+
+#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall into this category - it sometimes ends a sentence)
+v
+vs
+i.e
+rev
+e.g
+
+#Numbers only. These should only induce breaks when followed by a numeric sequence
+# add NUMERIC_ONLY after the word for this function
+#This case is mostly for the english "No." which can either be a sentence of its own, or
+#if followed by a number, a non-breaking prefix
+No #NUMERIC_ONLY#
+Nos
+Art #NUMERIC_ONLY#
+Nr
+pp #NUMERIC_ONLY#
diff --git a/contrib/web/bin/start-daemon-cluster.pl b/contrib/web/bin/start-daemon-cluster.pl new file mode 100644 index 000000000..7bf87842c --- /dev/null +++ b/contrib/web/bin/start-daemon-cluster.pl @@ -0,0 +1,35 @@ +#!/usr/bin/perl -w +use warnings; +use strict; +$|++; + +# file: start-daemon-cluster.pl + +# Herve Saint-Amand +# Universitaet des Saarlandes +# Thu May 15 08:22:13 2008 + +# Utility to start/stop the daemon processes on the 16 cluster machines. +# Config in here should match that given in translate.cgi (hostnames, ports) + +#------------------------------------------------------------------------------ + +my $stop = @ARGV && $ARGV[0] eq '-s'; + +foreach my $i (qw/01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16/) { + my $host = "cluster-$i"; + my $port = "90$i"; + + my @cmd = $stop ? + ('ssh', $host, 'killall', '-q', 'daemon.pl') + : + ('ssh', + '-L', "$port:localhost:$port", + $host, + '/local/herves/moses/daemon.pl', 'localhost', $port); + + print "@cmd\n"; + exec @cmd unless fork (); +} + +#------------------------------------------------------------------------------ diff --git a/contrib/web/bin/tokenizer.perl b/contrib/web/bin/tokenizer.perl new file mode 100644 index 000000000..4e1f5ac05 --- /dev/null +++ b/contrib/web/bin/tokenizer.perl @@ -0,0 +1,170 @@ +#!/usr/bin/perl -w + +# Sample Tokenizer +# written by Josh Schroeder, based on code by Philipp Koehn + +# This added by Herve Saint-Amand for compatibility with translate.cgi +$|++; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +use FindBin qw($Bin); +use strict; +#use Time::HiRes; + +my $mydir = "$Bin/nonbreaking_prefixes"; + +my %NONBREAKING_PREFIX = (); +my $language = "en"; +my $QUIET = 0; +my $HELP = 0; + +#my $start = [ Time::HiRes::gettimeofday( ) ]; + +while (@ARGV) { + $_ = shift; + /^-l$/ && ($language = shift, next); + /^-q$/ && ($QUIET = 1, next); + /^-h$/ && ($HELP = 1, next); +} + +if ($HELP) { + print "Usage ./tokenizer.perl (-l [en|de|...]) < textfile > tokenizedfile\n"; + exit; +} +if (!$QUIET) { + print STDERR "Tokenizer v3\n"; + print STDERR "Language: $language\n"; +} + +load_prefixes($language,\%NONBREAKING_PREFIX); + +if (scalar(%NONBREAKING_PREFIX) eq 0){ + print STDERR "Warning: No known abbreviations for language '$language'\n"; +} + +while(<STDIN>) { + if (/^<.+>$/ || /^\s*$/) { + #don't try to tokenize XML/HTML tag lines + print $_; + } + else { + print &tokenize($_); + } +} + +#my $duration = Time::HiRes::tv_interval( $start ); +#print STDERR ("EXECUTION TIME: ".$duration."\n"); + + +sub tokenize { + my($text) = @_; + chomp($text); + $text = " $text "; + + # seperate out all "other" special characters + $text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-])/ $1 /g; + + #multi-dots stay together + $text =~ s/\.([\.]+)/ DOTMULTI$1/g; + while($text =~ /DOTMULTI\./) { + $text =~ s/DOTMULTI\.([^\.])/DOTDOTMULTI $1/g; + $text =~ s/DOTMULTI\./DOTDOTMULTI/g; + } + + # seperate out "," except if within numbers (5,300) + $text =~ s/([^\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; + # separate , pre and post number + $text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; + $text =~ s/([^\p{IsN}])[,]([\p{IsN}])/$1 , $2/g; + + # turn `into ' + $text =~ s/\`/\'/g; + + #turn '' into " + $text =~ s/\'\'/ \" /g; + + if ($language eq "en") { + #split contractions right + $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([^\p{IsAlpha}\p{IsN}])[']([\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1 '$2/g; + #special case for "1990's" + $text =~ s/([\p{IsN}])[']([s])/$1 '$2/g; + } elsif (($language eq "fr") or ($language eq "it")) { + #split contractions left + $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([^\p{IsAlpha}])[']([\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; + $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1' $2/g; + } else { + $text =~ s/\'/ \' /g; + } + + #word token method + my @words = split(/\s/,$text); + $text = ""; + for (my $i=0;$i<(scalar(@words));$i++) { + my $word = $words[$i]; + if ( $word =~ /^(\S+)\.$/) { + my $pre = $1; + if (($pre =~ /\./ && $pre =~ /\p{IsAlpha}/) || ($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==1) || ($i<scalar(@words)-1 && ($words[$i+1] =~ /^[\p{IsLower}]/))) { + #no change + } elsif (($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==2) && ($i<scalar(@words)-1 && ($words[$i+1] =~ /^[0-9]+/))) { + #no change + } else { + $word = $pre." ."; + } + } + $text .= $word." "; + } + + # clean up extraneous spaces + $text =~ s/ +/ /g; + $text =~ s/^ //g; + $text =~ s/ $//g; + + #restore multi-dots + while($text =~ /DOTDOTMULTI/) { + $text =~ s/DOTDOTMULTI/DOTMULTI./g; + } + $text =~ s/DOTMULTI/./g; + + #ensure final line break + $text .= "\n" unless $text =~ /\n$/; + + return $text; +} + +sub load_prefixes { + my ($language, $PREFIX_REF) = @_; + + my $prefixfile = "$mydir/nonbreaking_prefix.$language"; + + #default back to English if we don't have a language-specific prefix file + if (!(-e $prefixfile)) { + $prefixfile = "$mydir/nonbreaking_prefix.en"; + print STDERR "WARNING: No known abbreviations for language '$language', attempting fall-back to English version...\n"; + die ("ERROR: No abbreviations files found in $mydir\n") unless (-e $prefixfile); + } + + if (-e "$prefixfile") { + open(PREFIX, "<:utf8", "$prefixfile"); + while (<PREFIX>) { + my $item = $_; + chomp($item); + if (($item) && (substr($item,0,1) ne "#")) { + if ($item =~ /(.*)[\s]+(\#NUMERIC_ONLY\#)/) { + $PREFIX_REF->{$1} = 2; + } else { + $PREFIX_REF->{$item} = 1; + } + } + } + close(PREFIX); + } + +} + diff --git a/contrib/web/index.cgi b/contrib/web/index.cgi new file mode 100644 index 000000000..dcc20b0aa --- /dev/null +++ b/contrib/web/index.cgi @@ -0,0 +1,103 @@ +#!/usr/bin/perl -Tw
+use warnings;
+use strict;
+$|++;
+
+# file: index.cgi
+
+# Herve Saint-Amand
+# Universitaet des Saarlandes
+# Tue May 13 20:09:25 2008
+
+# When we do Moses translation of Web pages there's a little tool frame at the
+# top of the screen. This script takes care of printing the frameset and that
+# top frame. It also invokes the main script with the proper URL param.
+
+# You're most probably not interested in the code that's in here. Have a look
+# a translate.cgi instead. That's where the meat is.
+
+#------------------------------------------------------------------------------
+# includes
+
+use CGI;
+use CGI::Carp qw/fatalsToBrowser/;
+
+use URI::Escape;
+
+#------------------------------------------------------------------------------
+# constants, global vars
+
+(my $SELF_URL = $ENV{QUERY_STRING}) =~ s![^/]*$!!;
+
+my $TRANSLATE_CGI = 'translate.cgi';
+
+#------------------------------------------------------------------------------
+# read CGI params
+
+my %params = %{{
+ url => undef,
+ frame => undef,
+}};
+
+my $cgi = new CGI;
+
+foreach my $p (keys %params) {
+ $params{$p} = $cgi->param ($p)
+ if (defined $cgi->param ($p));
+}
+
+#------------------------------------------------------------------------------
+# print out
+
+print "Content-Type: text/html\n\n";
+
+print
+ "<html>\n" .
+ " <head>\n" .
+ " <title>$params{url} -- Moses translation</title>\n" .
+ " <style>\n" .
+ " p, a, b, body {\n" .
+ " font-family: verdana;\n" .
+ " font-size: 9pt;\n" .
+ " }\n" .
+ " </style>\n" .
+ " </head>\n";
+
+if (!$params{url}) {
+ print
+ " <body bgcolor='#ffFFfF'>\n" .
+ " <h1>Moses Web Interface</h1>\n" .
+ " <form method='GET' action='$SELF_URL'>\n" .
+ " <input name='url' size='60'>\n" .
+ " <input type='submit' value='Translate'>\n" .
+ " </form>\n" .
+ " </body>\n";
+
+} else {
+
+ # check that we have a URL and it's absolute
+ $params{url} = "http://$params{url}"
+ unless ($params{url} =~ m!^[a-z]+://!);
+ my $URL = uri_escape ($params{url});
+
+ if (!$params{frame}) {
+ print
+ " <frameset rows='30,*' border='1' frameborder='1'>\n" .
+ " <frame src='$SELF_URL?frame=top&url=$URL'>\n" .
+ " <frame src='$TRANSLATE_CGI?url=$URL'>\n" .
+ " </frameset>\n";
+
+ } else {
+ print
+ " <script src='index.js'></script>\n" .
+ " <body bgcolor='#ccCCcC' onload='startCount()'>\n" .
+ " <b>Moses translation of\n" .
+ " <a href='$params{url}' target='_top'>$params{url}</a></b>\n" .
+ " <span id='status'></span>\n" .
+ " </body>\n";
+ }
+}
+
+print "</html>\n";
+
+#------------------------------------------------------------------------------
diff --git a/contrib/web/index.js b/contrib/web/index.js new file mode 100644 index 000000000..1d917ce73 --- /dev/null +++ b/contrib/web/index.js @@ -0,0 +1,26 @@ +// file: frameset.js
+
+// Herve Saint-Amand
+// Universitaet des Saarlandes
+// Tue May 13 10:00:42 2008
+
+//-----------------------------------------------------------------------------
+
+function startCount () {
+ var startTime = (new Date()).getTime ();
+ var element = document.getElementById ('status');
+
+ function step () {
+ var secs = parseInt (((new Date()).getTime() - startTime) / 1000);
+ var status = "(elapsed: " + secs + " seconds)";
+ if (top.numSentences != null)
+ status += " (" + top.numSentences + " segments)";
+ else
+ setTimeout (step, 1000);
+ element.innerHTML = status;
+ }
+
+ step ();
+}
+
+//-----------------------------------------------------------------------------
diff --git a/contrib/web/lib/RemoteProcess.pm b/contrib/web/lib/RemoteProcess.pm new file mode 100644 index 000000000..1a414cd77 --- /dev/null +++ b/contrib/web/lib/RemoteProcess.pm @@ -0,0 +1,51 @@ +# file: RemoteProcess.pm
+
+# Herve Saint-Amand
+# Universitaet des Saarlandes
+# Thu May 15 08:30:19 2008
+
+#------------------------------------------------------------------------------
+# includes
+
+package RemoteProcess;
+our @ISA = qw/Subprocess/;
+
+use warnings;
+use strict;
+
+use IO::Socket::INET;
+
+use Subprocess;
+
+#------------------------------------------------------------------------------
+# constructor
+
+sub new {
+ my ($class, $host, $port) = @_;
+
+ my $self = new Subprocess;
+ $self->{host} = $host;
+ $self->{port} = $port;
+ $self->{sock} = undef;
+
+ bless $self, $class;
+}
+
+#------------------------------------------------------------------------------
+# should have the same interface as Subprocess.pm
+
+sub start {
+ my ($self) = @_;
+
+ $self->{sock} = new IO::Socket::INET (%{{
+ PeerAddr => $self->{host},
+ PeerPort => $self->{port},
+ }}) || die "Can't connect to $self->{host}:$self->{port}";
+
+ $self->{child_in} = $self->{child_out} = $self->{sock};
+}
+
+#------------------------------------------------------------------------------
+
+1;
+
diff --git a/contrib/web/lib/Subprocess.pm b/contrib/web/lib/Subprocess.pm new file mode 100644 index 000000000..adc7a853a --- /dev/null +++ b/contrib/web/lib/Subprocess.pm @@ -0,0 +1,61 @@ +# file: Subprocess.pm
+
+# Herve Saint-Amand
+# Universitaet des Saarlandes
+# Wed May 14 09:55:46 2008
+
+# NOTE that to use this with Philipp Koehn's tokenizer.perl I had to modify
+# that script to autoflush its streams, by adding a '$|++' to it
+
+#------------------------------------------------------------------------------
+# includes
+
+package Subprocess;
+
+use warnings;
+use strict;
+
+use Encode;
+use IPC::Open2;
+
+#------------------------------------------------------------------------------
+# constructor
+
+sub new {
+ my ($class, @cmd) = @_;
+ bless {
+ cmd => \@cmd,
+ num_done => 0,
+ child_in => undef,
+ child_out => undef,
+ }, $class;
+}
+
+#------------------------------------------------------------------------------
+
+sub start {
+ my ($self) = @_;
+ open2 ($self->{child_out}, $self->{child_in}, @{$self->{cmd}});
+}
+
+sub do_line {
+ my ($self, $line) = @_;
+ my ($in, $out) = ($self->{child_in}, $self->{child_out});
+
+ $line =~ s/\s+/ /g;
+ print $in encode ('UTF-8', $line), "\n";
+ $in->flush ();
+
+ my $ret = decode ('UTF-8', scalar <$out>);
+ chomp $ret;
+
+ $self->{num_done}++;
+ return $ret;
+}
+
+sub num_done { shift->{num_done} }
+
+#------------------------------------------------------------------------------
+
+1;
+
diff --git a/contrib/web/translate.cgi b/contrib/web/translate.cgi new file mode 100644 index 000000000..ece7109fc --- /dev/null +++ b/contrib/web/translate.cgi @@ -0,0 +1,806 @@ +#!/usr/bin/perl -Tw
+use warnings;
+use strict;
+$|++;
+
+# file: translate.cgi
+
+# Herve Saint-Amand
+# saintamh [o] yahoo, com
+# Universitaet des Saarlandes
+# Mon May 12 14:10:54 2008
+
+# This CGI script takes a web page URL as a parameter, fetches that page,
+# translates it using the Moses decoder, and displays the translated version
+# to the user, similarily to how Google or BabelFish translate web pages.
+
+# I don't think I've ever written anything with such a high comment/code ratio,
+# so hopefully it should be understandable. Just read top to bottom.
+
+
+# TODO:
+#
+# - if the document contains <a name='anchor'></a> it will be lost
+# - don't insert spaces everywhere around soft tags
+# - charset autodetection would be nice, but it's not trivial
+
+#------------------------------------------------------------------------------
+# includes
+
+use CGI;
+use CGI::Carp qw/fatalsToBrowser/;
+
+# we use the 2nd perl thread API. I think this means you need perl 5.6 or
+# higher, compiled with thread support
+use threads;
+use threads::shared;
+
+use Encode;
+use HTML::Entities;
+use HTML::Parser;
+use LWP::UserAgent;
+use URI;
+use URI::Escape;
+
+use lib 'lib';
+use RemoteProcess;
+use Subprocess;
+
+#------------------------------------------------------------------------------
+# constants, config
+
+# In order to run this script, you must first start Moses as a sort of daemon
+# process that accepts connections on some INET port, reads the sentences sent
+# to it one line at a time and returns translations. The daemon.pl script that
+# comes with this script does just that -- starts an instance of Moses and
+# 'plugs' it to the net so it can be used from other machines or just other
+# processes on the same machine.
+#
+# This list here indicates where to find these instances of Moses. May be
+# localhost, or may be separate machines.
+#
+# On the current UniSaar setup we use SSH tunneling to connect to other hosts,
+# so from this script's POV they're all localhost. These ports are actually
+# forwarded to other machines. There wouldn't be much point in running 16
+# instances of Moses on the same machine.
+
+my @MOSES_ADDRESSES = map "localhost:90$_",
+ qw/01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16/;
+
+
+# The tokenizer tries to adapt its rules depending on the language it's dealing
+# with, so we indicate that here.
+
+my $INPUT_LANG = 'fr';
+my $OUTPUT_LANG = 'en';
+
+
+# In order to tokenize and detokenize strings in a way that stays consistent
+# with how it is done in the rest of the Moses system, we use the scripts that
+# come with Moses as external processes. These are the commands we must run to
+# start them.
+
+my @TOKENIZER_CMD = ('./bin/tokenizer.perl', '-l', $INPUT_LANG);
+my @DETOKENIZER_CMD = ('./bin/detokenizer.perl', '-l', $OUTPUT_LANG);
+
+
+# We call 'soft tags' HTML tags whose presence is tolerated inside
+# sentences. All other tags are assumed to be sentence-breakers and will be
+# used to chop up documents into independent sentences. These few, however, are
+# allowed within sentences.
+
+my %SOFT_TAGS = map {$_ => 1} qw/a b i u em font blink tt acronym/;
+
+
+# We call 'verbatim tags' HTML tags whose entire data is to be left untouched
+# and reprinted as-is. These also happen to be tags whose content is typically
+# not printed by the browser.
+
+my %VERBATIM_TAGS = map {$_ => 1} qw/script style/;
+
+
+# Some HTML tags have attributes that contain URLs. Since we'll be displaying
+# the page on another server than its usual source server, relative paths will
+# be broken, so we need to make all URLs absolute. These are the attributes
+# that will be so modified.
+
+my %URL_ATTRS = %{{
+ a => 'href',
+ img => 'src',
+ form => 'action',
+ link => 'href',
+ script => 'src',
+}};
+
+
+# Some HTML tags have attributes that can contain free text that is displayed
+# to the user. Data in attributes is not usually translated, but these should
+# be.
+#
+# Note that for implementation reasons these will always be treated as hard,
+# sentence-splitting tags. This could be changed but would require a
+# substantial re-write of this script.
+
+my %TEXT_ATTR = %{{ input => [qw/value/], img => [qw/alt title/], }};
+
+
+# Sentence splitting within a paragraph or block of text is done after
+# tokenizing. Tokens matched by this regex will be considered to end a
+# sentence, and hence be used in splitting the text into sentences.
+
+my $RE_EOS_TOKEN = qr/^(?:\.+|[\?!:;])$/;
+
+# This regex also matches sentence-ending tokens, but tokens matched by this
+# one will not be included in the sentence itself. Tokens matched by the
+# previous regex will be sent to Moses as part of the end of the sentence.
+# Tokens matches by this one will never be sent to Moses. Which is why the pipe
+# symbol, which Moses doesn't seem to like, must be in here.
+
+my $RE_SPLIT_TOKEN = qr!^[\|\-]+$!;
+
+#------------------------------------------------------------------------------
+# global vars
+
+# In cleaner code there wouldn't be global variables, but it simplified things
+# to put these here. Eventually I wouldn't mind removing this section.
+
+
+# This array is very central to the way this script works. The document will be
+# chopped up into a list of 'segments'. Each segment is either some HTML code
+# and whitespace which we don't translate or manipulate in any way, or a bit of
+# text to be translated. It's as if we highlighted in the HTML source the bits
+# of text that needed translation, and make each stripe of highlighter, and
+# each length of text between them, a segment.
+#
+# Segments that are untouched HTML are simply strings. If the whole document
+# contained no translatable text, this array would only contain strings.
+#
+# Segments that contain text to be translated are represented as arrayrefs. The
+# first element of that arrayref is the text to be translated, with any soft
+# tags within it replaced by placeholders of the type MOSESOPENTAG4. The
+# remaining elements contain the necessary info to reinsert these tags. The
+# placeholders are numbered, and the i-th placeholder corresponds to the
+# (i+1)-th element in the arrayref (element 0 being the text). That element is
+# itself an array ref, whose first element is the tag name and second element
+# is a hashref of attributes.
+#
+# So this document:
+#
+# <p>This is <a href="somewhere">a link</a> but it's not <b>bold</b></p>
+#
+# would be represented by this @segments array:
+#
+# 0: "<p>"
+# 1: [ 0: "This is MOSESOPENTAG0 a link MOSESCLOSETAG0 but it's not" .
+# " MOSESOPENTAG1 bold MOSESCLOSETAG1"
+# 1: [ "a", { href => "somewhere" } ]
+# 2: [ "b", {} ] ]
+# 2: "</p>"
+#
+# Finally, there's one hack to be mentioned: text in %TEXT_ATTR attributes
+# (defined above) also goes into a segment of its own. Since this text does
+# not contain tags, and to signal that the code for the popup containing
+# source text should not be inserted around this text, we replace the tag
+# information by the "__NOPOPUP__" string. So this document:
+#
+# <img src="blah" alt="This describes the image">
+#
+# would correspond to this @segments array:
+#
+# 0: "<img src=\"blah\" alt=\""
+# 1: [ "This describes the image", "__NOPOPUP__" ]
+# 2: "\">"
+#
+# This is a horrible hack. Yes.
+
+my @segments;
+
+
+# Finally, since this script is run in 'tainted' mode (-T switch) for basic
+# security reasons, and we'll be launching subprocesses, so we need to make
+# sure the PATH is clean otherwise Perl will refuse to do the system() calls.
+
+$ENV{PATH} = '';
+
+#------------------------------------------------------------------------------
+# Fetch the source page
+
+# get value of URL param, make sure it's absolute
+my $url = CGI->new->param ('url');
+die "No URL?" unless $url;
+$url = "http://$url" unless ($url =~ m!^[a-z]+://!);
+
+# configure Web client
+my $lwp = new LWP::UserAgent (%{{
+ agent => $ENV{HTTP_USER_AGENT} || 'Mozilla/5.0',
+ timeout => 5,
+}});
+
+# fetch the web page we want to translate
+my $res = $lwp->get ($url);
+die "Couldn't fetch page: " . $res->status_line unless $res->is_success;
+my $html = $res->decoded_content;
+
+# Find the page's base url. It may be different than the URL given to us as
+# parameter if for instance that URL redirects to a different one, or the
+# document contains a <base> tag.
+my $base_url = $res->base;
+
+# Decode entities, except some basics because it confuses our parsing. We need
+# this because Moses won't understand the entities. It sometimes introduces
+# minor display bugs, though. TODO: decode only alphanumerical entities?
+$html =~ s/&((?:lt|gt);?)/&$1/g;
+$html = decode_entities ($html);
+
+# Start printing HTML page
+print "Content-Type: text/html; charset=UTF-8\n\n";
+
+#------------------------------------------------------------------------------
+# Parser stack and state management
+
+# We're going to use a callback parser to parse the HTML file. As we walk the
+# HTML tree we maintain a buffer containing the current block if text to be
+# translated. These state variables contain that. The buffer is repeatedly
+# emptied and its contents pushed onto @segments.
+#
+# We also remove 'soft' tags from the text as we append it to the buffer,
+# replace them with placeholders, and save info about the tags we set aside in
+# @buf_tag_index. @buf_tag_stack keeps track of 'currently open' tags, so that
+# we can match closing tags to their opening tags.
+
+my $buf_text_has_content = 0;
+my $buf_text = '';
+my @buf_tag_index;
+my @buf_tag_stack;
+
+my $in_verbatim = 0;
+
+
+# This is called when we find soft tags within text to be translated. Arguments
+# are the tag name, a hash of tag attributes, and a boolean telling us whether
+# it's an opening or closing tag.
+#
+# We perform lookups in the above state variables, save the tag info in them if
+# necessary, and return a string which is the placeholder to replace that tag.
+
+sub make_placeholder {
+ my ($tag, $attr, $closing) = @_;
+ my $placeholder = '';
+
+ if ($closing) {
+
+ # try to match closing tags with their opening sibling
+ foreach my $i (reverse 0 .. $#buf_tag_stack) {
+ if ($buf_tag_stack[$i][0] eq $tag) {
+ $placeholder = 'MOSESCLOSETAG' . $buf_tag_stack[$i][1];
+ splice (@buf_tag_stack, $i, 1);
+ last;
+ }
+ }
+
+ # lone closing tags are added to the index but not the stack
+ if (!$placeholder) {
+ push (@buf_tag_index, [ $tag, $attr ]);
+ $placeholder = 'MOSESCLOSETAG' . $#buf_tag_index;
+ }
+
+ } else {
+ # opening tags are added to the index and the stack
+ push (@buf_tag_index, [ $tag, $attr ]);
+ push (@buf_tag_stack, [ $tag, $#buf_tag_index ]);
+ $placeholder = 'MOSESOPENTAG' . $#buf_tag_index;
+ }
+
+ return $placeholder;
+}
+
+
+# When we hit a hard tag, we call this to save any current text segment we have
+# to the @segments array.
+
+sub flush_buf_text {
+ if ($buf_text_has_content || @buf_tag_index) {
+ push (@segments, [ $buf_text, @buf_tag_index ] );
+ } else {
+ push (@segments, $buf_text);
+ }
+
+ $buf_text = '';
+ @buf_tag_index = ();
+ @buf_tag_stack = ();
+ $buf_text_has_content = 0;
+}
+
+#------------------------------------------------------------------------------
+# HTML parser
+
+# Parser callback for when we hit an opening or closing tag
+sub start_and_end_h {
+ my ($tag, $attr, $closing) = @_;
+
+ # keep track of whether we're in a verbatim segment
+ $in_verbatim = $closing ? 0 : $tag
+ if $VERBATIM_TAGS{$tag};
+
+ # make links absolute
+ my $url_attr = $URL_ATTRS{$tag};
+ &make_link_absolute ($tag, $attr, $url_attr)
+ if ($url_attr && $attr->{$url_attr});
+
+ # textual attributes require some trickery - FIXME this duplicates some of
+ # &print_tag
+ if ($TEXT_ATTR{$tag}) {
+ &flush_buf_text ();
+ my $found = 0;
+
+ # there's an example of how this works in the comments that precede the
+ # declaration of @segments, above
+ foreach my $text_attr (@{$TEXT_ATTR{$tag}}) {
+ if ($attr->{$text_attr}) {
+ push (@segments, ($found ? '"' : "<$tag") . " $text_attr=\"");
+ push (@segments, [ $attr->{$text_attr}, '__NOPOPUP__' ]);
+ delete $attr->{$text_attr};
+ $found = 1;
+ }
+ }
+
+ if ($found) {
+ my $self_close = delete $attr->{'/'} ? 1 : 0;
+ push (@segments, "\"" . join ('', map {
+ (my $v = $attr->{$_}) =~ s/\"/&\#34;/g;
+ " $_=\"$v\"";
+ } keys %{$attr}) . ($self_close ? ' /' : '') . '>');
+ } else {
+ push (@segments, &print_tag ($tag, $attr, $closing));
+ }
+
+ # if the tag is soft we buffer it, if it's hard we flush the buffer out
+ } elsif ($SOFT_TAGS{$tag}) {
+ my $placeholder = &make_placeholder ($tag, $attr, $closing);
+ $buf_text .= ' ' . $placeholder . ' ';
+ } else {
+ &flush_buf_text ();
+ push (@segments, &print_tag ($tag, $attr, $closing));
+ }
+
+ # add a <base> tag at the beginning of the <head> (do we need this?)
+ push (@segments, "<base src='$base_url'>\n")
+ if ($tag eq 'head' && !$closing);
+}
+
+
+# parser callback for text segments
+sub text_h {
+ my ($text) = @_;
+
+ if ($in_verbatim) {
+ # when in verbatim mode (in <script> or <style> tags), everything just
+ # gets reprinted as-is
+
+ # .. except this
+ $text =~ s/\@import\s+\"([^\n\"]+)\"/
+ '@import "' . URI->new_abs($1, $base_url)->as_string . '"';
+ /ge;
+
+ push (@segments, $text);
+
+ } else {
+ # otherwise add the text to the sentence buffer
+ $buf_text .= $text;
+ $buf_text_has_content ||= ($text =~ /\p{IsAlnum}/);
+ }
+}
+
+sub rest_h {
+ my ($text) = @_;
+ &flush_buf_text ();
+ push (@segments, $text);
+}
+
+
+my $parser = HTML::Parser->new (%{{
+ start_h => [\&start_and_end_h, 'tagname, attr' ],
+ text_h => [\&text_h, 'text' ],
+ declaration_h => [\&rest_h, 'text' ],
+ comment_h => [\&rest_h, 'text' ],
+
+ end_h => [sub {
+ &start_and_end_h (shift, {}, 1);
+ }, 'tagname' ],
+}});
+
+# parse it into @segments
+$parser->parse ($html);
+undef $parser;
+
+#------------------------------------------------------------------------------
+# Run translation threads
+
+# We have now parsed the who document to the @segments array. Now we start
+# the actual translation process.
+#
+# We start one thread for each Moses host defined in the configuration above.
+# All threads will then race to translate text segments, working down the
+# @segments array. They also print segments as soon as a sequence of segments
+# is done.
+
+
+# These are the variables that are shared between threads and used for
+# synchronisation.
+
+my @input :shared = map { ref $_ ? $_->[0] : undef } @segments;
+my @output :shared = map { ref $_ ? undef : $_ } @segments;
+my $next_job_i :shared = 0;
+my $num_printed :shared = 0;
+
+
+# This sub will be run in parallel by the threads
+my $thread_body = sub {
+ my ($moses_i) = @_;
+
+ # each thread uses it's own tokenizer and detokenizer subprocess
+ # (FIXME -- isn't this hugely inefficient?)
+ my $tokenizer = new Subprocess (@TOKENIZER_CMD);
+ my $detokenizer = new Subprocess (@DETOKENIZER_CMD);
+ $tokenizer->start;
+ $detokenizer->start;
+
+ # each thread also connects to its own Moses server
+ my ($host, $port) = split /:/, $MOSES_ADDRESSES[$moses_i];
+ my $moses = new RemoteProcess ($host, $port) ||
+ die "Can't connect to '$host:$port'";
+ $moses->start;
+
+ for (;;) {
+
+ # Snatch the next unassigned job from the queue
+ my $job_i;
+ { lock $next_job_i; $job_i = $next_job_i++; }
+ last if ($job_i > $#input);
+
+ # If it's a text job, translate it, otherwise just don't do anything
+ $output[$job_i] = &translate_text_with_placeholders
+ ($input[$job_i], $moses, $tokenizer, $detokenizer)
+ if (!defined $output[$job_i]);
+
+ # Print out any sequential block of done jobs
+ lock $num_printed;
+ while ($num_printed < @input && defined $output[$num_printed]) {
+ my $print;
+
+ if (ref $segments[$num_printed]) {
+
+ # replace placeholders by the original tags
+ my @buf_tag_index = @{$segments[$num_printed]};
+ shift @buf_tag_index;
+ $print = &replace_placeholders_by_tags
+ ($output[$num_printed], @buf_tag_index);
+
+ # wrap in code to popup the original text onmouseover
+ if ($buf_tag_index[0] ne '__NOPOPUP__') {
+ $print = &add_original_text_popup
+ ($input[$num_printed], $print);
+ } else {
+ $print =~ s/\"/&\#34;/g;
+ }
+
+ } else {
+ # HTML segments are just printed as-is
+ $print = $segments[$num_printed];
+ }
+
+ print encode ('UTF-8', $print);
+ $num_printed++;
+ }
+ }
+};
+
+if (@MOSES_ADDRESSES == 1) {
+
+ # If there's only one instance of Moses, there's no point in forking a
+ # single thread and waiting for it to complete, so we just run the thread
+ # code directly in the main thread
+ $thread_body->(0);
+
+} else {
+
+ # Start all threads and wait for them all to finish
+ my @threads = map {
+ threads->create ($thread_body, $_);
+ } (0 .. $#MOSES_ADDRESSES);
+ $_->join foreach @threads;
+
+}
+
+#------------------------------------------------------------------------------
+# Translation subs
+
+
+# This sub is called bt the translation thread for each text segment. The
+# arguments are the input text and pointers to the various external processes
+# needed for processing.
+#
+# At this stage the input text contains placeholders that look like
+# "MOSESOPENTAG2". We don't need to know which tag they stand for, but we do
+# need to set them aside, translate the remaining plain text, and reinsert them
+# at the correct place in the translation.
+
+sub translate_text_with_placeholders {
+ my ($input_text, $moses, $tokenizer, $detokenizer) = @_;
+ my $traced_text = '';
+
+ # Start by tokenizing the text, with placeholders still in it. The
+ # placeholders are designed to be interpreted as individual tokens by the
+ # tokenizer.
+ my @tokens = split /\s+/, $tokenizer->do_line ($input_text);
+
+ # remove placeholders, and for each remaining token, make a list of the
+ # tags that cover it
+ @tokens = ('START', @tokens, 'END');
+ my @tags_over_token = &_extract_placeholders (\@tokens);
+ @tokens = @tokens[1 .. $#tokens-1];
+
+ # translate sentence by sentence
+ my $token_base_i = 0;
+ while (@tokens > 0) {
+
+ # take a string of tokens up to the next sentence-ending token
+ my (@s_tokens, $split_token);
+ while (@tokens > 0) {
+ if ($tokens[0] =~ $RE_EOS_TOKEN) {
+ push (@s_tokens, shift @tokens);
+ last;
+ } elsif ($tokens[0] =~ $RE_SPLIT_TOKEN) {
+ $split_token = shift @tokens;
+ last;
+ } else {
+ push (@s_tokens, shift @tokens);
+ }
+ }
+
+ # Join together tokens into a plain text string. This is now ready to
+ # be shipped to Moses: all tags and placeholders have been removed,
+ # and it's a single sentence. We also lowercase as needed, and make
+ # a note of whether we did.
+ my $s_input_text = join (' ', @s_tokens);
+ my $was_ucfirst =
+ ($s_input_text =~ s/^(\p{IsUpper})(?=\p{IsLower})/lc $1;/e);
+ my $was_allcaps =
+ ($s_input_text =~ s/^([\p{IsUpper}\P{IsAlpha}]+)$/lc $1;/e);
+
+ # Translate the plain text sentence
+ # my $s_traced_text = &_translate_text_pig_latin ($s_input_text);
+ my $s_traced_text = &_translate_text_moses ($s_input_text, $moses);
+
+ # Early post-translation formatting fixes
+ $s_traced_text .= " $split_token" if $split_token;
+ $s_traced_text = ucfirst $s_traced_text if $was_ucfirst;
+ $s_traced_text = uc $s_traced_text if $was_allcaps;
+
+ # Update trace numbers to fit in the Grand Scheme of Things
+ $s_traced_text =~ s{\s*\|(\d+)-(\d+)\|}{
+ ' |' . ($1+$token_base_i) . '-' . ($2+$token_base_i) . '| ';
+ }ge;
+ $token_base_i += @s_tokens + ($split_token ? 1 : 0);
+
+ $traced_text .= $s_traced_text . ' ';
+ }
+
+ # Apply to every segment in the traced output the union of all tags
+ # that covered tokens in the corresponding source segment
+ my $output_text = &_reinsert_placeholders
+ ($traced_text, @tags_over_token);
+
+ # Try to remove spaces inserted by the tokenizer
+ $output_text = $detokenizer->do_line ($output_text);
+
+ return $output_text;
+}
+
+
+# This sub takes an array of tokens, some of which are placeholders for
+# formatting tags. Some of these tag placeholders are for opening tags, some
+# are for closing tags. What we do here is we remove all these placeholders
+# from the list and create an index of which of the remaining tokens are
+# covered by which tags (by which we mean, inside their scope).
+#
+# So for instance if the given array looks like this:
+#
+# [ "MOSESOPENTAG0", "MOSESOPENTAG1", "Hello", "MOSESCLOSETAG1",
+# "MOSESOPENTAG2", "world", "MOSESCLOSETAG2", "MOSESCLOSETAG0" ]
+#
+# after executing this sub the array will look like this:
+#
+# [ "Hello", "world" ]
+#
+# and the @tags_over_token index will have been created, containing this:
+#
+# [ [0,1], [0,2] ]
+#
+# indicating that the first token ("Hello") is covered by tags 0 and 1, and
+# that the 2nd token ("world") is covered by tags 0 and 2.
+
+sub _extract_placeholders {
+ my ($tokens) = @_;
+ my @tags_over_token = ([]);
+
+ while (@tags_over_token <= @$tokens) {
+ my $i = $#tags_over_token;
+ my @t = @{$tags_over_token[$i]};
+
+ if ($tokens->[$i] =~ /^MOSESOPENTAG(\d+)$/) {
+ $tags_over_token[$i] = [@t, $1];
+ splice (@{$tokens}, $i, 1);
+ } elsif ($tokens->[$i] =~ /^MOSESCLOSETAG(\d+)$/) {
+ if (grep $_ == $1, @t) {
+ $tags_over_token[$i] = [grep $_ != $1, @t];
+ } else {
+ push (@{$tags_over_token[$_]}, $1) foreach (0 .. $i-1);
+ }
+ splice (@{$tokens}, $i, 1);
+ } else {
+ push (@tags_over_token, [@t]);
+ }
+ }
+
+ return @tags_over_token;
+}
+
+
+# This sub does pretty much the opposite of the preceding sub. It gets as
+# argument the traced text output by Moses and the @tags_over_token array
+# computed by the preceding sub. The traced text looks something like this:
+#
+# Hallo |0-0| Welt |1-1|
+#
+# For each such segment which is between two traces, we will want to apply
+# to it the union of all tags that were over the corresponding source text.
+#
+# This sub does that, and returns the string, minus traces, plus reinserted
+# placeholders.
+
+sub _reinsert_placeholders {
+ my ($traced_text, @tags_over_token) = @_;
+
+ my %cur_open_tags = map {$_ => 1} @{$tags_over_token[0]};
+ my $output_text = '';
+
+ while ($traced_text =~ s/^(.+?)\s*\|(\d+)-+(\d+)\|\s*//) {
+ my ($segment, $from, $to) = ($1, $2+1, $3+1);
+
+ # list all tags that cover the source segment
+ my %segment_tags = map {$_ => 1} map {
+ @{$tags_over_token[$_]};
+ } ($from .. $to);
+
+ $output_text .= " MOSESCLOSETAG$_ "
+ foreach (grep !$segment_tags{$_}, keys %cur_open_tags);
+ $output_text .= " MOSESOPENTAG$_ "
+ foreach (grep !$cur_open_tags{$_}, keys %segment_tags);
+ %cur_open_tags = %segment_tags;
+
+ $output_text .= " $segment ";
+ }
+
+ my %final_tags = map {$_ => 1} @{$tags_over_token[-1]};
+ $output_text .= " MOSESCLOSETAG$_ "
+ foreach (grep !$final_tags{$_}, keys %cur_open_tags);
+ $output_text .= " MOSESOPENTAG$_ "
+ foreach (grep !$cur_open_tags{$_}, keys %final_tags);
+
+ $output_text .= $traced_text;
+ return $output_text;
+}
+
+
+# Finally this one replaces the placeholders by the actual tags.
+
+sub replace_placeholders_by_tags {
+ my ($buf_text, @buf_tag_index) = @_;
+
+ # replace the placeholders by the original tags
+ $buf_text =~ s{MOSES(OPEN|CLOSE)TAG(\d+)}{
+ &print_tag (@{$buf_tag_index[$2]}, $1 eq 'CLOSE');
+ }ge;
+
+ return $buf_text;
+}
+
+#------------------------------------------------------------------------------
+# Interfaces to actual plain-text translators. These take a plain string and
+# return a traced (Moses-style) translation
+
+
+# This sub is used when you want to debug everything in this script except the
+# actual translation. Translates to Pig Latin.
+
+sub _translate_text_pig_latin {
+ my ($text) = @_;
+
+ $text =~ s/\b([bcdfhj-np-tv-z]+)([a-z]+)/
+ ($1 eq ucfirst $1 ? ucfirst $2 : $2) .
+ ($2 eq lc $2 ? lc $1 : $1) .
+ 'ay';
+ /gei;
+
+ # insert fake traces
+ my $i = -1;
+ $text .= ' ';
+ $text =~ s/\s+/$i++; " |$i-$i| "/ge;
+
+ return $text;
+}
+
+
+# This one, given a handle to a Moses subprocess, will use that to translate
+# the text. Not much to see here actually.
+
+sub _translate_text_moses {
+ my ($text, $moses) = @_;
+
+ my $traced_text = $moses->do_line ($text);
+ unless ($traced_text) {
+ my @tokens = split /\s+/, $text;
+
+ # insert a fake trace if for some reason moses didn't return one
+ # (which most likely indicates something is quite wrong)
+ $traced_text = $text . " |0-$#tokens|";
+ }
+
+ return $traced_text;
+}
+
+#------------------------------------------------------------------------------
+# basic HTML manipulation subs
+
+sub make_link_absolute {
+ my ($tag_name, $attr_hash, $attr_name) = @_;
+
+ # make it absolute
+ $attr_hash->{$attr_name} = URI->new_abs
+ ($attr_hash->{$attr_name}, $base_url)->as_string;
+
+ # make it point back to us if it's a link
+ if ($tag_name eq 'a') {
+ $attr_hash->{$attr_name} = 'index.cgi?url=' .
+ uri_escape ($attr_hash->{$attr_name});
+ $attr_hash->{target} = '_top';
+ }
+}
+
+sub print_tag {
+ my ($tag_name, $attr_hash, $closing) = @_;
+ my $self_close = $attr_hash->{'/'} ? 1 : 0;
+
+ return '<' . ($closing ? '/' : '') . $tag_name .
+ ($closing ? '' : join ('', map {
+ my $v = $attr_hash->{$_};
+ $v =~ s/\"/&\#34;/g;
+ " $_=\"$v\"";
+ } keys %{$attr_hash})) .
+ ($self_close ? ' /' : '') . '>';
+}
+
+sub add_original_text_popup {
+ my ($input_text, $output_html) = @_;
+
+ $input_text =~ s/\"/&\#34;/g;
+ $input_text =~ s/MOSES(?:OPEN|CLOSE)TAG\d+//g;
+ $input_text =~ s/^\s+//;
+ $input_text =~ s/\s+$//;
+ $input_text =~ s/\s+/ /g;
+
+ # Using this technique for displaying the source text pop-up means we don't
+ # have to fiddle with JavaScript, but it also means you need the LongTitles
+ # extension installed if using Firefox.. *I* happen to have it, so..
+ return "<span title=\"$input_text\">$output_html</span>";
+}
+
+#------------------------------------------------------------------------------
+# conclusion
+
+# stop the top frame counter
+my $num_sentences = grep ref $_, @segments;
+print "<script> top.numSentences = $num_sentences </script>\n";
+
+#------------------------------------------------------------------------------
|