path: root/test/RunTests

#!/usr/bin/perl -w
#
# Test suite for vw:
#
# You may add arbitrary (train/test/varying-options) tests
# by adding data files and their expected reference STDOUT and STDERR
#
# See __DATA__ below for how to add more tests
#
require 5.014;

use Getopt::Std;
use vars qw($opt_d $opt_D $opt_c $opt_e $opt_f
            $opt_E $opt_o $opt_w $opt_y
            $opt_v $opt_V);

my $Epsilon = 1e-4;

my $VW;
my $LDA;

# External utilities we use. See init() for Windows specific actions.
my $Diff = 'diff';
my $Cat = 'cat';

# -V prefixes valgrind like this, we should adjust the default
# options over time to what looks most useful.
my $Valgrind = 'valgrind --quiet --error-exitcode=100' .
                ' --track-origins=yes';

my $DiffOpts = '-N --minimal --suppress-common-lines --ignore-all-space --strip-trailing-cr --side-by-side -W 160';

my @PathAdd = qw(. .. ../vowpalwabbit);

my @ToTest = ();

sub v($;@) {
    my $verbose_level = shift @_;
    return unless ($opt_v >= $verbose_level);

    if (@_ == 1) {
        print STDERR @_;
    } else {
        printf STDERR @_;
    }
}

sub usage(@) {
    print STDERR @_, "\n" if (@_);

    die "Usage: $0 [options] [testno...] [vw-executable]
    By default will run against the 1st 'vw' executable found in:
        @PathAdd  \$PATH

    Options:
        -c      print test-suite commands before running them
        -d      print diff output on significant diff failure
        -D      print diff output even if it is not significant
        -e      Abort on first diff error
        -w      Ignore white-space differences (diff --ignore-space-change)
        -f      Ignore small (< $Epsilon) floating-point differences (fuzzy compare)
        -E<e>   Tolerance epsilon <e> for fuzzy compares (default $Epsilon)
        -o      Overwrite reference file with new/different result
        -y      On error, copy bad files to (eg stderr.test21) for later comparison
        -v<L>   Verbosity <L> (small integer) is verbosity level
        -V      apply valgrind to vw commands

    [testno...]   Optional integer args: explicit test numbers (skip others)
";
}

sub mysystem {
    my $cmd = shift;
    v(1, "%s\n", $cmd);
    system($cmd);
}

sub valgrind_errfile($) {
    my $testno = shift;
    "Test-$testno.valgrind-err";
}

#
# which vw executable to test against
#
sub which_vw() {
    if (@ARGV == 1 || @ARGV == 2) {
        my $exe = $ARGV[0];
        if (-f $exe && -x $exe) {
            printf STDERR "Testing vw: %s\n", $exe;
            return $exe;
        } else {
            usage("$0: argument $exe: not an executable file");
        }
    } elsif (@ARGV == 0) {
        foreach my $dir (@PathAdd, split(':', $ENV{PATH})) {
            my $exe = "$dir/vw";
            if (-x $exe) {
                printf STDERR "Testing vw: %s\n", $exe;
                return $exe;
            }
        }
    }
    usage("can't find a 'vw' executable to test on");
}

sub which_lda() {
    if (@ARGV == 2) {
        my $exe = $ARGV[1];
        if (-f $exe && -x $exe) {
            printf STDERR "Testing lda: %s\n", $exe;
            return $exe;
        } else {
            usage("$0: argument $exe: not an executable file");
        }
    } elsif (@ARGV == 0 || @ARGV == 1) {
        foreach my $dir (@PathAdd, split(':', $ENV{PATH})) {
            my $exe = "$dir/vw";
            if (-x $exe) {
                printf STDERR "Testing lda: %s\n", $exe;
                return $exe;
            }
        }
    }
    usage("can't find a 'lda' executable to test on");
}

sub init() {
    $0 =~ s{.*/}{};
    getopts('wcdDefyE:ov:V') || usage();
    $opt_v = 0 unless (defined $opt_v and $opt_v);

    print STDERR "testing $^O ";
    if ($^O =~ /MSWin/i) {
        v(1, "OS is $^O\n");
        # On MS Windows we need to change paths to external executables
        # Assumes cygwin is installed
        $ENV{'PATH'} .= ':/cygdrive/c/cygwin/bin';
        # And just to be safe (probably not needed):
        $Diff  = 'c:\cygwin\bin\diff.exe';
        $Cat   = 'c:\cygwin\bin\cat.exe';
    }
    elsif ($^O =~ /cygwin/i){
        v("OS is $^O\n");
        # On MS Windows we need to change paths to external executables
        # Assumes cygwin is installed
        $ENV{'PATH'} .= ':/cygdrive/c/cygwin/bin';
        # And just to be safe (probably not needed):
        $Diff  = 'c:/cygwin/bin/diff.exe';
        $Cat   = 'c:/cygwin/bin/cat.exe';
    }
    $Epsilon = $opt_E if ($opt_E);
    $Diff .= ' --ignore-space-change' if ($opt_w);
    my @num_args = ();
    my @exe_args = ();
    foreach my $arg (@ARGV) {
        if ($arg =~ /^\d+$/) {  # a test number
            push(@num_args, $arg);
            next;
        }
        push(@exe_args, $arg);
    }
    if (@num_args) {
        @ToTest = sort { $a <=> $b } @num_args;
        # add dummy element so we don't become empty on last test
        push(@ToTest, -1);
    }
    @ARGV = @exe_args;

    $VW = which_vw();
    $LDA = which_lda();
}

sub copy_file {
    my ($src_file, $dst_file) = @_;
    use File::Copy;
    print STDERR "\t\t-> copying output to $dst_file\n";
    copy($src_file, $dst_file);
}

sub trim_spaces($) {
    my $str = shift;
    $str =~ s/^\s+//;
    $str =~ s/\s+$//;
    $str;
}

#
# ref_file($default_name)
#   Reference file existence: if we're on Windows, AND
#   an alternate reference-file exists, give precedence
#   to the alternate file (file with a '-mswin' suffix.)
#
sub ref_file($) {
    my $file = shift;
    if ($^O =~ /MSWin/i) {
        my $win_reffile = "$file-mswin";
        if (-e $win_reffile) {
            return $win_reffile;
        }
    }
    $file;
}

# __DATA__ test counter
my $TestNo = 0;

sub next_test() {
    my ($cmd, $out_ref, $err_ref, $pred_ref, $pred);

    $TestNo++;
    while (! eof(DATA)) {
        my $line = <DATA>;
        last if (defined($line) && $line =~ /^\s*$/);

        next if ($line =~ /^\s*#/);  # skip comment lines

        if ($line =~ /{VW}/) {
            # The command line
            $cmd = trim_spaces($line);
            $cmd =~ s/{VW}/$VW/;
            if ($cmd =~ /\s-p\s+(\S+)/) {
                # -p predict_file
                $pred = $1;
            }
            next;
        }
        if ($line =~ /{LDA}/) {
            # The command line
            $cmd = trim_spaces($line);
            $cmd =~ s/{LDA}/$LDA/;
            if ($cmd =~ /\s-p\s+(\S+)/) {
                # -p predict_file
                $pred = $1;
            }
            next;
        }
        if ($line =~ m/\.stdout\b/) {
            $out_ref = ref_file(trim_spaces($line));
            next;
        }
        if ($line =~ /\.stderr\b/) {
            $err_ref = ref_file(trim_spaces($line));
            next;
        }
        if ($line =~ /\.predict\b/) {
            $pred_ref = ref_file(trim_spaces($line));
            next;
        }
        # if we get here it is some unrecognized pattern
        printf STDERR "Unrecognized test spec line:\n\t%s\n", $line;
        print STDERR "Test lines must match one of the following patterns:\n";
        print STDERR "\tCommand to run:    {VW}\n";
        print STDERR "\tstdout reference:  *.stdout\n";
        print STDERR "\tstderr reference:  *.stderr\n";
        print STDERR "\tpredict reference: *.predict\n";
    }
    if (eof(DATA) && !defined $cmd) {
        return (undef, undef, undef, undef);
    }

    unless (defined $cmd) {
        die "$0: test $TestNo: command is undefined\n";
    }
    unless (defined $err_ref) {
        die "$0: test $TestNo: stderr ref: undefined\n";
    }
    # print STDERR "next_test: (\$cmd, $out_ref, $err_ref, $pred_ref, $pred)\n";
    if ($opt_V) {
        $cmd = sprintf("%s --log-file='%s' %s",
                        $Valgrind, valgrind_errfile($TestNo), $cmd);
    }
    ($cmd, $out_ref, $err_ref, $pred_ref, $pred);
}

#
# If the difference is small (least significant digits of numbers)
# treat it as ok. It may be a result of 32 vs 64 bit calculations.
#
use Scalar::Util qw(looks_like_number);

sub lenient_array_compare($$) {
    my ($w1_ref, $w2_ref) = @_;
    my (@w1) = @$w1_ref;
    my (@w2) = @$w2_ref;

    # print STDERR "lenient_array_compare: (@w1) (@w2)\n";
    return 1 if ($#w1 != $#w2); # arrays not of same size
    my $nelem = scalar @w1;
    for (my $i = 0; $i < $nelem; $i++) {
        my ($word1, $word2) = ($w1[$i], $w2[$i]);
        # print STDERR "\t$word1 == $word2 ?\n";
        next if ($word1 eq $word2);

        # There's some difference, is it significant?
        return 1 unless (looks_like_number($word1));
        return 1 unless (looks_like_number($word2));

        my $delta = abs($word1 - $word2);

        if ($delta > $Epsilon) {
            # We have a 'big enough' difference, but this difference
            # may still not be meaningful in all contexts:

            # Big numbers should be compared by ratio rather than
            # by difference

            # Must ensure we can divide (avoid div-by-0)
            if (abs($word2) <= 1.0) {
                # If numbers are so small (close to zero),
                # ($delta > $Epsilon) suffices for deciding that
                # the numbers are meaningfully different
                return 1;
            }
            # Now we can safely divide (since abs($word2) > 0)
            # and determine the ratio difference from 1.0
            my $ratio_delta = abs($word1/$word2 - 1.0);
            return 1 if ($ratio_delta > $Epsilon);
        }
    }
    # print STDERR "lenient_array_compare: no meaningful difference\n";
    return 0; # no meaningful difference
}

sub diff_lenient_float($$) {
    my ($reffile, $outfile) = @_;
    my $status = 0;

    my $tmpf = 'lenient-diff.tmp';
    mysystem("$Diff $DiffOpts $reffile $outfile >$tmpf");
    $status = $? >> 8;
    v(2, "diff produced $tmpf: status=$status\n");
    if (-s $tmpf) {
        # The diff has something in it.
        my $fuzzy_status = 0;   # assume innocent till proven guilty
        open(my $sdiff, $tmpf) || die "$0: diff_lenient_float: $tmpf: $!\n";
        while (<$sdiff>) {
            chomp;
            my ($line1, $line2) = split(/\s*\|\s*/, $_);

            v(3, "line1: $line1\n");
            v(3, "line2: $line2\n");

            # Break lines into tokens/words
            my (@w1) = split(' ', $line1);
            my (@w2) = split(' ', $line2);
            if (lenient_array_compare(\@w1, \@w2) != 0) {
                $fuzzy_status = 1;
                last;
            }
        }
        close $sdiff;
        $status = $fuzzy_status;
    }
    $status;
}

#
# perl internal way to emulate 'touch'
#
sub touch(@) {
    my $now = time;
    utime $now, $now, @_;
}

sub diff($$) {
    my ($reffile, $outfile) = @_;
    my $status = 0;
    $reffile = '' unless (defined $reffile);

    # Special case, empty file w/o reference is not considered a failure.
    # This is a most common case with stdout.
    unless (-e $reffile) {
        if (-s $outfile > 0) {
            warn "$0: test $TestNo: stdout ref: $reffile: $!\n";
            exit 1 if ($opt_e);
            return 2 unless ($opt_o);
        } else {
            # Empty output without a ref is not considered a failure
            v(1, "$0: test $TestNo: empty output w/o reference: ignored.\n");
            return 0;
        }
    }

    # Actually run the diff
    my $diff_cmd = "$Diff $DiffOpts $reffile $outfile";
    mysystem("$diff_cmd >diff.tmp");
    $status = $? >> 8;
    v(2, "$diff_cmd >diff.tmp: status=$status\n");

    if (-s 'diff.tmp') {
        # There's some difference
        v(2, "diff.tmp has something in it. Is it meaningful?\n");

        if ($opt_f && -e $reffile && -e $outfile &&
            diff_lenient_float($reffile, $outfile) == 0) {

            print STDERR "$0: test $TestNo: minor (<$Epsilon) precision differences ignored\n";
            $status = 0;
        }
        if ($opt_D or ($opt_d && $status)) {
            # Print the diff only iff:
            #   1) -D is in effect  OR
            #   2) -d is in effect and diff is significant
            printf STDERR "--- %s\n", $diff_cmd;
            mysystem("$Cat diff.tmp")
        }
        if ($opt_o) {
            print STDERR "-o: overwriting reference:\n";

            if (-e $reffile) {
                print STDERR "\t$reffile -> $reffile.prev\n";
                rename($reffile, "$reffile.prev") ||
                    die "FATAL: rename($reffile, $reffile.prev): $!\n";
            }
            print STDERR "\t$outfile -> $reffile\n";
            rename($outfile, $reffile) ||
                die "FATAL: rename($outfile, $reffile): $!\n";

            unless ($opt_e) {
                $status = 0;
            }
        }
    }
    $status;
}

#
# check_for_time_regression()
#   Compare last overall time to run to current to catch
#   performance regressions
#
my $LastTimeFile = 'RunTests.last.times';

sub write_times($@) {
    my ($file, @times) = @_;
    open(my $fh, ">$file") || die "$0: can't open(>$file): $!\n";
    print $fh join(' ', @times), "\n";
    close $fh;
}
sub read_times($) {
    my ($file) = @_;
    open(my $fh, $file) || die "$0: can't open($file): $!\n";
    my $line = <$fh>; chomp $line;
    close $fh;
    return (split(' ', $line));
}

sub check_for_time_regression() {
    my $tolerate_regress = 1.02;
    my $pct_change = 0.0;
    my ($overall_time0, $overall_time1);
    my ($user0, $system0, $cuser0, $csystem0);
    my ($user1, $system1, $cuser1, $csystem1) = times;
    $overall_time1 = $cuser1 + $csystem1;

    if (-e $LastTimeFile) {
        ($user0, $system0, $cuser0, $csystem0) = read_times($LastTimeFile);
        if (!(defined $csystem0) or !(defined $cuser0)) {
            die "$0: undefined times in saved times file: $LastTimeFile," .
                    " try removing it\n"
        }
        $overall_time0 = $cuser0 + $csystem0;
        $pct_change = 100 * ($overall_time1 - $overall_time0) / (1e-4+$overall_time0);

        if ($overall_time0 == 0) {
            die "$0: Bad times in saved times file: $LastTimeFile," .
                    " try removing it\n"
        } elsif ($overall_time1/$overall_time0 > $tolerate_regress) {
            printf STDERR "$0: RUNTIME REGRESSION: " .
                    "%.2f sec vs last time %.2f sec. (%.2f%% worse)\n",
                    $overall_time1, $overall_time0, $pct_change;
        }
    }
    write_times($LastTimeFile, $user1, $system1, $cuser1, $csystem1);
    printf STDERR
        "$0 runtime: user %g, system %g, total %g sec (%+.2f%% vs. last)\n",
                $cuser1, $csystem1, $overall_time1, $pct_change;
}

sub run_tests() {
    print STDERR "$0: '-D' to see any diff output\n"
        unless ($opt_D);
    print STDERR "$0: '-d' to see only significant diff output\n"
        unless ($opt_d);
    print STDERR "$0: '-o' to force overwrite references\n"
        unless ($opt_o);
    print STDERR "$0: '-e' to abort on first failure\n"
        unless ($opt_e);

    my ($cmd, $out_ref, $err_ref, $pred_ref);
    my ($outf, $errf, $predf);

    mkdir('models', 0755) unless (-d 'models');

    unlink(glob('*.tmp'));
    unlink(glob('*.cache'));
    unlink(glob('*/*.cache'));

    while (($cmd, $out_ref, $err_ref, $pred_ref, $predf) = next_test()) {
        last unless (defined $cmd);
        if (@ToTest) {
            if ($ToTest[0] != $TestNo) {
                # warn "$0: test $TestNo: skipped\n";
                next;
            } else {
                shift(@ToTest);
            }
        }

        ($outf, $errf) = ('stdout.tmp', 'stderr.tmp');
        unlink $outf, $errf;    # just in case, to be safer

        # run the test
        print STDERR "Test $TestNo: ($cmd) >$outf 2>$errf\n" if ($opt_c);
        mysystem("($cmd) >$outf 2>$errf");
        my $status = $? >> 8;
        if ($status) {
            if ($opt_V && $status == 100) {
                my $errfile = valgrind_errfile($TestNo);
                warn "$0: test $TestNo: FAILED: valgrind errors in $errfile\n";
            } else {
                warn "$0: test $TestNo: '$cmd' failed: status=$status\n";
            }
            exit $status if ($opt_e);
            next;
        }

        # command succeded
        # -- compare stdout
        $status = diff($out_ref, $outf);
        if ($status) {
            printf STDERR "%s: test %d: FAILED: ref(%s) != stdout(%s)\n",
                $0, $TestNo, $out_ref, $outf;
            if ($opt_y) { copy_file($outf, $outf . '.test' . $TestNo); }
            exit $status if ($opt_e);
        } else {
            if (defined $out_ref) {
                print STDERR "$0: test $TestNo: stdout OK\n";
            } else {
                v(1, "$0: test $TestNo: stdout OK (no reference)\n");
            }
        }

        # -- compare stderr
        if (! -e $err_ref  and  ! $opt_o) {
            print STDERR "$0: test $TestNo: FAILED: stderr ref: $err_ref: $!\n";
            exit 1 if ($opt_e);
            next;
        }
        $status = diff($err_ref, $errf);
        if ($status) {
            printf STDERR "%s: test %d: FAILED: ref(%s) != stderr(%s)\n",
                $0, $TestNo, $err_ref, $errf;
            if ($opt_y) { copy_file($errf, "$errf.test.$TestNo"); }
            exit $status if ($opt_e);
        } else {
            print STDERR "$0: test $TestNo: stderr OK\n";
        }
        # -- compare predict
        next unless (defined $pred_ref);
        $predf = 'predict.tmp' unless (defined $predf);
        $status = diff($pred_ref, $predf);
        if ($status) {
            printf STDERR "%s: test %d: FAILED: ref(%s) != predict(%s)\n",
                $0, $TestNo, $pred_ref, $predf;
            if ($opt_y) { copy_file($predf, "$predf.test.$TestNo"); }
            exit $status if ($opt_e);
        } else {
            print STDERR "$0: test $TestNo: predict OK\n";
        }
    }
    check_for_time_regression();
}

# --- main
init();
run_tests();

#
# Add tests below the __DATA__ line
# Each test is a series of lines, terminated by an empty line (or EOF)
#
# Each test is comprised of:
#   1st line-item is the command to run, {VW} represents the vw
#   executable.
#
#   By default, 'vw' in the parent dir (../vw) is tested.
#   To run against a different reference executable, just pass the
#   executable as an argument to RunTests
#
# The next (output) line-items are reference files to compare outputs to:
#    The expected (reference file) standard output
#    The expected (reference file) standard error
#    The expected (reference file) for predictions (-p ...)
#    [The above reference files can come in any order.
#     Their 'type' is determined by their extensions:
#            .stdout  .stderr  .predict
#    ]
#
# All filenames are relative to this (test) directory
#
# The temporary output file-names (as opposed to the reference ones)
# are implicit:
#    (stdout.tmp  stderr.tmp  predict.tmp)
# Except: if -p ... appears in the command, it will be used as the
# (explicit) predictions file.
#
# Windows note:
#
# Due to differences in Random-Number-Generators in Windows,
# floating-point outputs may differ in some tests (not all).
#
# To minimize the need for changes (leverage existing tests and
# reference files as much as possible), on Windows we check for
# existance of files with '-mswin' suffix:
#   *.stderr-mswin
#   *.stdout-mswin
#   *.predict-mswin
# and if any of them exists, we use it instead.
#
__DATA__
# Test 1:
{VW} -k -l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.dat -f models/0001.model -c --passes 8 --invariant --ngram 3 --skips 1 --holdout_off
    train-sets/ref/0001.stderr

# Test 2: checking predictions as well
{VW} -k -t train-sets/0001.dat -i models/0001.model -p 001.predict.tmp --invariant
    test-sets/ref/0001.stderr
    pred-sets/ref/0001.predict

# Test 3: without -d, training only
{VW} -k train-sets/0002.dat -f models/0002.model --invariant
    train-sets/ref/0002.stderr

# Test 4: same, with -d
{VW} -k -d train-sets/0002.dat -f models/0002.model --invariant
    train-sets/ref/0002.stdout
    train-sets/ref/0002.stderr

# Test 5: add -q .., adaptive, and more (same input, different outputs)
{VW} -k --initial_t 1 --adaptive --invariant -q Tf -q ff -f models/0002a.model train-sets/0002.dat
    train-sets/ref/0002a.stderr

# Test 6: run predictions on Test 4 model
# Pretending the labels aren't there
{VW} -k -t -i models/0002.model -d train-sets/0002.dat -p 0002b.predict
    test-sets/ref/0002b.stderr
    pred-sets/ref/0002b.predict

# Test 7: using normalized adaptive updates and a low --power_t
{VW} -k --power_t 0.45 -f models/0002c.model train-sets/0002.dat
    train-sets/ref/0002c.stderr

# Test 8: predicts on test 7 model
{VW} -k -t -i models/0002c.model -d train-sets/0002.dat -p 0002c.predict
    test-sets/ref/0002c.stderr
    pred-sets/ref/0002c.predict

# Test 9: label-dependent features with csoaa_ldf
{VW} -k -c -d train-sets/cs_test.ldf -p cs_test.ldf.csoaa.predict --passes 10 --invariant --csoaa_ldf multiline --holdout_off
    train-sets/ref/cs_test.ldf.csoaa.stderr
    train-sets/ref/cs_test.ldf.csoaa.predict

# Test 10: label-dependent features with wap_ldf
{VW} -k -c -d train-sets/cs_test.ldf -p cs_test.ldf.wap.predict --passes 10 --invariant --wap_ldf multiline --holdout_off
    train-sets/ref/cs_test.ldf.wap.stderr
    train-sets/ref/cs_test.ldf.wap.predict

# Test 11: one-against-all
{VW} -k --oaa 10 -c --passes 10 train-sets/multiclass --holdout_off
    train-sets/ref/oaa.stderr

# Test 12: Error Correcting Tournament
{VW} -k --ect 10 --error 3 -c --passes 10 --invariant train-sets/multiclass --holdout_off
    train-sets/ref/multiclass.stderr

# Test 13: Run searn on wsj_small for 12 passes, 4 passes per policy, extra features
{VW} -k -c -d train-sets/wsj_small.dat.gz --passes 12 --invariant --searn_passes_per_policy 4 --searn_task sequence --searn 5 --searn_history 2 --searn_bigrams --searn_features 1 --quiet --holdout_off
    train-sets/ref/searn_wsj.stderr

# Test 14: Run searn (wap) on wsj_small for 2 passes, 1 pass per policy, extra features
{VW} -k -b 19 -c -d train-sets/wsj_small.dat.gz --passes 2 --invariant --searn_passes_per_policy 1 --searn_task sequence --searn 5 --wap 5 --searn_history 2 --searn_bigrams --searn_features 1 --quiet --holdout_off
    train-sets/ref/searn_wsj2.dat.stdout
    train-sets/ref/searn_wsj2.dat.stderr

# Test 15: LBFGS on zero derivative input
{VW} -k -c -d train-sets/zero.dat --loss_function=squared -b 20 --bfgs --mem 7 --passes 5 --l2 1.0 --holdout_off
    train-sets/ref/zero.stdout
    train-sets/ref/zero.stderr

# Test 16: LBFGS early termination
{VW} -k -c -d train-sets/rcv1_small.dat --loss_function=logistic -b 20 --bfgs --mem 7 --passes 20 --termination 0.001 --l2 1.0 --holdout_off
    train-sets/ref/rcv1_small.stdout
    train-sets/ref/rcv1_small.stderr

# Test 17: Run LDA with 100 topics on 1000 Wikipedia articles
{LDA} -k --lda 100 --lda_alpha 0.01 --lda_rho 0.01 --lda_D 1000 -b 13 --minibatch 128 --invariant train-sets/wiki1K.dat
    train-sets/ref/wiki1K.stderr

# Test 18: Run searn on seq_small for 12 passes, 4 passes per policy
{VW} -k -c -d train-sets/seq_small --passes 12 --invariant --searn_passes_per_policy 4 --searn 4 --searn_task sequence --quiet --holdout_off
    train-sets/ref/searn_small.stderr

# Test 19: neural network 3-parity with 2 hidden units
{VW} -k -c -d train-sets/3parity --hash all --passes 3000 -b 16 --nn 2 -l 10 --invariant -f models/0021.model --random_seed 15 --quiet --holdout_off
    train-sets/ref/3parity.stderr

# Test 20: neural network 3-parity with 2 hidden units (predict)
{VW} -d train-sets/3parity --hash all -t -i models/0021.model -p 0022.predict.tmp
    pred-sets/ref/0022.stderr
    pred-sets/ref/0022.predict

# Test 21: cubic features -- on a parity test case
{VW} -k -c -f models/xxor.model train-sets/xxor.dat --cubic abc --passes 100 --holdout_off --progress 1.33333
    train-sets/ref/xxor.stderr

# Test 22: matrix factorization -- training
{VW} -k -d train-sets/ml100k_small_train -b 16 -q ui --rank 10 --l2 2e-6 --learning_rate 0.05 --passes 2 --decay_learning_rate 0.97 --power_t 0 -f movielens.reg --cache_file movielens.cache --loss_function classic --holdout_off
    train-sets/ref/ml100k_small.stdout
    train-sets/ref/ml100k_small.stderr

# Test 23: matrix factorization -- testing
{VW} -i movielens.reg -t -d test-sets/ml100k_small_test
    test-sets/ref/ml100k_small.stdout
    test-sets/ref/ml100k_small.stderr

# Test 24: active-learning -- training
{VW} -k --active_simulation --active_mellowness 0.000001 -d train-sets/rcv1_small.dat -l 10 --initial_t 10 --progress 1.25
    train-sets/ref/active-simulation.t24.stderr

# Test 25: bagging -- training regressor
{VW} -k -d train-sets/0002.dat -f models/bs.reg.model --bs 4 -p bs.reg.predict
    train-sets/ref/bs.reg.stderr
    train-sets/ref/bs.reg.predict

# Test 26: bagging -- predicting with bagged regressor
{VW} -d train-sets/0002.dat -i models/bs.prreg.model -p bs.prreg.predict -t
    train-sets/ref/bs.prreg.stderr
    train-sets/ref/bs.prreg.predict

# Test 27: bagging -- binary classifiers
{VW} -d train-sets/0001.dat -f models/bs.vote.model --bs 4 --bs_type vote -p bs.vote.predict
    train-sets/ref/bs.vote.stderr
    train-sets/ref/bs.vote.predict

# Test 28: bagging -- predict with bagged classifier
{VW} -d train-sets/0001.dat -i models/bs.prvote.model -p bs.prvote.predict -t
    train-sets/ref/bs.prvote.stderr
    train-sets/ref/bs.prvote.predict

# Test 29: affix features
{VW} -d train-sets/affix_test.dat -k -c --passes 10 --holdout_off --affix -2
    train-sets/ref/affix_test.stderr

# Test 30: train --l1 regularized model
{VW} -d train-sets/0001.dat -f models/mask.model --invert_hash mask.predict --l1 0.01
    train-sets/ref/mask.stderr

# Test 31: train model using --feature_mask
{VW} -d train-sets/0001.dat --invert_hash remask.predict --feature_mask models/mask.model -f models/remask.model
    train-sets/ref/remask.stderr

# Test 32: train model using --feature_mask and --initial_regressor
{VW} -d train-sets/0001.dat --feature_mask models/mask.model -i models/remask.model
    train-sets/ref/remask.final.stderr

# Test 33: train model for topk recommender
{VW} -d train-sets/topk.vw -f topk.model -q MF --passes 100 --cache_file topk-train.cache -k --holdout_off
    train-sets/ref/topk-train.stderr

# Test 34: train model for topk recommender
{VW} -P 1 -d train-sets/topk.vw -i topk.model --top 2 -p topk.predict
    train-sets/ref/topk-rec.stderr
    train-sets/ref/topk-rec.predict

# Test 35: non-centered data-set where constant >> 0
#   To test the new --constant option without which performance is very weak
{VW} -k --passes 100 -c --holdout_off --constant 1000 -d train-sets/big-constant.dat
    train-sets/ref/big-constant.stderr

# Test 36: new option: --progress w/ integer arg
{VW} -k -d train-sets/0001.dat --progress 10
    train-sets/ref/progress-10.stderr

# Test 37: new-option: --progress w/ floating-point arg
#           + alternate short form (-P)
{VW} -k -d train-sets/0001.dat -P 0.5
    train-sets/ref/progress-0.5.stderr

# Test 38: --nn without --quiet to avoid nn regressions
#   (Needs to be a simple test, not one sensitive to symmetry breaking)
{VW} -k -d train-sets/0001.dat --nn 1
    train-sets/ref/nn-1-noquiet.stderr